diff options
242 files changed, 11900 insertions, 4043 deletions
diff --git a/contrib/compiler-rt/LICENSE.TXT b/contrib/compiler-rt/LICENSE.TXT index 0134694e4e5f..1c94ad5d8911 100644 --- a/contrib/compiler-rt/LICENSE.TXT +++ b/contrib/compiler-rt/LICENSE.TXT @@ -14,7 +14,7 @@ Full text of the relevant licenses is included below. University of Illinois/NCSA Open Source License -Copyright (c) 2009-2018 by the contributors listed in CREDITS.TXT +Copyright (c) 2009-2019 by the contributors listed in CREDITS.TXT All rights reserved. diff --git a/contrib/compiler-rt/include/sanitizer/allocator_interface.h b/contrib/compiler-rt/include/sanitizer/allocator_interface.h index 89f328301db3..e44c4a13ed67 100644 --- a/contrib/compiler-rt/include/sanitizer/allocator_interface.h +++ b/contrib/compiler-rt/include/sanitizer/allocator_interface.h @@ -82,7 +82,6 @@ extern "C" { Currently available with ASan only. */ void __sanitizer_purge_allocator(void); - #ifdef __cplusplus } // extern "C" #endif diff --git a/contrib/compiler-rt/include/sanitizer/common_interface_defs.h b/contrib/compiler-rt/include/sanitizer/common_interface_defs.h index d11cb1addc27..bf015eb237cd 100644 --- a/contrib/compiler-rt/include/sanitizer/common_interface_defs.h +++ b/contrib/compiler-rt/include/sanitizer/common_interface_defs.h @@ -124,6 +124,12 @@ extern "C" { // Symbolizes the supplied 'pc' using the format string 'fmt'. // Outputs at most 'out_buf_size' bytes into 'out_buf'. + // If 'out_buf' is not empty then output is zero or more non empty C strings + // followed by single empty C string. Multiple strings can be returned if PC + // corresponds to inlined function. Inlined frames are printed in the order + // from "most-inlined" to the "least-inlined", so the last frame should be the + // not inlined function. + // Inlined frames can be removed with 'symbolize_inline_frames=0'. // The format syntax is described in // lib/sanitizer_common/sanitizer_stacktrace_printer.h. void __sanitizer_symbolize_pc(void *pc, const char *fmt, char *out_buf, diff --git a/contrib/compiler-rt/include/sanitizer/hwasan_interface.h b/contrib/compiler-rt/include/sanitizer/hwasan_interface.h index 0c306cf279e2..1affd875a11b 100644 --- a/contrib/compiler-rt/include/sanitizer/hwasan_interface.h +++ b/contrib/compiler-rt/include/sanitizer/hwasan_interface.h @@ -19,6 +19,12 @@ #ifdef __cplusplus extern "C" { #endif + // Initialize shadow but not the rest of the runtime. + // Does not call libc unless there is an error. + // Can be called multiple times, or not at all (in which case shadow will + // be initialized in compiler-inserted __hwasan_init() call). + void __hwasan_shadow_init(void); + // This function may be optionally provided by user and should return // a string containing HWASan runtime options. See asan_flags.h for details. const char* __hwasan_default_options(void); @@ -26,6 +32,51 @@ extern "C" { void __hwasan_enable_allocator_tagging(void); void __hwasan_disable_allocator_tagging(void); + // Mark region of memory with the given tag. Both address and size need to be + // 16-byte aligned. + void __hwasan_tag_memory(const volatile void *p, unsigned char tag, + size_t size); + + /// Set pointer tag. Previous tag is lost. + void *__hwasan_tag_pointer(const volatile void *p, unsigned char tag); + + // Set memory tag from the current SP address to the given address to zero. + // This is meant to annotate longjmp and other non-local jumps. + // This function needs to know the (almost) exact destination frame address; + // clearing shadow for the entire thread stack like __asan_handle_no_return + // does would cause false reports. + void __hwasan_handle_longjmp(const void *sp_dst); + + // Libc hook for thread creation. Should be called in the child thread before + // any instrumented code. + void __hwasan_thread_enter(); + + // Libc hook for thread destruction. No instrumented code should run after + // this call. + void __hwasan_thread_exit(); + + // Print shadow and origin for the memory range to stderr in a human-readable + // format. + void __hwasan_print_shadow(const volatile void *x, size_t size); + + // Print one-line report about the memory usage of the current process. + void __hwasan_print_memory_usage(); + + int __sanitizer_posix_memalign(void **memptr, size_t alignment, size_t size); + void * __sanitizer_memalign(size_t alignment, size_t size); + void * __sanitizer_aligned_alloc(size_t alignment, size_t size); + void * __sanitizer___libc_memalign(size_t alignment, size_t size); + void * __sanitizer_valloc(size_t size); + void * __sanitizer_pvalloc(size_t size); + void __sanitizer_free(void *ptr); + void __sanitizer_cfree(void *ptr); + size_t __sanitizer_malloc_usable_size(const void *ptr); + struct mallinfo __sanitizer_mallinfo(); + int __sanitizer_mallopt(int cmd, int value); + void __sanitizer_malloc_stats(void); + void * __sanitizer_calloc(size_t nmemb, size_t size); + void * __sanitizer_realloc(void *ptr, size_t size); + void * __sanitizer_malloc(size_t size); #ifdef __cplusplus } // extern "C" #endif diff --git a/contrib/compiler-rt/include/sanitizer/netbsd_syscall_hooks.h b/contrib/compiler-rt/include/sanitizer/netbsd_syscall_hooks.h index 4c6c6a88af29..b69f3d2be39f 100644 --- a/contrib/compiler-rt/include/sanitizer/netbsd_syscall_hooks.h +++ b/contrib/compiler-rt/include/sanitizer/netbsd_syscall_hooks.h @@ -21,8 +21,8 @@ // DO NOT EDIT! THIS FILE HAS BEEN GENERATED! // // Generated with: generate_netbsd_syscalls.awk -// Generated date: 2018-03-03 -// Generated from: syscalls.master,v 1.291 2018/01/06 16:41:23 kamil Exp +// Generated date: 2018-10-30 +// Generated from: syscalls.master,v 1.293 2018/07/31 13:00:13 rjs Exp // //===----------------------------------------------------------------------===// #ifndef SANITIZER_NETBSD_SYSCALL_HOOKS_H @@ -986,7 +986,15 @@ #define __sanitizer_syscall_post_fpathconf(res, fd, name) \ __sanitizer_syscall_post_impl_fpathconf(res, (long long)(fd), \ (long long)(name)) -/* syscall 193 has been skipped */ +#define __sanitizer_syscall_pre_getsockopt2(s, level, name, val, avalsize) \ + __sanitizer_syscall_pre_impl_getsockopt2( \ + (long long)(s), (long long)(level), (long long)(name), (long long)(val), \ + (long long)(avalsize)) +#define __sanitizer_syscall_post_getsockopt2(res, s, level, name, val, \ + avalsize) \ + __sanitizer_syscall_post_impl_getsockopt2( \ + res, (long long)(s), (long long)(level), (long long)(name), \ + (long long)(val), (long long)(avalsize)) #define __sanitizer_syscall_pre_getrlimit(which, rlp) \ __sanitizer_syscall_pre_impl_getrlimit((long long)(which), (long long)(rlp)) #define __sanitizer_syscall_post_getrlimit(res, which, rlp) \ @@ -1752,18 +1760,8 @@ __sanitizer_syscall_post_impl___sigaction_sigtramp( \ res, (long long)(signum), (long long)(nsa), (long long)(osa), \ (long long)(tramp), (long long)(vers)) -#define __sanitizer_syscall_pre_pmc_get_info(ctr, op, args) \ - __sanitizer_syscall_pre_impl_pmc_get_info((long long)(ctr), (long long)(op), \ - (long long)(args)) -#define __sanitizer_syscall_post_pmc_get_info(res, ctr, op, args) \ - __sanitizer_syscall_post_impl_pmc_get_info( \ - res, (long long)(ctr), (long long)(op), (long long)(args)) -#define __sanitizer_syscall_pre_pmc_control(ctr, op, args) \ - __sanitizer_syscall_pre_impl_pmc_control((long long)(ctr), (long long)(op), \ - (long long)(args)) -#define __sanitizer_syscall_post_pmc_control(res, ctr, op, args) \ - __sanitizer_syscall_post_impl_pmc_control( \ - res, (long long)(ctr), (long long)(op), (long long)(args)) +/* syscall 341 has been skipped */ +/* syscall 342 has been skipped */ #define __sanitizer_syscall_pre_rasctl(addr, len, op) \ __sanitizer_syscall_pre_impl_rasctl((long long)(addr), (long long)(len), \ (long long)(op)) @@ -3444,7 +3442,13 @@ void __sanitizer_syscall_post_impl_pathconf(long long res, long long path, void __sanitizer_syscall_pre_impl_fpathconf(long long fd, long long name); void __sanitizer_syscall_post_impl_fpathconf(long long res, long long fd, long long name); -/* syscall 193 has been skipped */ +void __sanitizer_syscall_pre_impl_getsockopt2(long long s, long long level, + long long name, long long val, + long long avalsize); +void __sanitizer_syscall_post_impl_getsockopt2(long long res, long long s, + long long level, long long name, + long long val, + long long avalsize); void __sanitizer_syscall_pre_impl_getrlimit(long long which, long long rlp); void __sanitizer_syscall_post_impl_getrlimit(long long res, long long which, long long rlp); @@ -4001,14 +4005,8 @@ void __sanitizer_syscall_pre_impl___sigaction_sigtramp(long long signum, void __sanitizer_syscall_post_impl___sigaction_sigtramp( long long res, long long signum, long long nsa, long long osa, long long tramp, long long vers); -void __sanitizer_syscall_pre_impl_pmc_get_info(long long ctr, long long op, - long long args); -void __sanitizer_syscall_post_impl_pmc_get_info(long long res, long long ctr, - long long op, long long args); -void __sanitizer_syscall_pre_impl_pmc_control(long long ctr, long long op, - long long args); -void __sanitizer_syscall_post_impl_pmc_control(long long res, long long ctr, - long long op, long long args); +/* syscall 341 has been skipped */ +/* syscall 342 has been skipped */ void __sanitizer_syscall_pre_impl_rasctl(long long addr, long long len, long long op); void __sanitizer_syscall_post_impl_rasctl(long long res, long long addr, diff --git a/contrib/compiler-rt/include/xray/xray_log_interface.h b/contrib/compiler-rt/include/xray/xray_log_interface.h index 5f8b3a4a06a4..399467860e73 100644 --- a/contrib/compiler-rt/include/xray/xray_log_interface.h +++ b/contrib/compiler-rt/include/xray/xray_log_interface.h @@ -158,8 +158,8 @@ struct XRayLogImpl { /// The log initialization routine provided by the implementation, always /// provided with the following parameters: /// - /// - buffer size - /// - maximum number of buffers + /// - buffer size (unused) + /// - maximum number of buffers (unused) /// - a pointer to an argument struct that the implementation MUST handle /// - the size of the argument struct /// @@ -355,25 +355,4 @@ XRayLogFlushStatus __xray_log_process_buffers(void (*Processor)(const char *, } // extern "C" -namespace __xray { - -/// DEPRECATED: Use __xray_log_init_mode(...) instead, and provide flag -/// configuration strings to set the options instead. -/// Options used by the LLVM XRay FDR logging implementation. -struct FDRLoggingOptions { - bool ReportErrors = false; - int Fd = -1; -}; - -/// DEPRECATED: Use __xray_log_init_mode(...) instead, and provide flag -/// configuration strings to set the options instead. -/// Options used by the LLVM XRay Basic (Naive) logging implementation. -struct BasicLoggingOptions { - int DurationFilterMicros = 0; - size_t MaxStackDepth = 0; - size_t ThreadBufferSize = 0; -}; - -} // namespace __xray - #endif // XRAY_XRAY_LOG_INTERFACE_H diff --git a/contrib/compiler-rt/lib/asan/asan_allocator.h b/contrib/compiler-rt/lib/asan/asan_allocator.h index 93d6f29c5bf5..c9b37dc7a6d4 100644 --- a/contrib/compiler-rt/lib/asan/asan_allocator.h +++ b/contrib/compiler-rt/lib/asan/asan_allocator.h @@ -148,6 +148,7 @@ const uptr kAllocatorSpace = 0x600000000000ULL; const uptr kAllocatorSize = 0x40000000000ULL; // 4T. typedef DefaultSizeClassMap SizeClassMap; # endif +template <typename AddressSpaceViewTy> struct AP64 { // Allocator64 parameters. Deliberately using a short name. static const uptr kSpaceBeg = kAllocatorSpace; static const uptr kSpaceSize = kAllocatorSize; @@ -155,37 +156,57 @@ struct AP64 { // Allocator64 parameters. Deliberately using a short name. typedef __asan::SizeClassMap SizeClassMap; typedef AsanMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; + using AddressSpaceView = AddressSpaceViewTy; }; -typedef SizeClassAllocator64<AP64> PrimaryAllocator; +template <typename AddressSpaceView> +using PrimaryAllocatorASVT = SizeClassAllocator64<AP64<AddressSpaceView>>; +using PrimaryAllocator = PrimaryAllocatorASVT<LocalAddressSpaceView>; #else // Fallback to SizeClassAllocator32. static const uptr kRegionSizeLog = 20; static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog; # if SANITIZER_WORDSIZE == 32 -typedef FlatByteMap<kNumRegions> ByteMap; +template <typename AddressSpaceView> +using ByteMapASVT = FlatByteMap<kNumRegions, AddressSpaceView>; # elif SANITIZER_WORDSIZE == 64 -typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap; +template <typename AddressSpaceView> +using ByteMapASVT = + TwoLevelByteMap<(kNumRegions >> 12), 1 << 12, AddressSpaceView>; # endif typedef CompactSizeClassMap SizeClassMap; +template <typename AddressSpaceViewTy> struct AP32 { static const uptr kSpaceBeg = 0; static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; static const uptr kMetadataSize = 16; typedef __asan::SizeClassMap SizeClassMap; static const uptr kRegionSizeLog = __asan::kRegionSizeLog; - typedef __asan::ByteMap ByteMap; + using AddressSpaceView = AddressSpaceViewTy; + using ByteMap = __asan::ByteMapASVT<AddressSpaceView>; typedef AsanMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; -typedef SizeClassAllocator32<AP32> PrimaryAllocator; +template <typename AddressSpaceView> +using PrimaryAllocatorASVT = SizeClassAllocator32<AP32<AddressSpaceView> >; +using PrimaryAllocator = PrimaryAllocatorASVT<LocalAddressSpaceView>; #endif // SANITIZER_CAN_USE_ALLOCATOR64 static const uptr kNumberOfSizeClasses = SizeClassMap::kNumClasses; -typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; -typedef LargeMmapAllocator<AsanMapUnmapCallback> SecondaryAllocator; -typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, - SecondaryAllocator> AsanAllocator; - +template <typename AddressSpaceView> +using AllocatorCacheASVT = + SizeClassAllocatorLocalCache<PrimaryAllocatorASVT<AddressSpaceView>>; +using AllocatorCache = AllocatorCacheASVT<LocalAddressSpaceView>; + +template <typename AddressSpaceView> +using SecondaryAllocatorASVT = + LargeMmapAllocator<AsanMapUnmapCallback, DefaultLargeMmapAllocatorPtrArray, + AddressSpaceView>; +template <typename AddressSpaceView> +using AsanAllocatorASVT = + CombinedAllocator<PrimaryAllocatorASVT<AddressSpaceView>, + AllocatorCacheASVT<AddressSpaceView>, + SecondaryAllocatorASVT<AddressSpaceView>>; +using AsanAllocator = AsanAllocatorASVT<LocalAddressSpaceView>; struct AsanThreadLocalMallocStorage { uptr quarantine_cache[16]; diff --git a/contrib/compiler-rt/lib/asan/asan_errors.cc b/contrib/compiler-rt/lib/asan/asan_errors.cc index 33d0613f79f4..0ecd30dcacfd 100644 --- a/contrib/compiler-rt/lib/asan/asan_errors.cc +++ b/contrib/compiler-rt/lib/asan/asan_errors.cc @@ -125,9 +125,8 @@ void ErrorAllocTypeMismatch::Print() { Decorator d; Printf("%s", d.Error()); Report("ERROR: AddressSanitizer: %s (%s vs %s) on %p\n", - scariness.GetDescription(), - alloc_names[alloc_type], dealloc_names[dealloc_type], - addr_description.addr); + scariness.GetDescription(), alloc_names[alloc_type], + dealloc_names[dealloc_type], addr_description.Address()); Printf("%s", d.Default()); CHECK_GT(dealloc_stack->size, 0); scariness.Print(); diff --git a/contrib/compiler-rt/lib/asan/asan_errors.h b/contrib/compiler-rt/lib/asan/asan_errors.h index 574197ebff86..7ddd7e94e0fc 100644 --- a/contrib/compiler-rt/lib/asan/asan_errors.h +++ b/contrib/compiler-rt/lib/asan/asan_errors.h @@ -110,8 +110,8 @@ struct ErrorFreeNotMalloced : ErrorBase { struct ErrorAllocTypeMismatch : ErrorBase { const BufferedStackTrace *dealloc_stack; - HeapAddressDescription addr_description; AllocType alloc_type, dealloc_type; + AddressDescription addr_description; ErrorAllocTypeMismatch() = default; // (*) ErrorAllocTypeMismatch(u32 tid, BufferedStackTrace *stack, uptr addr, @@ -119,9 +119,8 @@ struct ErrorAllocTypeMismatch : ErrorBase { : ErrorBase(tid, 10, "alloc-dealloc-mismatch"), dealloc_stack(stack), alloc_type(alloc_type_), - dealloc_type(dealloc_type_) { - GetHeapAddressInformation(addr, 1, &addr_description); - }; + dealloc_type(dealloc_type_), + addr_description(addr, 1, false) {} void Print(); }; diff --git a/contrib/compiler-rt/lib/asan/asan_flags.inc b/contrib/compiler-rt/lib/asan/asan_flags.inc index 4af94c56fca0..a9c97d53b31f 100644 --- a/contrib/compiler-rt/lib/asan/asan_flags.inc +++ b/contrib/compiler-rt/lib/asan/asan_flags.inc @@ -152,8 +152,6 @@ ASAN_FLAG(const char *, suppressions, "", "Suppressions file name.") ASAN_FLAG(bool, halt_on_error, true, "Crash the program after printing the first error report " "(WARNING: USE AT YOUR OWN RISK!)") -ASAN_FLAG(bool, use_odr_indicator, false, - "Use special ODR indicator symbol for ODR violation detection") ASAN_FLAG(bool, allocator_frees_and_returns_null_on_realloc_zero, true, "realloc(p, 0) is equivalent to free(p) by default (Same as the " "POSIX standard). If set to false, realloc(p, 0) will return a " diff --git a/contrib/compiler-rt/lib/asan/asan_fuchsia.cc b/contrib/compiler-rt/lib/asan/asan_fuchsia.cc index 0b5bff4f565e..34399c92310b 100644 --- a/contrib/compiler-rt/lib/asan/asan_fuchsia.cc +++ b/contrib/compiler-rt/lib/asan/asan_fuchsia.cc @@ -190,6 +190,13 @@ static void ThreadExitHook(void *hook, uptr os_id) { AsanThread::TSDDtor(per_thread); } +bool HandleDlopenInit() { + // Not supported on this platform. + static_assert(!SANITIZER_SUPPORTS_INIT_FOR_DLOPEN, + "Expected SANITIZER_SUPPORTS_INIT_FOR_DLOPEN to be false"); + return false; +} + } // namespace __asan // These are declared (in extern "C") by <zircon/sanitizer.h>. diff --git a/contrib/compiler-rt/lib/asan/asan_globals.cc b/contrib/compiler-rt/lib/asan/asan_globals.cc index 898f7f40d31b..146234ac6c64 100644 --- a/contrib/compiler-rt/lib/asan/asan_globals.cc +++ b/contrib/compiler-rt/lib/asan/asan_globals.cc @@ -83,9 +83,11 @@ static bool IsAddressNearGlobal(uptr addr, const __asan_global &g) { } static void ReportGlobal(const Global &g, const char *prefix) { - Report("%s Global[%p]: beg=%p size=%zu/%zu name=%s module=%s dyn_init=%zu\n", - prefix, &g, (void *)g.beg, g.size, g.size_with_redzone, g.name, - g.module_name, g.has_dynamic_init); + Report( + "%s Global[%p]: beg=%p size=%zu/%zu name=%s module=%s dyn_init=%zu " + "odr_indicator=%p\n", + prefix, &g, (void *)g.beg, g.size, g.size_with_redzone, g.name, + g.module_name, g.has_dynamic_init, (void *)g.odr_indicator); if (g.location) { Report(" location (%p): name=%s[%p], %d %d\n", g.location, g.location->filename, g.location->filename, g.location->line_no, @@ -133,6 +135,9 @@ enum GlobalSymbolState { // this method in case compiler instruments global variables through their // local aliases. static void CheckODRViolationViaIndicator(const Global *g) { + // Instrumentation requests to skip ODR check. + if (g->odr_indicator == UINTPTR_MAX) + return; u8 *odr_indicator = reinterpret_cast<u8 *>(g->odr_indicator); if (*odr_indicator == UNREGISTERED) { *odr_indicator = REGISTERED; @@ -183,9 +188,7 @@ static void CheckODRViolationViaPoisoning(const Global *g) { // This routine chooses between two different methods of ODR violation // detection. static inline bool UseODRIndicator(const Global *g) { - // Use ODR indicator method iff use_odr_indicator flag is set and - // indicator symbol address is not 0. - return flags()->use_odr_indicator && g->odr_indicator > 0; + return g->odr_indicator > 0; } // Register a global variable. @@ -248,7 +251,7 @@ static void UnregisterGlobal(const Global *g) { // implementation. It might not be worth doing anyway. // Release ODR indicator. - if (UseODRIndicator(g)) { + if (UseODRIndicator(g) && g->odr_indicator != UINTPTR_MAX) { u8 *odr_indicator = reinterpret_cast<u8 *>(g->odr_indicator); *odr_indicator = UNREGISTERED; } diff --git a/contrib/compiler-rt/lib/asan/asan_globals_win.cc b/contrib/compiler-rt/lib/asan/asan_globals_win.cc index 29ab5ebf16d4..0e75992bfc8d 100644 --- a/contrib/compiler-rt/lib/asan/asan_globals_win.cc +++ b/contrib/compiler-rt/lib/asan/asan_globals_win.cc @@ -29,7 +29,7 @@ static void call_on_globals(void (*hook)(__asan_global *, uptr)) { __asan_global *end = &__asan_globals_end; uptr bytediff = (uptr)end - (uptr)start; if (bytediff % sizeof(__asan_global) != 0) { -#ifdef SANITIZER_DLL_THUNK +#if defined(SANITIZER_DLL_THUNK) || defined(SANITIZER_DYNAMIC_RUNTIME_THUNK) __debugbreak(); #else CHECK("corrupt asan global array"); diff --git a/contrib/compiler-rt/lib/asan/asan_internal.h b/contrib/compiler-rt/lib/asan/asan_internal.h index 654878cd15f0..57869497c7d3 100644 --- a/contrib/compiler-rt/lib/asan/asan_internal.h +++ b/contrib/compiler-rt/lib/asan/asan_internal.h @@ -111,6 +111,11 @@ void *AsanDlSymNext(const char *sym); void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name); +// Returns `true` iff most of ASan init process should be skipped due to the +// ASan library being loaded via `dlopen()`. Platforms may perform any +// `dlopen()` specific initialization inside this function. +bool HandleDlopenInit(); + // Add convenient macro for interface functions that may be represented as // weak hooks. #define ASAN_MALLOC_HOOK(ptr, size) \ diff --git a/contrib/compiler-rt/lib/asan/asan_linux.cc b/contrib/compiler-rt/lib/asan/asan_linux.cc index 625f32d408df..a150b1955d60 100644 --- a/contrib/compiler-rt/lib/asan/asan_linux.cc +++ b/contrib/compiler-rt/lib/asan/asan_linux.cc @@ -248,6 +248,13 @@ void *AsanDlSymNext(const char *sym) { return dlsym(RTLD_NEXT, sym); } +bool HandleDlopenInit() { + // Not supported on this platform. + static_assert(!SANITIZER_SUPPORTS_INIT_FOR_DLOPEN, + "Expected SANITIZER_SUPPORTS_INIT_FOR_DLOPEN to be false"); + return false; +} + } // namespace __asan #endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || diff --git a/contrib/compiler-rt/lib/asan/asan_malloc_linux.cc b/contrib/compiler-rt/lib/asan/asan_malloc_linux.cc index 76bdff999d62..0a534fe21168 100644 --- a/contrib/compiler-rt/lib/asan/asan_malloc_linux.cc +++ b/contrib/compiler-rt/lib/asan/asan_malloc_linux.cc @@ -209,7 +209,7 @@ INTERCEPTOR(struct fake_mallinfo, mallinfo, void) { } INTERCEPTOR(int, mallopt, int cmd, int value) { - return -1; + return 0; } #endif // SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO diff --git a/contrib/compiler-rt/lib/asan/asan_malloc_mac.cc b/contrib/compiler-rt/lib/asan/asan_malloc_mac.cc index 733ba2d86e13..27281f1bc834 100644 --- a/contrib/compiler-rt/lib/asan/asan_malloc_mac.cc +++ b/contrib/compiler-rt/lib/asan/asan_malloc_mac.cc @@ -61,4 +61,25 @@ using namespace __asan; #include "sanitizer_common/sanitizer_malloc_mac.inc" +namespace COMMON_MALLOC_NAMESPACE { +bool HandleDlopenInit() { + static_assert(SANITIZER_SUPPORTS_INIT_FOR_DLOPEN, + "Expected SANITIZER_SUPPORTS_INIT_FOR_DLOPEN to be true"); + // We have no reliable way of knowing how we are being loaded + // so make it a requirement on Apple platforms to set this environment + // variable to indicate that we want to perform initialization via + // dlopen(). + auto init_str = GetEnv("APPLE_ASAN_INIT_FOR_DLOPEN"); + if (!init_str) + return false; + if (internal_strncmp(init_str, "1", 1) != 0) + return false; + // When we are loaded via `dlopen()` path we still initialize the malloc zone + // so Symbolication clients (e.g. `leaks`) that load the ASan allocator can + // find an initialized malloc zone. + InitMallocZoneFields(); + return true; +} +} // namespace COMMON_MALLOC_NAMESPACE + #endif diff --git a/contrib/compiler-rt/lib/asan/asan_malloc_win.cc b/contrib/compiler-rt/lib/asan/asan_malloc_win.cc index efa058243979..887936431931 100644 --- a/contrib/compiler-rt/lib/asan/asan_malloc_win.cc +++ b/contrib/compiler-rt/lib/asan/asan_malloc_win.cc @@ -14,8 +14,17 @@ #include "sanitizer_common/sanitizer_platform.h" #if SANITIZER_WINDOWS -#define WIN32_LEAN_AND_MEAN -#include <windows.h> +// Intentionally not including windows.h here, to avoid the risk of +// pulling in conflicting declarations of these functions. (With mingw-w64, +// there's a risk of windows.h pulling in stdint.h.) +typedef int BOOL; +typedef void *HANDLE; +typedef const void *LPCVOID; +typedef void *LPVOID; + +#define HEAP_ZERO_MEMORY 0x00000008 +#define HEAP_REALLOC_IN_PLACE_ONLY 0x00000010 + #include "asan_allocator.h" #include "asan_interceptors.h" @@ -125,13 +134,18 @@ void *_recalloc_base(void *p, size_t n, size_t elem_size) { } ALLOCATION_FUNCTION_ATTRIBUTE -size_t _msize(const void *ptr) { +size_t _msize(void *ptr) { GET_CURRENT_PC_BP_SP; (void)sp; return asan_malloc_usable_size(ptr, pc, bp); } ALLOCATION_FUNCTION_ATTRIBUTE +size_t _msize_base(void *ptr) { + return _msize(ptr); +} + +ALLOCATION_FUNCTION_ATTRIBUTE void *_expand(void *memblock, size_t size) { // _expand is used in realloc-like functions to resize the buffer if possible. // We don't want memory to stand still while resizing buffers, so return 0. @@ -226,6 +240,7 @@ void ReplaceSystemMalloc() { TryToOverrideFunction("_recalloc_base", (uptr)_recalloc); TryToOverrideFunction("_recalloc_crt", (uptr)_recalloc); TryToOverrideFunction("_msize", (uptr)_msize); + TryToOverrideFunction("_msize_base", (uptr)_msize); TryToOverrideFunction("_expand", (uptr)_expand); TryToOverrideFunction("_expand_base", (uptr)_expand); diff --git a/contrib/compiler-rt/lib/asan/asan_new_delete.cc b/contrib/compiler-rt/lib/asan/asan_new_delete.cc index 30efd61a9680..e6053c1fe87f 100644 --- a/contrib/compiler-rt/lib/asan/asan_new_delete.cc +++ b/contrib/compiler-rt/lib/asan/asan_new_delete.cc @@ -26,7 +26,7 @@ // anyway by passing extra -export flags to the linker, which is exactly that // dllexport would normally do. We need to export them in order to make the // VS2015 dynamic CRT (MD) work. -#if SANITIZER_WINDOWS +#if SANITIZER_WINDOWS && defined(_MSC_VER) #define CXX_OPERATOR_ATTRIBUTE #define COMMENT_EXPORT(sym) __pragma(comment(linker, "/export:" sym)) #ifdef _WIN64 diff --git a/contrib/compiler-rt/lib/asan/asan_posix.cc b/contrib/compiler-rt/lib/asan/asan_posix.cc index 17c28b0aea2a..ca99c04b3ad8 100644 --- a/contrib/compiler-rt/lib/asan/asan_posix.cc +++ b/contrib/compiler-rt/lib/asan/asan_posix.cc @@ -40,6 +40,51 @@ void AsanOnDeadlySignal(int signo, void *siginfo, void *context) { // ---------------------- TSD ---------------- {{{1 +#if SANITIZER_NETBSD || SANITIZER_FREEBSD +// Thread Static Data cannot be used in early init on NetBSD and FreeBSD. +// Reuse the Asan TSD API for compatibility with existing code +// with an alternative implementation. + +static void (*tsd_destructor)(void *tsd) = nullptr; + +struct tsd_key { + tsd_key() : key(nullptr) {} + ~tsd_key() { + CHECK(tsd_destructor); + if (key) + (*tsd_destructor)(key); + } + void *key; +}; + +static thread_local struct tsd_key key; + +void AsanTSDInit(void (*destructor)(void *tsd)) { + CHECK(!tsd_destructor); + tsd_destructor = destructor; +} + +void *AsanTSDGet() { + CHECK(tsd_destructor); + return key.key; +} + +void AsanTSDSet(void *tsd) { + CHECK(tsd_destructor); + CHECK(tsd); + CHECK(!key.key); + key.key = tsd; +} + +void PlatformTSDDtor(void *tsd) { + CHECK(tsd_destructor); + CHECK_EQ(key.key, tsd); + key.key = nullptr; + // Make sure that signal handler can not see a stale current thread pointer. + atomic_signal_fence(memory_order_seq_cst); + AsanThread::TSDDtor(tsd); +} +#else static pthread_key_t tsd_key; static bool tsd_key_inited = false; void AsanTSDInit(void (*destructor)(void *tsd)) { @@ -67,6 +112,7 @@ void PlatformTSDDtor(void *tsd) { } AsanThread::TSDDtor(tsd); } +#endif } // namespace __asan #endif // SANITIZER_POSIX diff --git a/contrib/compiler-rt/lib/asan/asan_report.h b/contrib/compiler-rt/lib/asan/asan_report.h index f7153d4810d0..b0c167dda756 100644 --- a/contrib/compiler-rt/lib/asan/asan_report.h +++ b/contrib/compiler-rt/lib/asan/asan_report.h @@ -12,6 +12,9 @@ // ASan-private header for error reporting functions. //===----------------------------------------------------------------------===// +#ifndef ASAN_REPORT_H +#define ASAN_REPORT_H + #include "asan_allocator.h" #include "asan_internal.h" #include "asan_thread.h" @@ -92,3 +95,4 @@ void ReportMacCfReallocUnknown(uptr addr, uptr zone_ptr, BufferedStackTrace *stack); } // namespace __asan +#endif // ASAN_REPORT_H diff --git a/contrib/compiler-rt/lib/asan/asan_rtems.cc b/contrib/compiler-rt/lib/asan/asan_rtems.cc index a4af940057eb..b48cc6a75d78 100644 --- a/contrib/compiler-rt/lib/asan/asan_rtems.cc +++ b/contrib/compiler-rt/lib/asan/asan_rtems.cc @@ -213,6 +213,12 @@ static void HandleExit() { } } +bool HandleDlopenInit() { + // Not supported on this platform. + static_assert(!SANITIZER_SUPPORTS_INIT_FOR_DLOPEN, + "Expected SANITIZER_SUPPORTS_INIT_FOR_DLOPEN to be false"); + return false; +} } // namespace __asan // These are declared (in extern "C") by <some_path/sanitizer.h>. diff --git a/contrib/compiler-rt/lib/asan/asan_rtl.cc b/contrib/compiler-rt/lib/asan/asan_rtl.cc index 4cff736f213a..13344f3b8887 100644 --- a/contrib/compiler-rt/lib/asan/asan_rtl.cc +++ b/contrib/compiler-rt/lib/asan/asan_rtl.cc @@ -383,6 +383,19 @@ void PrintAddressSpaceLayout() { kHighShadowBeg > kMidMemEnd); } +#if defined(__thumb__) && defined(__linux__) +#define START_BACKGROUND_THREAD_IN_ASAN_INTERNAL +#endif + +#ifndef START_BACKGROUND_THREAD_IN_ASAN_INTERNAL +static bool UNUSED __local_asan_dyninit = [] { + MaybeStartBackgroudThread(); + SetSoftRssLimitExceededCallback(AsanSoftRssLimitExceededCallback); + + return false; +}(); +#endif + static void AsanInitInternal() { if (LIKELY(asan_inited)) return; SanitizerToolName = "AddressSanitizer"; @@ -396,6 +409,14 @@ static void AsanInitInternal() { // initialization steps look at flags(). InitializeFlags(); + // Stop performing init at this point if we are being loaded via + // dlopen() and the platform supports it. + if (SANITIZER_SUPPORTS_INIT_FOR_DLOPEN && UNLIKELY(HandleDlopenInit())) { + asan_init_is_running = false; + VReport(1, "AddressSanitizer init is being performed for dlopen().\n"); + return; + } + AsanCheckIncompatibleRT(); AsanCheckDynamicRTPrereqs(); AvoidCVE_2016_2143(); @@ -420,6 +441,8 @@ static void AsanInitInternal() { __asan_option_detect_stack_use_after_return = flags()->detect_stack_use_after_return; + __sanitizer::InitializePlatformEarly(); + // Re-exec ourselves if we need to set additional env or command line args. MaybeReexec(); @@ -447,8 +470,10 @@ static void AsanInitInternal() { allocator_options.SetFrom(flags(), common_flags()); InitializeAllocator(allocator_options); +#ifdef START_BACKGROUND_THREAD_IN_ASAN_INTERNAL MaybeStartBackgroudThread(); SetSoftRssLimitExceededCallback(AsanSoftRssLimitExceededCallback); +#endif // On Linux AsanThread::ThreadStart() calls malloc() that's why asan_inited // should be set to 1 prior to initializing the threads. diff --git a/contrib/compiler-rt/lib/asan/asan_thread.cc b/contrib/compiler-rt/lib/asan/asan_thread.cc index faf423d305b7..0895e4ce0d93 100644 --- a/contrib/compiler-rt/lib/asan/asan_thread.cc +++ b/contrib/compiler-rt/lib/asan/asan_thread.cc @@ -223,9 +223,11 @@ void AsanThread::Init(const InitOptions *options) { atomic_store(&stack_switching_, false, memory_order_release); CHECK_EQ(this->stack_size(), 0U); SetThreadStackAndTls(options); - CHECK_GT(this->stack_size(), 0U); - CHECK(AddrIsInMem(stack_bottom_)); - CHECK(AddrIsInMem(stack_top_ - 1)); + if (stack_top_ != stack_bottom_) { + CHECK_GT(this->stack_size(), 0U); + CHECK(AddrIsInMem(stack_bottom_)); + CHECK(AddrIsInMem(stack_top_ - 1)); + } ClearShadowForThreadStackAndTLS(); fake_stack_ = nullptr; if (__asan_option_detect_stack_use_after_return) @@ -289,20 +291,23 @@ void AsanThread::SetThreadStackAndTls(const InitOptions *options) { DCHECK_EQ(options, nullptr); uptr tls_size = 0; uptr stack_size = 0; - GetThreadStackAndTls(tid() == 0, const_cast<uptr *>(&stack_bottom_), - const_cast<uptr *>(&stack_size), &tls_begin_, &tls_size); + GetThreadStackAndTls(tid() == 0, &stack_bottom_, &stack_size, &tls_begin_, + &tls_size); stack_top_ = stack_bottom_ + stack_size; tls_end_ = tls_begin_ + tls_size; dtls_ = DTLS_Get(); - int local; - CHECK(AddrIsInStack((uptr)&local)); + if (stack_top_ != stack_bottom_) { + int local; + CHECK(AddrIsInStack((uptr)&local)); + } } #endif // !SANITIZER_FUCHSIA && !SANITIZER_RTEMS void AsanThread::ClearShadowForThreadStackAndTLS() { - PoisonShadow(stack_bottom_, stack_top_ - stack_bottom_, 0); + if (stack_top_ != stack_bottom_) + PoisonShadow(stack_bottom_, stack_top_ - stack_bottom_, 0); if (tls_begin_ != tls_end_) { uptr tls_begin_aligned = RoundDownTo(tls_begin_, SHADOW_GRANULARITY); uptr tls_end_aligned = RoundUpTo(tls_end_, SHADOW_GRANULARITY); @@ -314,6 +319,9 @@ void AsanThread::ClearShadowForThreadStackAndTLS() { bool AsanThread::GetStackFrameAccessByAddr(uptr addr, StackFrameAccess *access) { + if (stack_top_ == stack_bottom_) + return false; + uptr bottom = 0; if (AddrIsInStack(addr)) { bottom = stack_bottom(); diff --git a/contrib/compiler-rt/lib/asan/asan_win.cc b/contrib/compiler-rt/lib/asan/asan_win.cc index 67125d38ad4a..068f4a5d247d 100644 --- a/contrib/compiler-rt/lib/asan/asan_win.cc +++ b/contrib/compiler-rt/lib/asan/asan_win.cc @@ -159,6 +159,14 @@ INTERCEPTOR_WINAPI(DWORD, CreateThread, namespace __asan { void InitializePlatformInterceptors() { + // The interceptors were not designed to be removable, so we have to keep this + // module alive for the life of the process. + HMODULE pinned; + CHECK(GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_PIN, + (LPCWSTR)&InitializePlatformInterceptors, + &pinned)); + ASAN_INTERCEPT_FUNC(CreateThread); ASAN_INTERCEPT_FUNC(SetUnhandledExceptionFilter); @@ -314,6 +322,13 @@ int __asan_set_seh_filter() { return 0; } +bool HandleDlopenInit() { + // Not supported on this platform. + static_assert(!SANITIZER_SUPPORTS_INIT_FOR_DLOPEN, + "Expected SANITIZER_SUPPORTS_INIT_FOR_DLOPEN to be false"); + return false; +} + #if !ASAN_DYNAMIC // The CRT runs initializers in this order: // - C initializers, from XIA to XIZ diff --git a/contrib/compiler-rt/lib/asan/asan_win_dll_thunk.cc b/contrib/compiler-rt/lib/asan/asan_win_dll_thunk.cc index c6a313d24026..df593ab92de6 100644 --- a/contrib/compiler-rt/lib/asan/asan_win_dll_thunk.cc +++ b/contrib/compiler-rt/lib/asan/asan_win_dll_thunk.cc @@ -48,6 +48,7 @@ INTERCEPT_WRAP_W_WWW(_recalloc) INTERCEPT_WRAP_W_WWW(_recalloc_base) INTERCEPT_WRAP_W_W(_msize) +INTERCEPT_WRAP_W_W(_msize_base) INTERCEPT_WRAP_W_W(_expand) INTERCEPT_WRAP_W_W(_expand_dbg) diff --git a/contrib/compiler-rt/lib/builtins/arm/addsf3.S b/contrib/compiler-rt/lib/builtins/arm/addsf3.S index 362b5c147ea6..74723cbeff74 100644 --- a/contrib/compiler-rt/lib/builtins/arm/addsf3.S +++ b/contrib/compiler-rt/lib/builtins/arm/addsf3.S @@ -178,7 +178,7 @@ LOCAL_LABEL(do_substraction): push {r0, r1, r2, r3} movs r0, r4 - bl __clzsi2 + bl SYMBOL_NAME(__clzsi2) movs r5, r0 pop {r0, r1, r2, r3} // shift = rep_clz(aSignificand) - rep_clz(implicitBit << 3); diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S index 87dd03dce94d..adc2d55d90f5 100644 --- a/contrib/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S +++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S @@ -55,7 +55,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cdcmpeq) mov ip, #APSR_C msr APSR_nzcvq, ip #else - msr CPSR_f, #APSR_C + msr APSR_nzcvq, #APSR_C #endif JMP(lr) #endif @@ -115,11 +115,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cdcmple) movne ip, #(APSR_C) 1: -#if defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7EM__) msr APSR_nzcvq, ip -#else - msr CPSR_f, ip -#endif pop {r0-r3} POP_PC() #endif diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S index c5fee6b6a08e..4b1de997687f 100644 --- a/contrib/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S +++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S @@ -55,7 +55,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cfcmpeq) mov ip, #APSR_C msr APSR_nzcvq, ip #else - msr CPSR_f, #APSR_C + msr APSR_nzcvq, #APSR_C #endif JMP(lr) #endif @@ -115,11 +115,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cfcmple) movne ip, #(APSR_C) 1: -#if defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7EM__) msr APSR_nzcvq, ip -#else - msr CPSR_f, ip -#endif pop {r0-r3} POP_PC() #endif diff --git a/contrib/compiler-rt/lib/builtins/clzdi2.c b/contrib/compiler-rt/lib/builtins/clzdi2.c index b56d98f5c01f..1819e6be436b 100644 --- a/contrib/compiler-rt/lib/builtins/clzdi2.c +++ b/contrib/compiler-rt/lib/builtins/clzdi2.c @@ -16,8 +16,13 @@ /* Returns: the number of leading 0-bits */ -#if !defined(__clang__) && (defined(__sparc64__) || defined(__mips64) || defined(__riscv__)) -/* gcc resolves __builtin_clz -> __clzdi2 leading to infinite recursion */ +#if !defined(__clang__) && \ + ((defined(__sparc__) && defined(__arch64__)) || \ + defined(__mips64) || \ + (defined(__riscv) && __SIZEOF_POINTER__ >= 8)) +/* On 64-bit architectures with neither a native clz instruction nor a native + * ctz instruction, gcc resolves __builtin_clz to __clzdi2 rather than + * __clzsi2, leading to infinite recursion. */ #define __builtin_clz(a) __clzsi2(a) extern si_int __clzsi2(si_int); #endif diff --git a/contrib/compiler-rt/lib/builtins/cpu_model.c b/contrib/compiler-rt/lib/builtins/cpu_model.c index 43b913390fc4..fb2b899fc70a 100644 --- a/contrib/compiler-rt/lib/builtins/cpu_model.c +++ b/contrib/compiler-rt/lib/builtins/cpu_model.c @@ -55,6 +55,9 @@ enum ProcessorTypes { AMD_BTVER2, AMDFAM17H, INTEL_KNM, + INTEL_GOLDMONT, + INTEL_GOLDMONT_PLUS, + INTEL_TREMONT, CPU_TYPE_MAX }; @@ -76,6 +79,8 @@ enum ProcessorSubtypes { INTEL_COREI7_SKYLAKE, INTEL_COREI7_SKYLAKE_AVX512, INTEL_COREI7_CANNONLAKE, + INTEL_COREI7_ICELAKE_CLIENT, + INTEL_COREI7_ICELAKE_SERVER, CPU_SUBTYPE_MAX }; @@ -110,7 +115,12 @@ enum ProcessorFeatures { FEATURE_AVX512IFMA, FEATURE_AVX5124VNNIW, FEATURE_AVX5124FMAPS, - FEATURE_AVX512VPOPCNTDQ + FEATURE_AVX512VPOPCNTDQ, + FEATURE_AVX512VBMI2, + FEATURE_GFNI, + FEATURE_VPCLMULQDQ, + FEATURE_AVX512VNNI, + FEATURE_AVX512BITALG }; // The check below for i386 was copied from clang's cpuid.h (__get_cpuid_max). @@ -364,6 +374,14 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, case 0x4c: // really airmont *Type = INTEL_SILVERMONT; break; // "silvermont" + // Goldmont: + case 0x5c: // Apollo Lake + case 0x5f: // Denverton + *Type = INTEL_GOLDMONT; + break; // "goldmont" + case 0x7a: + *Type = INTEL_GOLDMONT_PLUS; + break; case 0x57: *Type = INTEL_KNL; // knl @@ -438,35 +456,45 @@ static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model, } static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf, - unsigned *FeaturesOut) { + unsigned *FeaturesOut, + unsigned *Features2Out) { unsigned Features = 0; + unsigned Features2 = 0; unsigned EAX, EBX; +#define setFeature(F) \ + do { \ + if (F < 32) \ + Features |= 1U << (F & 0x1f); \ + else if (F < 64) \ + Features2 |= 1U << ((F - 32) & 0x1f); \ + } while (0) + if ((EDX >> 15) & 1) - Features |= 1 << FEATURE_CMOV; + setFeature(FEATURE_CMOV); if ((EDX >> 23) & 1) - Features |= 1 << FEATURE_MMX; + setFeature(FEATURE_MMX); if ((EDX >> 25) & 1) - Features |= 1 << FEATURE_SSE; + setFeature(FEATURE_SSE); if ((EDX >> 26) & 1) - Features |= 1 << FEATURE_SSE2; + setFeature(FEATURE_SSE2); if ((ECX >> 0) & 1) - Features |= 1 << FEATURE_SSE3; + setFeature(FEATURE_SSE3); if ((ECX >> 1) & 1) - Features |= 1 << FEATURE_PCLMUL; + setFeature(FEATURE_PCLMUL); if ((ECX >> 9) & 1) - Features |= 1 << FEATURE_SSSE3; + setFeature(FEATURE_SSSE3); if ((ECX >> 12) & 1) - Features |= 1 << FEATURE_FMA; + setFeature(FEATURE_FMA); if ((ECX >> 19) & 1) - Features |= 1 << FEATURE_SSE4_1; + setFeature(FEATURE_SSE4_1); if ((ECX >> 20) & 1) - Features |= 1 << FEATURE_SSE4_2; + setFeature(FEATURE_SSE4_2); if ((ECX >> 23) & 1) - Features |= 1 << FEATURE_POPCNT; + setFeature(FEATURE_POPCNT); if ((ECX >> 25) & 1) - Features |= 1 << FEATURE_AES; + setFeature(FEATURE_AES); // If CPUID indicates support for XSAVE, XRESTORE and AVX, and XGETBV // indicates that the AVX registers will be saved and restored on context @@ -477,43 +505,53 @@ static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf, bool HasAVX512Save = HasAVX && ((EAX & 0xe0) == 0xe0); if (HasAVX) - Features |= 1 << FEATURE_AVX; + setFeature(FEATURE_AVX); bool HasLeaf7 = MaxLeaf >= 0x7 && !getX86CpuIDAndInfoEx(0x7, 0x0, &EAX, &EBX, &ECX, &EDX); if (HasLeaf7 && ((EBX >> 3) & 1)) - Features |= 1 << FEATURE_BMI; + setFeature(FEATURE_BMI); if (HasLeaf7 && ((EBX >> 5) & 1) && HasAVX) - Features |= 1 << FEATURE_AVX2; + setFeature(FEATURE_AVX2); if (HasLeaf7 && ((EBX >> 9) & 1)) - Features |= 1 << FEATURE_BMI2; + setFeature(FEATURE_BMI2); if (HasLeaf7 && ((EBX >> 16) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX512F; + setFeature(FEATURE_AVX512F); if (HasLeaf7 && ((EBX >> 17) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX512DQ; + setFeature(FEATURE_AVX512DQ); if (HasLeaf7 && ((EBX >> 21) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX512IFMA; + setFeature(FEATURE_AVX512IFMA); if (HasLeaf7 && ((EBX >> 26) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX512PF; + setFeature(FEATURE_AVX512PF); if (HasLeaf7 && ((EBX >> 27) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX512ER; + setFeature(FEATURE_AVX512ER); if (HasLeaf7 && ((EBX >> 28) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX512CD; + setFeature(FEATURE_AVX512CD); if (HasLeaf7 && ((EBX >> 30) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX512BW; + setFeature(FEATURE_AVX512BW); if (HasLeaf7 && ((EBX >> 31) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX512VL; + setFeature(FEATURE_AVX512VL); if (HasLeaf7 && ((ECX >> 1) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX512VBMI; + setFeature(FEATURE_AVX512VBMI); + if (HasLeaf7 && ((ECX >> 6) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512VBMI2); + if (HasLeaf7 && ((ECX >> 8) & 1)) + setFeature(FEATURE_GFNI); + if (HasLeaf7 && ((ECX >> 10) & 1) && HasAVX) + setFeature(FEATURE_VPCLMULQDQ); + if (HasLeaf7 && ((ECX >> 11) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512VNNI); + if (HasLeaf7 && ((ECX >> 12) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512BITALG); if (HasLeaf7 && ((ECX >> 14) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX512VPOPCNTDQ; + setFeature(FEATURE_AVX512VPOPCNTDQ); if (HasLeaf7 && ((EDX >> 2) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX5124VNNIW; + setFeature(FEATURE_AVX5124VNNIW); if (HasLeaf7 && ((EDX >> 3) & 1) && HasAVX512Save) - Features |= 1 << FEATURE_AVX5124FMAPS; + setFeature(FEATURE_AVX5124FMAPS); unsigned MaxExtLevel; getX86CpuIDAndInfo(0x80000000, &MaxExtLevel, &EBX, &ECX, &EDX); @@ -521,13 +559,15 @@ static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf, bool HasExtLeaf1 = MaxExtLevel >= 0x80000001 && !getX86CpuIDAndInfo(0x80000001, &EAX, &EBX, &ECX, &EDX); if (HasExtLeaf1 && ((ECX >> 6) & 1)) - Features |= 1 << FEATURE_SSE4_A; + setFeature(FEATURE_SSE4_A); if (HasExtLeaf1 && ((ECX >> 11) & 1)) - Features |= 1 << FEATURE_XOP; + setFeature(FEATURE_XOP); if (HasExtLeaf1 && ((ECX >> 16) & 1)) - Features |= 1 << FEATURE_FMA4; + setFeature(FEATURE_FMA4); *FeaturesOut = Features; + *Features2Out = Features2; +#undef setFeature } #if defined(HAVE_INIT_PRIORITY) @@ -548,8 +588,9 @@ struct __processor_model { unsigned int __cpu_subtype; unsigned int __cpu_features[1]; } __cpu_model = {0, 0, 0, {0}}; +unsigned int __cpu_features2; -/* A constructor function that is sets __cpu_model and __cpu_features with +/* A constructor function that is sets __cpu_model and __cpu_features2 with the right values. This needs to run only once. This constructor is given the highest priority and it should run before constructors without the priority set. However, it still runs after ifunc initializers and @@ -562,6 +603,7 @@ __cpu_indicator_init(void) { unsigned Vendor; unsigned Model, Family, Brand_id; unsigned Features = 0; + unsigned Features2 = 0; /* This function needs to run just once. */ if (__cpu_model.__cpu_vendor) @@ -580,8 +622,9 @@ __cpu_indicator_init(void) { Brand_id = EBX & 0xff; /* Find available features. */ - getAvailableFeatures(ECX, EDX, MaxLeaf, &Features); + getAvailableFeatures(ECX, EDX, MaxLeaf, &Features, &Features2); __cpu_model.__cpu_features[0] = Features; + __cpu_features2 = Features2; if (Vendor == SIG_INTEL) { /* Get CPU type. */ diff --git a/contrib/compiler-rt/lib/builtins/ctzdi2.c b/contrib/compiler-rt/lib/builtins/ctzdi2.c index eecde29718d7..ef6d7fea1365 100644 --- a/contrib/compiler-rt/lib/builtins/ctzdi2.c +++ b/contrib/compiler-rt/lib/builtins/ctzdi2.c @@ -16,8 +16,13 @@ /* Returns: the number of trailing 0-bits */ -#if !defined(__clang__) && (defined(__sparc64__) || defined(__mips64) || defined(__riscv__)) -/* gcc resolves __builtin_ctz -> __ctzdi2 leading to infinite recursion */ +#if !defined(__clang__) && \ + ((defined(__sparc__) && defined(__arch64__)) || \ + defined(__mips64) || \ + (defined(__riscv) && __SIZEOF_POINTER__ >= 8)) +/* On 64-bit architectures with neither a native clz instruction nor a native + * ctz instruction, gcc resolves __builtin_ctz to __ctzdi2 rather than + * __ctzsi2, leading to infinite recursion. */ #define __builtin_ctz(a) __ctzsi2(a) extern si_int __ctzsi2(si_int); #endif diff --git a/contrib/compiler-rt/lib/builtins/divdc3.c b/contrib/compiler-rt/lib/builtins/divdc3.c index 3c88390b5e77..392d6ecacde1 100644 --- a/contrib/compiler-rt/lib/builtins/divdc3.c +++ b/contrib/compiler-rt/lib/builtins/divdc3.c @@ -12,6 +12,8 @@ * ===----------------------------------------------------------------------=== */ +#define DOUBLE_PRECISION +#include "fp_lib.h" #include "int_lib.h" #include "int_math.h" @@ -21,7 +23,7 @@ COMPILER_RT_ABI Dcomplex __divdc3(double __a, double __b, double __c, double __d) { int __ilogbw = 0; - double __logbw = crt_logb(crt_fmax(crt_fabs(__c), crt_fabs(__d))); + double __logbw = __compiler_rt_logb(crt_fmax(crt_fabs(__c), crt_fabs(__d))); if (crt_isfinite(__logbw)) { __ilogbw = (int)__logbw; diff --git a/contrib/compiler-rt/lib/builtins/divdf3.c b/contrib/compiler-rt/lib/builtins/divdf3.c index 04a4dc5571ca..411c82ebb87a 100644 --- a/contrib/compiler-rt/lib/builtins/divdf3.c +++ b/contrib/compiler-rt/lib/builtins/divdf3.c @@ -21,36 +21,36 @@ COMPILER_RT_ABI fp_t __divdf3(fp_t a, fp_t b) { - + const unsigned int aExponent = toRep(a) >> significandBits & maxExponent; const unsigned int bExponent = toRep(b) >> significandBits & maxExponent; const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit; - + rep_t aSignificand = toRep(a) & significandMask; rep_t bSignificand = toRep(b) & significandMask; int scale = 0; - + // Detect if a or b is zero, denormal, infinity, or NaN. if (aExponent-1U >= maxExponent-1U || bExponent-1U >= maxExponent-1U) { - + const rep_t aAbs = toRep(a) & absMask; const rep_t bAbs = toRep(b) & absMask; - + // NaN / anything = qNaN if (aAbs > infRep) return fromRep(toRep(a) | quietBit); // anything / NaN = qNaN if (bAbs > infRep) return fromRep(toRep(b) | quietBit); - + if (aAbs == infRep) { // infinity / infinity = NaN if (bAbs == infRep) return fromRep(qnanRep); // infinity / anything else = +/- infinity else return fromRep(aAbs | quotientSign); } - + // anything else / infinity = +/- 0 if (bAbs == infRep) return fromRep(quotientSign); - + if (!aAbs) { // zero / zero = NaN if (!bAbs) return fromRep(qnanRep); @@ -59,28 +59,28 @@ __divdf3(fp_t a, fp_t b) { } // anything else / zero = +/- infinity if (!bAbs) return fromRep(infRep | quotientSign); - + // one or both of a or b is denormal, the other (if applicable) is a // normal number. Renormalize one or both of a and b, and set scale to // include the necessary exponent adjustment. if (aAbs < implicitBit) scale += normalize(&aSignificand); if (bAbs < implicitBit) scale -= normalize(&bSignificand); } - + // Or in the implicit significand bit. (If we fell through from the // denormal path it was already set by normalize( ), but setting it twice // won't hurt anything.) aSignificand |= implicitBit; bSignificand |= implicitBit; int quotientExponent = aExponent - bExponent + scale; - + // Align the significand of b as a Q31 fixed-point number in the range // [1, 2.0) and get a Q32 approximate reciprocal using a small minimax // polynomial approximation: reciprocal = 3/4 + 1/sqrt(2) - b/2. This // is accurate to about 3.5 binary digits. const uint32_t q31b = bSignificand >> 21; uint32_t recip32 = UINT32_C(0x7504f333) - q31b; - + // Now refine the reciprocal estimate using a Newton-Raphson iteration: // // x1 = x0 * (2 - x0 * b) @@ -95,13 +95,13 @@ __divdf3(fp_t a, fp_t b) { recip32 = (uint64_t)recip32 * correction32 >> 31; correction32 = -((uint64_t)recip32 * q31b >> 32); recip32 = (uint64_t)recip32 * correction32 >> 31; - + // recip32 might have overflowed to exactly zero in the preceding // computation if the high word of b is exactly 1.0. This would sabotage // the full-width final stage of the computation that follows, so we adjust // recip32 downward by one bit. recip32--; - + // We need to perform one more iteration to get us to 56 binary digits; // The last iteration needs to happen with extra precision. const uint32_t q63blo = bSignificand << 11; @@ -110,14 +110,14 @@ __divdf3(fp_t a, fp_t b) { uint32_t cHi = correction >> 32; uint32_t cLo = correction; reciprocal = (uint64_t)recip32*cHi + ((uint64_t)recip32*cLo >> 32); - + // We already adjusted the 32-bit estimate, now we need to adjust the final // 64-bit reciprocal estimate downward to ensure that it is strictly smaller // than the infinitely precise exact reciprocal. Because the computation // of the Newton-Raphson step is truncating at every step, this adjustment // is small; most of the work is already done. reciprocal -= 2; - + // The numerical reciprocal is accurate to within 2^-56, lies in the // interval [0.5, 1.0), and is strictly smaller than the true reciprocal // of b. Multiplying a by this reciprocal thus gives a numerical q = a/b @@ -127,12 +127,12 @@ __divdf3(fp_t a, fp_t b) { // 2. q is in the interval [0.5, 2.0) // 3. the error in q is bounded away from 2^-53 (actually, we have a // couple of bits to spare, but this is all we need). - + // We need a 64 x 64 multiply high to compute q, which isn't a basic // operation in C, so we need to be a little bit fussy. rep_t quotient, quotientLo; wideMultiply(aSignificand << 2, reciprocal, "ient, "ientLo); - + // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0). // In either case, we are going to compute a residual of the form // @@ -141,7 +141,7 @@ __divdf3(fp_t a, fp_t b) { // We know from the construction of q that r satisfies: // // 0 <= r < ulp(q)*b - // + // // if r is greater than 1/2 ulp(q)*b, then q rounds up. Otherwise, we // already have the correct result. The exact halfway case cannot occur. // We also take this time to right shift quotient if it falls in the [1,2) @@ -154,20 +154,20 @@ __divdf3(fp_t a, fp_t b) { quotient >>= 1; residual = (aSignificand << 52) - quotient * bSignificand; } - + const int writtenExponent = quotientExponent + exponentBias; - + if (writtenExponent >= maxExponent) { // If we have overflowed the exponent, return infinity. return fromRep(infRep | quotientSign); } - + else if (writtenExponent < 1) { // Flush denormals to zero. In the future, it would be nice to add // code to round them correctly. return fromRep(quotientSign); } - + else { const bool round = (residual << 1) > bSignificand; // Clear the implicit bit diff --git a/contrib/compiler-rt/lib/builtins/divsc3.c b/contrib/compiler-rt/lib/builtins/divsc3.c index 42a48315e66d..0d18a256c3de 100644 --- a/contrib/compiler-rt/lib/builtins/divsc3.c +++ b/contrib/compiler-rt/lib/builtins/divsc3.c @@ -12,6 +12,8 @@ *===----------------------------------------------------------------------=== */ +#define SINGLE_PRECISION +#include "fp_lib.h" #include "int_lib.h" #include "int_math.h" @@ -21,7 +23,8 @@ COMPILER_RT_ABI Fcomplex __divsc3(float __a, float __b, float __c, float __d) { int __ilogbw = 0; - float __logbw = crt_logbf(crt_fmaxf(crt_fabsf(__c), crt_fabsf(__d))); + float __logbw = + __compiler_rt_logbf(crt_fmaxf(crt_fabsf(__c), crt_fabsf(__d))); if (crt_isfinite(__logbw)) { __ilogbw = (int)__logbw; diff --git a/contrib/compiler-rt/lib/builtins/divsf3.c b/contrib/compiler-rt/lib/builtins/divsf3.c index 65294d70fc61..a74917fd1de5 100644 --- a/contrib/compiler-rt/lib/builtins/divsf3.c +++ b/contrib/compiler-rt/lib/builtins/divsf3.c @@ -21,36 +21,36 @@ COMPILER_RT_ABI fp_t __divsf3(fp_t a, fp_t b) { - + const unsigned int aExponent = toRep(a) >> significandBits & maxExponent; const unsigned int bExponent = toRep(b) >> significandBits & maxExponent; const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit; - + rep_t aSignificand = toRep(a) & significandMask; rep_t bSignificand = toRep(b) & significandMask; int scale = 0; - + // Detect if a or b is zero, denormal, infinity, or NaN. if (aExponent-1U >= maxExponent-1U || bExponent-1U >= maxExponent-1U) { - + const rep_t aAbs = toRep(a) & absMask; const rep_t bAbs = toRep(b) & absMask; - + // NaN / anything = qNaN if (aAbs > infRep) return fromRep(toRep(a) | quietBit); // anything / NaN = qNaN if (bAbs > infRep) return fromRep(toRep(b) | quietBit); - + if (aAbs == infRep) { // infinity / infinity = NaN if (bAbs == infRep) return fromRep(qnanRep); // infinity / anything else = +/- infinity else return fromRep(aAbs | quotientSign); } - + // anything else / infinity = +/- 0 if (bAbs == infRep) return fromRep(quotientSign); - + if (!aAbs) { // zero / zero = NaN if (!bAbs) return fromRep(qnanRep); @@ -59,28 +59,28 @@ __divsf3(fp_t a, fp_t b) { } // anything else / zero = +/- infinity if (!bAbs) return fromRep(infRep | quotientSign); - + // one or both of a or b is denormal, the other (if applicable) is a // normal number. Renormalize one or both of a and b, and set scale to // include the necessary exponent adjustment. if (aAbs < implicitBit) scale += normalize(&aSignificand); if (bAbs < implicitBit) scale -= normalize(&bSignificand); } - + // Or in the implicit significand bit. (If we fell through from the // denormal path it was already set by normalize( ), but setting it twice // won't hurt anything.) aSignificand |= implicitBit; bSignificand |= implicitBit; int quotientExponent = aExponent - bExponent + scale; - + // Align the significand of b as a Q31 fixed-point number in the range // [1, 2.0) and get a Q32 approximate reciprocal using a small minimax // polynomial approximation: reciprocal = 3/4 + 1/sqrt(2) - b/2. This // is accurate to about 3.5 binary digits. uint32_t q31b = bSignificand << 8; uint32_t reciprocal = UINT32_C(0x7504f333) - q31b; - + // Now refine the reciprocal estimate using a Newton-Raphson iteration: // // x1 = x0 * (2 - x0 * b) @@ -95,7 +95,7 @@ __divsf3(fp_t a, fp_t b) { reciprocal = (uint64_t)reciprocal * correction >> 31; correction = -((uint64_t)reciprocal * q31b >> 32); reciprocal = (uint64_t)reciprocal * correction >> 31; - + // Exhaustive testing shows that the error in reciprocal after three steps // is in the interval [-0x1.f58108p-31, 0x1.d0e48cp-29], in line with our // expectations. We bump the reciprocal by a tiny value to force the error @@ -103,7 +103,7 @@ __divsf3(fp_t a, fp_t b) { // be specific). This also causes 1/1 to give a sensible approximation // instead of zero (due to overflow). reciprocal -= 2; - + // The numerical reciprocal is accurate to within 2^-28, lies in the // interval [0x1.000000eep-1, 0x1.fffffffcp-1], and is strictly smaller // than the true reciprocal of b. Multiplying a by this reciprocal thus @@ -115,9 +115,9 @@ __divsf3(fp_t a, fp_t b) { // from the fact that we truncate the product, and the 2^27 term // is the error in the reciprocal of b scaled by the maximum // possible value of a. As a consequence of this error bound, - // either q or nextafter(q) is the correctly rounded + // either q or nextafter(q) is the correctly rounded rep_t quotient = (uint64_t)reciprocal*(aSignificand << 1) >> 32; - + // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0). // In either case, we are going to compute a residual of the form // @@ -126,7 +126,7 @@ __divsf3(fp_t a, fp_t b) { // We know from the construction of q that r satisfies: // // 0 <= r < ulp(q)*b - // + // // if r is greater than 1/2 ulp(q)*b, then q rounds up. Otherwise, we // already have the correct result. The exact halfway case cannot occur. // We also take this time to right shift quotient if it falls in the [1,2) @@ -141,18 +141,18 @@ __divsf3(fp_t a, fp_t b) { } const int writtenExponent = quotientExponent + exponentBias; - + if (writtenExponent >= maxExponent) { // If we have overflowed the exponent, return infinity. return fromRep(infRep | quotientSign); } - + else if (writtenExponent < 1) { // Flush denormals to zero. In the future, it would be nice to add // code to round them correctly. return fromRep(quotientSign); } - + else { const bool round = (residual << 1) > bSignificand; // Clear the implicit bit diff --git a/contrib/compiler-rt/lib/builtins/divtc3.c b/contrib/compiler-rt/lib/builtins/divtc3.c index 16e538ba4a33..e5ea00d841e3 100644 --- a/contrib/compiler-rt/lib/builtins/divtc3.c +++ b/contrib/compiler-rt/lib/builtins/divtc3.c @@ -12,6 +12,8 @@ *===----------------------------------------------------------------------=== */ +#define QUAD_PRECISION +#include "fp_lib.h" #include "int_lib.h" #include "int_math.h" @@ -21,7 +23,8 @@ COMPILER_RT_ABI Lcomplex __divtc3(long double __a, long double __b, long double __c, long double __d) { int __ilogbw = 0; - long double __logbw = crt_logbl(crt_fmaxl(crt_fabsl(__c), crt_fabsl(__d))); + long double __logbw = + __compiler_rt_logbl(crt_fmaxl(crt_fabsl(__c), crt_fabsl(__d))); if (crt_isfinite(__logbw)) { __ilogbw = (int)__logbw; diff --git a/contrib/compiler-rt/lib/builtins/emutls.c b/contrib/compiler-rt/lib/builtins/emutls.c index 07d436e267d8..ef95a1c260c1 100644 --- a/contrib/compiler-rt/lib/builtins/emutls.c +++ b/contrib/compiler-rt/lib/builtins/emutls.c @@ -42,6 +42,7 @@ static void emutls_shutdown(emutls_address_array *array); static pthread_mutex_t emutls_mutex = PTHREAD_MUTEX_INITIALIZER; static pthread_key_t emutls_pthread_key; +static bool emutls_key_created = false; typedef unsigned int gcc_word __attribute__((mode(word))); typedef unsigned int gcc_pointer __attribute__((mode(pointer))); @@ -109,6 +110,7 @@ static void emutls_key_destructor(void* ptr) { static __inline void emutls_init(void) { if (pthread_key_create(&emutls_pthread_key, emutls_key_destructor) != 0) abort(); + emutls_key_created = true; } static __inline void emutls_init_once(void) { @@ -390,3 +392,14 @@ void* __emutls_get_address(__emutls_control* control) { array->data[index] = emutls_allocate_object(control); return array->data[index]; } + +#ifdef __BIONIC__ +/* Called by Bionic on dlclose to delete the emutls pthread key. */ +__attribute__((visibility("hidden"))) +void __emutls_unregister_key(void) { + if (emutls_key_created) { + pthread_key_delete(emutls_pthread_key); + emutls_key_created = false; + } +} +#endif diff --git a/contrib/compiler-rt/lib/builtins/fp_lib.h b/contrib/compiler-rt/lib/builtins/fp_lib.h index 223fb980aaed..a0e19ab6a8f6 100644 --- a/contrib/compiler-rt/lib/builtins/fp_lib.h +++ b/contrib/compiler-rt/lib/builtins/fp_lib.h @@ -25,6 +25,7 @@ #include <stdbool.h> #include <limits.h> #include "int_lib.h" +#include "int_math.h" // x86_64 FreeBSD prior v9.3 define fixed-width types incorrectly in // 32-bit mode. @@ -265,6 +266,62 @@ static __inline void wideRightShiftWithSticky(rep_t *hi, rep_t *lo, unsigned int *hi = 0; } } + +// Implements logb methods (logb, logbf, logbl) for IEEE-754. This avoids +// pulling in a libm dependency from compiler-rt, but is not meant to replace +// it (i.e. code calling logb() should get the one from libm, not this), hence +// the __compiler_rt prefix. +static __inline fp_t __compiler_rt_logbX(fp_t x) { + rep_t rep = toRep(x); + int exp = (rep & exponentMask) >> significandBits; + + // Abnormal cases: + // 1) +/- inf returns +inf; NaN returns NaN + // 2) 0.0 returns -inf + if (exp == maxExponent) { + if (((rep & signBit) == 0) || (x != x)) { + return x; // NaN or +inf: return x + } else { + return -x; // -inf: return -x + } + } else if (x == 0.0) { + // 0.0: return -inf + return fromRep(infRep | signBit); + } + + if (exp != 0) { + // Normal number + return exp - exponentBias; // Unbias exponent + } else { + // Subnormal number; normalize and repeat + rep &= absMask; + const int shift = 1 - normalize(&rep); + exp = (rep & exponentMask) >> significandBits; + return exp - exponentBias - shift; // Unbias exponent + } +} +#endif + +#if defined(SINGLE_PRECISION) +static __inline fp_t __compiler_rt_logbf(fp_t x) { + return __compiler_rt_logbX(x); +} +#elif defined(DOUBLE_PRECISION) +static __inline fp_t __compiler_rt_logb(fp_t x) { + return __compiler_rt_logbX(x); +} +#elif defined(QUAD_PRECISION) + #if defined(CRT_LDBL_128BIT) +static __inline fp_t __compiler_rt_logbl(fp_t x) { + return __compiler_rt_logbX(x); +} + #else +// The generic implementation only works for ieee754 floating point. For other +// floating point types, continue to rely on the libm implementation for now. +static __inline long double __compiler_rt_logbl(long double x) { + return crt_logbl(x); +} + #endif #endif #endif // FP_LIB_HEADER diff --git a/contrib/compiler-rt/lib/builtins/gcc_personality_v0.c b/contrib/compiler-rt/lib/builtins/gcc_personality_v0.c index 0bc765624564..68581ef16434 100644 --- a/contrib/compiler-rt/lib/builtins/gcc_personality_v0.c +++ b/contrib/compiler-rt/lib/builtins/gcc_personality_v0.c @@ -206,8 +206,8 @@ __gcc_personality_v0(int version, _Unwind_Action actions, if ( lsda == (uint8_t*) 0 ) return continueUnwind(exceptionObject, context); - uintptr_t pc = _Unwind_GetIP(context)-1; - uintptr_t funcStart = _Unwind_GetRegionStart(context); + uintptr_t pc = (uintptr_t)_Unwind_GetIP(context)-1; + uintptr_t funcStart = (uintptr_t)_Unwind_GetRegionStart(context); uintptr_t pcOffset = pc - funcStart; /* Parse LSDA header. */ @@ -249,4 +249,3 @@ __gcc_personality_v0(int version, _Unwind_Action actions, /* No landing pad found, continue unwinding. */ return continueUnwind(exceptionObject, context); } - diff --git a/contrib/compiler-rt/lib/builtins/int_lib.h b/contrib/compiler-rt/lib/builtins/int_lib.h index 93394cebf7ea..e07867d5580f 100644 --- a/contrib/compiler-rt/lib/builtins/int_lib.h +++ b/contrib/compiler-rt/lib/builtins/int_lib.h @@ -45,7 +45,7 @@ #define AEABI_RTABI __attribute__((__pcs__("aapcs"))) -#ifdef _MSC_VER +#if defined(_MSC_VER) && !defined(__clang__) #define ALWAYS_INLINE __forceinline #define NOINLINE __declspec(noinline) #define NORETURN __declspec(noreturn) diff --git a/contrib/compiler-rt/lib/builtins/int_math.h b/contrib/compiler-rt/lib/builtins/int_math.h index fc81fb7f0220..aa3d0721a8ab 100644 --- a/contrib/compiler-rt/lib/builtins/int_math.h +++ b/contrib/compiler-rt/lib/builtins/int_math.h @@ -92,12 +92,8 @@ #endif #if defined(_MSC_VER) && !defined(__clang__) -#define crt_logb(x) logb((x)) -#define crt_logbf(x) logbf((x)) #define crt_logbl(x) logbl((x)) #else -#define crt_logb(x) __builtin_logb((x)) -#define crt_logbf(x) __builtin_logbf((x)) #define crt_logbl(x) __builtin_logbl((x)) #endif diff --git a/contrib/compiler-rt/lib/builtins/int_types.h b/contrib/compiler-rt/lib/builtins/int_types.h index 19002c68bd9d..4c91b75bac6c 100644 --- a/contrib/compiler-rt/lib/builtins/int_types.h +++ b/contrib/compiler-rt/lib/builtins/int_types.h @@ -60,10 +60,19 @@ typedef union }s; } udwords; -#if (defined(__LP64__) || defined(__wasm__) || defined(__mips64)) || defined(__riscv) +#if defined(__LP64__) || defined(__wasm__) || defined(__mips64) || \ + defined(__riscv) || defined(_WIN64) #define CRT_HAS_128BIT #endif +/* MSVC doesn't have a working 128bit integer type. Users should really compile + * compiler-rt with clang, but if they happen to be doing a standalone build for + * asan or something else, disable the 128 bit parts so things sort of work. + */ +#if defined(_MSC_VER) && !defined(__clang__) +#undef CRT_HAS_128BIT +#endif + #ifdef CRT_HAS_128BIT typedef int ti_int __attribute__ ((mode (TI))); typedef unsigned tu_int __attribute__ ((mode (TI))); @@ -139,6 +148,18 @@ typedef struct #endif /* _YUGA_LITTLE_ENDIAN */ } uqwords; +/* Check if the target supports 80 bit extended precision long doubles. + * Notably, on x86 Windows, MSVC only provides a 64-bit long double, but GCC + * still makes it 80 bits. Clang will match whatever compiler it is trying to + * be compatible with. + */ +#if ((defined(__i386__) || defined(__x86_64__)) && !defined(_MSC_VER)) || \ + defined(__m68k__) || defined(__ia64__) +#define HAS_80_BIT_LONG_DOUBLE 1 +#else +#define HAS_80_BIT_LONG_DOUBLE 0 +#endif + #ifndef _STANDALONE typedef union { diff --git a/contrib/compiler-rt/lib/builtins/int_util.c b/contrib/compiler-rt/lib/builtins/int_util.c index de87410dbca2..752f2015580e 100644 --- a/contrib/compiler-rt/lib/builtins/int_util.c +++ b/contrib/compiler-rt/lib/builtins/int_util.c @@ -27,7 +27,7 @@ NORETURN extern void panic(const char *, ...); #ifndef _WIN32 __attribute__((visibility("hidden"))) #endif -void compilerrt_abort_impl(const char *file, int line, const char *function) { +void __compilerrt_abort_impl(const char *file, int line, const char *function) { panic("%s:%d: abort in %s", file, line, function); } @@ -41,7 +41,7 @@ NORETURN extern void __assert_rtn(const char *func, const char *file, int line, __attribute__((weak)) __attribute__((visibility("hidden"))) #endif -void compilerrt_abort_impl(const char *file, int line, const char *function) { +void __compilerrt_abort_impl(const char *file, int line, const char *function) { __assert_rtn(function, file, line, "libcompiler_rt abort"); } @@ -51,7 +51,7 @@ void compilerrt_abort_impl(const char *file, int line, const char *function) { __attribute__((weak)) __attribute__((visibility("hidden"))) #endif -void compilerrt_abort_impl(const char *file, int line, const char *function) { +void __compilerrt_abort_impl(const char *file, int line, const char *function) { __builtin_trap(); } @@ -64,7 +64,7 @@ void compilerrt_abort_impl(const char *file, int line, const char *function) { __attribute__((weak)) __attribute__((visibility("hidden"))) #endif -void compilerrt_abort_impl(const char *file, int line, const char *function) { +void __compilerrt_abort_impl(const char *file, int line, const char *function) { abort(); } diff --git a/contrib/compiler-rt/lib/builtins/int_util.h b/contrib/compiler-rt/lib/builtins/int_util.h index a7b20ed66244..c3c87381ad8a 100644 --- a/contrib/compiler-rt/lib/builtins/int_util.h +++ b/contrib/compiler-rt/lib/builtins/int_util.h @@ -20,10 +20,10 @@ #define INT_UTIL_H /** \brief Trigger a program abort (or panic for kernel code). */ -#define compilerrt_abort() compilerrt_abort_impl(__FILE__, __LINE__, __func__) +#define compilerrt_abort() __compilerrt_abort_impl(__FILE__, __LINE__, __func__) -NORETURN void compilerrt_abort_impl(const char *file, int line, - const char *function); +NORETURN void __compilerrt_abort_impl(const char *file, int line, + const char *function); #define COMPILE_TIME_ASSERT(expr) COMPILE_TIME_ASSERT1(expr, __COUNTER__) #define COMPILE_TIME_ASSERT1(expr, cnt) COMPILE_TIME_ASSERT2(expr, cnt) diff --git a/contrib/compiler-rt/lib/builtins/os_version_check.c b/contrib/compiler-rt/lib/builtins/os_version_check.c index 772e33333c0f..e0d40edc7e38 100644 --- a/contrib/compiler-rt/lib/builtins/os_version_check.c +++ b/contrib/compiler-rt/lib/builtins/os_version_check.c @@ -15,7 +15,6 @@ #ifdef __APPLE__ -#include <CoreFoundation/CoreFoundation.h> #include <TargetConditionals.h> #include <dispatch/dispatch.h> #include <dlfcn.h> @@ -28,6 +27,33 @@ static int32_t GlobalMajor, GlobalMinor, GlobalSubminor; static dispatch_once_t DispatchOnceCounter; +/* We can't include <CoreFoundation/CoreFoundation.h> directly from here, so + * just forward declare everything that we need from it. */ + +typedef const void *CFDataRef, *CFAllocatorRef, *CFPropertyListRef, + *CFStringRef, *CFDictionaryRef, *CFTypeRef, *CFErrorRef; + +#if __LLP64__ +typedef unsigned long long CFTypeID; +typedef unsigned long long CFOptionFlags; +typedef signed long long CFIndex; +#else +typedef unsigned long CFTypeID; +typedef unsigned long CFOptionFlags; +typedef signed long CFIndex; +#endif + +typedef unsigned char UInt8; +typedef _Bool Boolean; +typedef CFIndex CFPropertyListFormat; +typedef uint32_t CFStringEncoding; + +/* kCFStringEncodingASCII analog. */ +#define CF_STRING_ENCODING_ASCII 0x0600 +/* kCFStringEncodingUTF8 analog. */ +#define CF_STRING_ENCODING_UTF8 0x08000100 +#define CF_PROPERTY_LIST_IMMUTABLE 0 + typedef CFDataRef (*CFDataCreateWithBytesNoCopyFuncTy)(CFAllocatorRef, const UInt8 *, CFIndex, CFAllocatorRef); @@ -55,8 +81,7 @@ static void parseSystemVersionPList(void *Unused) { const void *NullAllocator = dlsym(RTLD_DEFAULT, "kCFAllocatorNull"); if (!NullAllocator) return; - const CFAllocatorRef kCFAllocatorNull = - *(const CFAllocatorRef *)NullAllocator; + const CFAllocatorRef AllocatorNull = *(const CFAllocatorRef *)NullAllocator; CFDataCreateWithBytesNoCopyFuncTy CFDataCreateWithBytesNoCopyFunc = (CFDataCreateWithBytesNoCopyFuncTy)dlsym(RTLD_DEFAULT, "CFDataCreateWithBytesNoCopy"); @@ -140,21 +165,21 @@ static void parseSystemVersionPList(void *Unused) { /* Get the file buffer into CF's format. We pass in a null allocator here * * because we free PListBuf ourselves */ FileContentsRef = (*CFDataCreateWithBytesNoCopyFunc)( - NULL, PListBuf, (CFIndex)NumRead, kCFAllocatorNull); + NULL, PListBuf, (CFIndex)NumRead, AllocatorNull); if (!FileContentsRef) goto Fail; if (CFPropertyListCreateWithDataFunc) PListRef = (*CFPropertyListCreateWithDataFunc)( - NULL, FileContentsRef, kCFPropertyListImmutable, NULL, NULL); + NULL, FileContentsRef, CF_PROPERTY_LIST_IMMUTABLE, NULL, NULL); else PListRef = (*CFPropertyListCreateFromXMLDataFunc)( - NULL, FileContentsRef, kCFPropertyListImmutable, NULL); + NULL, FileContentsRef, CF_PROPERTY_LIST_IMMUTABLE, NULL); if (!PListRef) goto Fail; CFStringRef ProductVersion = (*CFStringCreateWithCStringNoCopyFunc)( - NULL, "ProductVersion", kCFStringEncodingASCII, kCFAllocatorNull); + NULL, "ProductVersion", CF_STRING_ENCODING_ASCII, AllocatorNull); if (!ProductVersion) goto Fail; CFTypeRef OpaqueValue = (*CFDictionaryGetValueFunc)(PListRef, ProductVersion); @@ -165,7 +190,7 @@ static void parseSystemVersionPList(void *Unused) { char VersionStr[32]; if (!(*CFStringGetCStringFunc)((CFStringRef)OpaqueValue, VersionStr, - sizeof(VersionStr), kCFStringEncodingUTF8)) + sizeof(VersionStr), CF_STRING_ENCODING_UTF8)) goto Fail; sscanf(VersionStr, "%d.%d.%d", &GlobalMajor, &GlobalMinor, &GlobalSubminor); diff --git a/contrib/compiler-rt/lib/builtins/ppc/divtc3.c b/contrib/compiler-rt/lib/builtins/ppc/divtc3.c index 8ec41c528ab9..ef532b841145 100644 --- a/contrib/compiler-rt/lib/builtins/ppc/divtc3.c +++ b/contrib/compiler-rt/lib/builtins/ppc/divtc3.c @@ -4,6 +4,11 @@ #include "DD.h" #include "../int_math.h" +// Use DOUBLE_PRECISION because the soft-fp method we use is logb (on the upper +// half of the long doubles), even though this file defines complex division for +// 128-bit floats. +#define DOUBLE_PRECISION +#include "../fp_lib.h" #if !defined(CRT_INFINITY) && defined(HUGE_VAL) #define CRT_INFINITY HUGE_VAL @@ -21,9 +26,10 @@ __divtc3(long double a, long double b, long double c, long double d) DD dDD = { .ld = d }; int ilogbw = 0; - const double logbw = crt_logb(crt_fmax(crt_fabs(cDD.s.hi), crt_fabs(dDD.s.hi) )); - - if (crt_isfinite(logbw)) + const double logbw = __compiler_rt_logb( + crt_fmax(crt_fabs(cDD.s.hi), crt_fabs(dDD.s.hi))); + + if (crt_isfinite(logbw)) { ilogbw = (int)logbw; diff --git a/contrib/compiler-rt/lib/builtins/ppc/fixunstfti.c b/contrib/compiler-rt/lib/builtins/ppc/fixunstfti.c new file mode 100644 index 000000000000..fa21084cb576 --- /dev/null +++ b/contrib/compiler-rt/lib/builtins/ppc/fixunstfti.c @@ -0,0 +1,106 @@ +//===-- lib/builtins/ppc/fixunstfti.c - Convert long double->int128 *-C -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements converting the 128bit IBM/PowerPC long double (double- +// double) data type to an unsigned 128 bit integer. +// +//===----------------------------------------------------------------------===// + +#include "../int_math.h" +#define BIAS 1023 + +/* Convert long double into an unsigned 128-bit integer. */ +__uint128_t __fixunstfti(long double input) { + + /* If we are trying to convert a NaN, return the NaN bit pattern. */ + if (crt_isnan(input)) { + return ((__uint128_t)0x7FF8000000000000ll) << 64 | + (__uint128_t)0x0000000000000000ll; + } + + __uint128_t result, hiResult, loResult; + int hiExponent, loExponent, shift; + /* The long double representation, with the high and low portions of + * the long double, and the corresponding bit patterns of each double. */ + union { + long double ld; + double d[2]; /* [0] is the high double, [1] is the low double. */ + unsigned long long ull[2]; /* High and low doubles as 64-bit integers. */ + } ldUnion; + + /* If the long double is less than 1.0 or negative, + * return 0.0. */ + if (input < 1.0) + return 0.0; + + /* Retrieve the 64-bit patterns of high and low doubles. + * Compute the unbiased exponent of both high and low doubles by + * removing the signs, isolating the exponent, and subtracting + * the bias from it. */ + ldUnion.ld = input; + hiExponent = ((ldUnion.ull[0] & 0x7FFFFFFFFFFFFFFFll) >> 52) - BIAS; + loExponent = ((ldUnion.ull[1] & 0x7FFFFFFFFFFFFFFFll) >> 52) - BIAS; + + /* Convert each double into int64; they will be added to the int128 result. + * CASE 1: High or low double fits in int64 + * - Convert the each double normally into int64. + * + * CASE 2: High or low double does not fit in int64 + * - Scale the double to fit within a 64-bit integer + * - Calculate the shift (amount to scale the double by in the int128) + * - Clear all the bits of the exponent (with 0x800FFFFFFFFFFFFF) + * - Add BIAS+53 (0x4350000000000000) to exponent to correct the value + * - Scale (move) the double to the correct place in the int128 + * (Move it by 2^53 places) + * + * Note: If the high double is assumed to be positive, an unsigned conversion + * from long double to 64-bit integer is needed. The low double can be either + * positive or negative, so a signed conversion is needed to retain the result + * of the low double and to ensure it does not simply get converted to 0. */ + + /* CASE 1 - High double fits in int64. */ + if (hiExponent < 63) { + hiResult = (unsigned long long)ldUnion.d[0]; + } else if (hiExponent < 128) { + /* CASE 2 - High double does not fit in int64, scale and convert it. */ + shift = hiExponent - 54; + ldUnion.ull[0] &= 0x800FFFFFFFFFFFFFll; + ldUnion.ull[0] |= 0x4350000000000000ll; + hiResult = (unsigned long long)ldUnion.d[0]; + hiResult <<= shift; + } else { + /* Detect cases for overflow. When the exponent of the high + * double is greater than 128 bits and when the long double + * input is positive, return the max 128-bit integer. + * For negative inputs with exponents > 128, return 1, like gcc. */ + if (ldUnion.d[0] > 0) { + return ((__uint128_t)0xFFFFFFFFFFFFFFFFll) << 64 | + (__uint128_t)0xFFFFFFFFFFFFFFFFll; + } else { + return ((__uint128_t)0x0000000000000000ll) << 64 | + (__uint128_t)0x0000000000000001ll; + } + } + + /* CASE 1 - Low double fits in int64. */ + if (loExponent < 63) { + loResult = (long long)ldUnion.d[1]; + } else { + /* CASE 2 - Low double does not fit in int64, scale and convert it. */ + shift = loExponent - 54; + ldUnion.ull[1] &= 0x800FFFFFFFFFFFFFll; + ldUnion.ull[1] |= 0x4350000000000000ll; + loResult = (long long)ldUnion.d[1]; + loResult <<= shift; + } + + /* Add the high and low doublewords together to form a 128 bit integer. */ + result = loResult + hiResult; + return result; +} diff --git a/contrib/compiler-rt/lib/builtins/ppc/floattitf.c b/contrib/compiler-rt/lib/builtins/ppc/floattitf.c new file mode 100644 index 000000000000..b8e297b6b8d9 --- /dev/null +++ b/contrib/compiler-rt/lib/builtins/ppc/floattitf.c @@ -0,0 +1,48 @@ +//===-- lib/builtins/ppc/floattitf.c - Convert int128->long double -*-C -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements converting a signed 128 bit integer to a 128bit IBM / +// PowerPC long double (double-double) value. +// +//===----------------------------------------------------------------------===// + +#include <stdint.h> + +/* Conversions from signed and unsigned 64-bit int to long double. */ +long double __floatditf(int64_t); +long double __floatunditf(uint64_t); + +/* Convert a signed 128-bit integer to long double. + * This uses the following property: Let hi and lo be 64-bits each, + * and let signed_val_k() and unsigned_val_k() be the value of the + * argument interpreted as a signed or unsigned k-bit integer. Then, + * + * signed_val_128(hi,lo) = signed_val_64(hi) * 2^64 + unsigned_val_64(lo) + * = (long double)hi * 2^64 + (long double)lo, + * + * where (long double)hi and (long double)lo are signed and + * unsigned 64-bit integer to long double conversions, respectively. + */ +long double __floattitf(__int128_t arg) { + /* Split the int128 argument into 64-bit high and low int64 parts. */ + int64_t ArgHiPart = (int64_t)(arg >> 64); + uint64_t ArgLoPart = (uint64_t)arg; + + /* Convert each 64-bit part into long double. The high part + * must be a signed conversion and the low part an unsigned conversion + * to ensure the correct result. */ + long double ConvertedHiPart = __floatditf(ArgHiPart); + long double ConvertedLoPart = __floatunditf(ArgLoPart); + + /* The low bit of ArgHiPart corresponds to the 2^64 bit in arg. + * Multiply the high part by 2^64 to undo the right shift by 64-bits + * done in the splitting. Then, add to the low part to obtain the + * final result. */ + return ((ConvertedHiPart * 0x1.0p64) + ConvertedLoPart); +} diff --git a/contrib/compiler-rt/lib/cfi/cfi.cc b/contrib/compiler-rt/lib/cfi/cfi.cc index a2f127f93cda..b0a943759678 100644 --- a/contrib/compiler-rt/lib/cfi/cfi.cc +++ b/contrib/compiler-rt/lib/cfi/cfi.cc @@ -13,15 +13,33 @@ #include <assert.h> #include <elf.h> + +#include "sanitizer_common/sanitizer_common.h" +#if SANITIZER_FREEBSD +#include <sys/link_elf.h> +#endif #include <link.h> #include <string.h> +#include <stdlib.h> #include <sys/mman.h> +#if SANITIZER_LINUX typedef ElfW(Phdr) Elf_Phdr; typedef ElfW(Ehdr) Elf_Ehdr; +typedef ElfW(Addr) Elf_Addr; +typedef ElfW(Sym) Elf_Sym; +typedef ElfW(Dyn) Elf_Dyn; +#elif SANITIZER_FREEBSD +#if SANITIZER_WORDSIZE == 64 +#define ElfW64_Dyn Elf_Dyn +#define ElfW64_Sym Elf_Sym +#else +#define ElfW32_Dyn Elf_Dyn +#define ElfW32_Sym Elf_Sym +#endif +#endif #include "interception/interception.h" -#include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_flag_parser.h" #include "ubsan/ubsan_init.h" #include "ubsan/ubsan_flags.h" @@ -154,15 +172,25 @@ void ShadowBuilder::Add(uptr begin, uptr end, uptr cfi_check) { *s = sv; } -#if SANITIZER_LINUX +#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD void ShadowBuilder::Install() { MprotectReadOnly(shadow_, GetShadowSize()); uptr main_shadow = GetShadow(); if (main_shadow) { // Update. +#if SANITIZER_LINUX void *res = mremap((void *)shadow_, GetShadowSize(), GetShadowSize(), MREMAP_MAYMOVE | MREMAP_FIXED, (void *)main_shadow); CHECK(res != MAP_FAILED); +#elif SANITIZER_NETBSD + void *res = mremap((void *)shadow_, GetShadowSize(), (void *)main_shadow, + GetShadowSize(), MAP_FIXED); + CHECK(res != MAP_FAILED); +#else + void *res = MmapFixedOrDie(shadow_, GetShadowSize()); + CHECK(res != MAP_FAILED); + ::memcpy(&shadow_, &main_shadow, GetShadowSize()); +#endif } else { // Initial setup. CHECK_EQ(kCfiShadowLimitsStorageSize, GetPageSizeCached()); @@ -183,17 +211,17 @@ void ShadowBuilder::Install() { // dlopen(RTLD_NOLOAD | RTLD_LAZY) // dlsym("__cfi_check"). uptr find_cfi_check_in_dso(dl_phdr_info *info) { - const ElfW(Dyn) *dynamic = nullptr; + const Elf_Dyn *dynamic = nullptr; for (int i = 0; i < info->dlpi_phnum; ++i) { if (info->dlpi_phdr[i].p_type == PT_DYNAMIC) { dynamic = - (const ElfW(Dyn) *)(info->dlpi_addr + info->dlpi_phdr[i].p_vaddr); + (const Elf_Dyn *)(info->dlpi_addr + info->dlpi_phdr[i].p_vaddr); break; } } if (!dynamic) return 0; uptr strtab = 0, symtab = 0, strsz = 0; - for (const ElfW(Dyn) *p = dynamic; p->d_tag != PT_NULL; ++p) { + for (const Elf_Dyn *p = dynamic; p->d_tag != PT_NULL; ++p) { if (p->d_tag == DT_SYMTAB) symtab = p->d_un.d_ptr; else if (p->d_tag == DT_STRTAB) @@ -227,7 +255,7 @@ uptr find_cfi_check_in_dso(dl_phdr_info *info) { return 0; } - for (const ElfW(Sym) *p = (const ElfW(Sym) *)symtab; (ElfW(Addr))p < strtab; + for (const Elf_Sym *p = (const Elf_Sym *)symtab; (Elf_Addr)p < strtab; ++p) { // There is no reliable way to find the end of the symbol table. In // lld-produces files, there are other sections between symtab and strtab. diff --git a/contrib/compiler-rt/lib/cfi/cfi_blacklist.txt b/contrib/compiler-rt/lib/cfi/cfi_blacklist.txt index 3d73508f5707..4a0f03949ca8 100644 --- a/contrib/compiler-rt/lib/cfi/cfi_blacklist.txt +++ b/contrib/compiler-rt/lib/cfi/cfi_blacklist.txt @@ -1,13 +1,11 @@ [cfi-unrelated-cast] # The specification of std::get_temporary_buffer mandates a cast to -# uninitialized T* (libstdc++, libc++, MSVC stdlib). +# uninitialized T* (libstdc++, MSVC stdlib). fun:_ZSt20get_temporary_buffer* -fun:_ZNSt3__120get_temporary_buffer* fun:*get_temporary_buffer@.*@std@@* -# STL address-of magic (libstdc++, libc++). +# STL address-of magic (libstdc++). fun:*__addressof* -fun:_ZNSt3__19addressof* # Windows C++ stdlib headers that contain bad unrelated casts. src:*xmemory0 diff --git a/contrib/compiler-rt/lib/dfsan/dfsan.cc b/contrib/compiler-rt/lib/dfsan/dfsan.cc index d4dbebc43a78..585bdceac764 100644 --- a/contrib/compiler-rt/lib/dfsan/dfsan.cc +++ b/contrib/compiler-rt/lib/dfsan/dfsan.cc @@ -423,7 +423,7 @@ static void dfsan_fini() { static void dfsan_init(int argc, char **argv, char **envp) { InitializeFlags(); - InitializePlatformEarly(); + ::InitializePlatformEarly(); if (!MmapFixedNoReserve(ShadowAddr(), UnusedAddr() - ShadowAddr())) Die(); diff --git a/contrib/compiler-rt/lib/esan/esan_interceptors.cpp b/contrib/compiler-rt/lib/esan/esan_interceptors.cpp index 0c596f1cf37e..833faa2cd5bc 100644 --- a/contrib/compiler-rt/lib/esan/esan_interceptors.cpp +++ b/contrib/compiler-rt/lib/esan/esan_interceptors.cpp @@ -327,7 +327,7 @@ INTERCEPTOR(int, rmdir, char *path) { // Signal-related interceptors //===----------------------------------------------------------------------===// -#if SANITIZER_LINUX +#if SANITIZER_LINUX || SANITIZER_FREEBSD typedef void (*signal_handler_t)(int); INTERCEPTOR(signal_handler_t, signal, int signum, signal_handler_t handler) { void *ctx; @@ -344,7 +344,7 @@ INTERCEPTOR(signal_handler_t, signal, int signum, signal_handler_t handler) { #define ESAN_MAYBE_INTERCEPT_SIGNAL #endif -#if SANITIZER_LINUX +#if SANITIZER_LINUX || SANITIZER_FREEBSD DECLARE_REAL(int, sigaction, int signum, const struct sigaction *act, struct sigaction *oldact) INTERCEPTOR(int, sigaction, int signum, const struct sigaction *act, @@ -363,7 +363,11 @@ int real_sigaction(int signum, const void *act, void *oldact) { if (REAL(sigaction) == nullptr) { // With an instrumented allocator, this is called during interceptor init // and we need a raw syscall solution. +#if SANITIZER_LINUX return internal_sigaction_syscall(signum, act, oldact); +#else + return internal_sigaction(signum, act, oldact); +#endif } return REAL(sigaction)(signum, (const struct sigaction *)act, (struct sigaction *)oldact); @@ -376,7 +380,7 @@ int real_sigaction(int signum, const void *act, void *oldact) { #define ESAN_MAYBE_INTERCEPT_SIGACTION #endif -#if SANITIZER_LINUX +#if SANITIZER_LINUX || SANITIZER_FREEBSD INTERCEPTOR(int, sigprocmask, int how, __sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset) { void *ctx; diff --git a/contrib/compiler-rt/lib/esan/esan_shadow.h b/contrib/compiler-rt/lib/esan/esan_shadow.h index 72a919a190d8..b76a9ccbd9e4 100644 --- a/contrib/compiler-rt/lib/esan/esan_shadow.h +++ b/contrib/compiler-rt/lib/esan/esan_shadow.h @@ -30,7 +30,7 @@ struct ApplicationRegion { bool ShadowMergedWithPrev; }; -#if SANITIZER_LINUX && defined(__x86_64__) +#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && defined(__x86_64__) // Linux x86_64 // // Application memory falls into these 5 regions (ignoring the corner case diff --git a/contrib/compiler-rt/lib/esan/esan_sideline.h b/contrib/compiler-rt/lib/esan/esan_sideline.h index 04aff22f4cf0..74551fbde850 100644 --- a/contrib/compiler-rt/lib/esan/esan_sideline.h +++ b/contrib/compiler-rt/lib/esan/esan_sideline.h @@ -17,6 +17,7 @@ #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_internal_defs.h" +#include "sanitizer_common/sanitizer_platform_limits_freebsd.h" #include "sanitizer_common/sanitizer_platform_limits_posix.h" namespace __esan { diff --git a/contrib/compiler-rt/lib/esan/esan_sideline_bsd.cpp b/contrib/compiler-rt/lib/esan/esan_sideline_bsd.cpp new file mode 100644 index 000000000000..3134d3776730 --- /dev/null +++ b/contrib/compiler-rt/lib/esan/esan_sideline_bsd.cpp @@ -0,0 +1,35 @@ +//===-- esan_sideline_bsd.cpp -----------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of EfficiencySanitizer, a family of performance tuners. +// +// Support for a separate or "sideline" tool thread on FreeBSD. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#if SANITIZER_FREEBSD + +#include "esan_sideline.h" + +namespace __esan { + +static SidelineThread *TheThread; + +bool SidelineThread::launchThread(SidelineFunc takeSample, void *Arg, + u32 FreqMilliSec) { + return true; +} + +bool SidelineThread::joinThread() { + return true; +} + +} // namespace __esan + +#endif // SANITIZER_FREEBSD diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerBuiltins.h b/contrib/compiler-rt/lib/fuzzer/FuzzerBuiltins.h new file mode 100644 index 000000000000..a80938d9a54d --- /dev/null +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerBuiltins.h @@ -0,0 +1,36 @@ +//===- FuzzerBuiltins.h - Internal header for builtins ----------*- C++ -* ===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// Wrapper functions and marcos around builtin functions. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_BUILTINS_H +#define LLVM_FUZZER_BUILTINS_H + +#include "FuzzerDefs.h" + +#if !LIBFUZZER_MSVC +#include <cstdint> + +#define GET_CALLER_PC() __builtin_return_address(0) + +namespace fuzzer { + +inline uint8_t Bswap(uint8_t x) { return x; } +inline uint16_t Bswap(uint16_t x) { return __builtin_bswap16(x); } +inline uint32_t Bswap(uint32_t x) { return __builtin_bswap32(x); } +inline uint64_t Bswap(uint64_t x) { return __builtin_bswap64(x); } + +inline uint32_t Clzll(unsigned long long X) { return __builtin_clzll(X); } +inline uint32_t Clz(unsigned long long X) { return __builtin_clz(X); } +inline int Popcountll(unsigned long long X) { return __builtin_popcountll(X); } + +} // namespace fuzzer + +#endif // !LIBFUZZER_MSVC +#endif // LLVM_FUZZER_BUILTINS_H diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerBuiltinsMsvc.h b/contrib/compiler-rt/lib/fuzzer/FuzzerBuiltinsMsvc.h new file mode 100644 index 000000000000..67dd57ff9a93 --- /dev/null +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerBuiltinsMsvc.h @@ -0,0 +1,59 @@ +//===- FuzzerBuiltinsMSVC.h - Internal header for builtins ------*- C++ -* ===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// Wrapper functions and marcos that use intrinsics instead of builtin functions +// which cannot be compiled by MSVC. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_BUILTINS_MSVC_H +#define LLVM_FUZZER_BUILTINS_MSVC_H + +#include "FuzzerDefs.h" + +#if LIBFUZZER_MSVC +#if !defined(_M_ARM) && !defined(_M_X64) +#error "_BitScanReverse64 unavailable on this platform so MSVC is unsupported." +#endif +#include <intrin.h> +#include <cstdint> +#include <cstdlib> + +// __builtin_return_address() cannot be compiled with MSVC. Use the equivalent +// from <intrin.h> +#define GET_CALLER_PC() reinterpret_cast<uintptr_t>(_ReturnAddress()) + +namespace fuzzer { + +inline uint8_t Bswap(uint8_t x) { return x; } +// Use alternatives to __builtin functions from <stdlib.h> and <intrin.h> on +// Windows since the builtins are not supported by MSVC. +inline uint16_t Bswap(uint16_t x) { return _byteswap_ushort(x); } +inline uint32_t Bswap(uint32_t x) { return _byteswap_ulong(x); } +inline uint64_t Bswap(uint64_t x) { return _byteswap_uint64(x); } + +// The functions below were mostly copied from +// compiler-rt/lib/builtins/int_lib.h which defines the __builtin functions used +// outside of Windows. +inline uint32_t Clzll(uint64_t X) { + unsigned long LeadZeroIdx = 0; + if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx; + return 64; +} + +inline uint32_t Clz(uint32_t X) { + unsigned long LeadZeroIdx = 0; + if (_BitScanReverse(&LeadZeroIdx, X)) return 31 - LeadZeroIdx; + return 32; +} + +inline int Popcountll(unsigned long long X) { return __popcnt64(X); } + +} // namespace fuzzer + +#endif // LIBFUZER_MSVC +#endif // LLVM_FUZZER_BUILTINS_MSVC_H diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerCommand.h b/contrib/compiler-rt/lib/fuzzer/FuzzerCommand.h index 255f571ecf31..9d258a228f89 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerCommand.h +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerCommand.h @@ -81,7 +81,7 @@ public: } // Like hasArgument, but checks for "-[Flag]=...". - bool hasFlag(const std::string &Flag) { + bool hasFlag(const std::string &Flag) const { std::string Arg("-" + Flag + "="); auto IsMatch = [&](const std::string &Other) { return Arg.compare(0, std::string::npos, Other, 0, Arg.length()) == 0; @@ -92,7 +92,7 @@ public: // Returns the value of the first instance of a given flag, or an empty string // if the flag isn't present. Ignores any occurrences after // "-ignore_remaining_args=1", if present. - std::string getFlagValue(const std::string &Flag) { + std::string getFlagValue(const std::string &Flag) const { std::string Arg("-" + Flag + "="); auto IsMatch = [&](const std::string &Other) { return Arg.compare(0, std::string::npos, Other, 0, Arg.length()) == 0; diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerCorpus.h b/contrib/compiler-rt/lib/fuzzer/FuzzerCorpus.h index 8ad14656cffc..f844c07c7572 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerCorpus.h +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerCorpus.h @@ -238,12 +238,6 @@ class InputCorpus { return false; } - bool IsFeatureNew(size_t Idx, uint32_t NewSize, bool Shrink) { - assert(NewSize); - uint32_t OldSize = GetFeature(Idx % kFeatureSetSize); - return OldSize == 0 || (Shrink && OldSize > NewSize); - } - size_t NumFeatures() const { return NumAddedFeatures; } size_t NumFeatureUpdates() const { return NumUpdatedFeatures; } diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerDefs.h b/contrib/compiler-rt/lib/fuzzer/FuzzerDefs.h index a35c7a181b74..c3dccbcd86f5 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerDefs.h +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerDefs.h @@ -82,6 +82,13 @@ #error "Support for your platform has not been implemented" #endif +#if defined(_MSC_VER) && !defined(__clang__) +// MSVC compiler is being used. +#define LIBFUZZER_MSVC 1 +#else +#define LIBFUZZER_MSVC 0 +#endif + #ifndef __has_attribute # define __has_attribute(x) 0 #endif @@ -129,8 +136,15 @@ #if LIBFUZZER_WINDOWS #define ATTRIBUTE_INTERFACE __declspec(dllexport) +// This is used for __sancov_lowest_stack which is needed for +// -fsanitize-coverage=stack-depth. That feature is not yet available on +// Windows, so make the symbol static to avoid linking errors. +#define ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC \ + __attribute__((tls_model("initial-exec"))) thread_local static #else #define ATTRIBUTE_INTERFACE __attribute__((visibility("default"))) +#define ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC \ + ATTRIBUTE_INTERFACE __attribute__((tls_model("initial-exec"))) thread_local #endif namespace fuzzer { @@ -176,11 +190,6 @@ typedef int (*UserCallback)(const uint8_t *Data, size_t Size); int FuzzerDriver(int *argc, char ***argv, UserCallback Callback); -inline uint8_t Bswap(uint8_t x) { return x; } -inline uint16_t Bswap(uint16_t x) { return __builtin_bswap16(x); } -inline uint32_t Bswap(uint32_t x) { return __builtin_bswap32(x); } -inline uint64_t Bswap(uint64_t x) { return __builtin_bswap64(x); } - uint8_t *ExtraCountersBegin(); uint8_t *ExtraCountersEnd(); void ClearExtraCounters(); diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerDriver.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerDriver.cpp index 783474a39e16..ff2a639ac4a3 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerDriver.cpp +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerDriver.cpp @@ -615,13 +615,8 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Options.PrintNewCovPcs = Flags.print_pcs; Options.PrintNewCovFuncs = Flags.print_funcs; Options.PrintFinalStats = Flags.print_final_stats; - Options.PrintMutationStats = Flags.print_mutation_stats; Options.PrintCorpusStats = Flags.print_corpus_stats; Options.PrintCoverage = Flags.print_coverage; - Options.PrintUnstableStats = Flags.print_unstable_stats; - if (Flags.handle_unstable == TracePC::MinUnstable || - Flags.handle_unstable == TracePC::ZeroUnstable) - Options.HandleUnstable = Flags.handle_unstable; Options.DumpCoverage = Flags.dump_coverage; if (Flags.exit_on_src_pos) Options.ExitOnSrcPos = Flags.exit_on_src_pos; diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsDlsymWin.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsDlsymWin.cpp deleted file mode 100644 index 321b3ec5d414..000000000000 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsDlsymWin.cpp +++ /dev/null @@ -1,62 +0,0 @@ -//===- FuzzerExtFunctionsDlsymWin.cpp - Interface to external functions ---===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// Implementation using dynamic loading for Windows. -//===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" -#if LIBFUZZER_WINDOWS - -#include "FuzzerExtFunctions.h" -#include "FuzzerIO.h" -#include "Windows.h" - -// This must be included after Windows.h. -#include "Psapi.h" - -namespace fuzzer { - -ExternalFunctions::ExternalFunctions() { - HMODULE Modules[1024]; - DWORD BytesNeeded; - HANDLE CurrentProcess = GetCurrentProcess(); - - if (!EnumProcessModules(CurrentProcess, Modules, sizeof(Modules), - &BytesNeeded)) { - Printf("EnumProcessModules failed (error: %d).\n", GetLastError()); - exit(1); - } - - if (sizeof(Modules) < BytesNeeded) { - Printf("Error: the array is not big enough to hold all loaded modules.\n"); - exit(1); - } - - for (size_t i = 0; i < (BytesNeeded / sizeof(HMODULE)); i++) - { - FARPROC Fn; -#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ - if (this->NAME == nullptr) { \ - Fn = GetProcAddress(Modules[i], #NAME); \ - if (Fn == nullptr) \ - Fn = GetProcAddress(Modules[i], #NAME "__dll"); \ - this->NAME = (decltype(ExternalFunctions::NAME)) Fn; \ - } -#include "FuzzerExtFunctions.def" -#undef EXT_FUNC - } - -#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ - if (this->NAME == nullptr && WARN) \ - Printf("WARNING: Failed to find function \"%s\".\n", #NAME); -#include "FuzzerExtFunctions.def" -#undef EXT_FUNC -} - -} // namespace fuzzer - -#endif // LIBFUZZER_WINDOWS diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWeak.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWeak.cpp index a4e56fc27b8d..6a6ef4932f41 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWeak.cpp +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWeak.cpp @@ -22,7 +22,7 @@ extern "C" { // Declare these symbols as weak to allow them to be optionally defined. #define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ - __attribute__((weak)) RETURN_TYPE NAME FUNC_SIG + __attribute__((weak, visibility("default"))) RETURN_TYPE NAME FUNC_SIG #include "FuzzerExtFunctions.def" diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWeakAlias.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWeakAlias.cpp deleted file mode 100644 index e10f7b4dcac2..000000000000 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWeakAlias.cpp +++ /dev/null @@ -1,56 +0,0 @@ -//===- FuzzerExtFunctionsWeakAlias.cpp - Interface to external functions --===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// Implementation using weak aliases. Works for Windows. -//===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" -#if LIBFUZZER_WINDOWS - -#include "FuzzerExtFunctions.h" -#include "FuzzerIO.h" - -using namespace fuzzer; - -extern "C" { -// Declare these symbols as weak to allow them to be optionally defined. -#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ - RETURN_TYPE NAME##Def FUNC_SIG { \ - Printf("ERROR: Function \"%s\" not defined.\n", #NAME); \ - exit(1); \ - } \ - RETURN_TYPE NAME FUNC_SIG __attribute__((weak, alias(#NAME "Def"))); - -#include "FuzzerExtFunctions.def" - -#undef EXT_FUNC -} - -template <typename T> -static T *GetFnPtr(T *Fun, T *FunDef, const char *FnName, bool WarnIfMissing) { - if (Fun == FunDef) { - if (WarnIfMissing) - Printf("WARNING: Failed to find function \"%s\".\n", FnName); - return nullptr; - } - return Fun; -} - -namespace fuzzer { - -ExternalFunctions::ExternalFunctions() { -#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ - this->NAME = GetFnPtr<decltype(::NAME)>(::NAME, ::NAME##Def, #NAME, WARN); - -#include "FuzzerExtFunctions.def" - -#undef EXT_FUNC -} - -} // namespace fuzzer - -#endif // LIBFUZZER_WINDOWS diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWindows.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWindows.cpp new file mode 100644 index 000000000000..b01871439ca0 --- /dev/null +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerExtFunctionsWindows.cpp @@ -0,0 +1,83 @@ +//=== FuzzerExtWindows.cpp - Interface to external functions --------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// Implementation of FuzzerExtFunctions for Windows. Uses alternatename when +// compiled with MSVC. Uses weak aliases when compiled with clang. Unfortunately +// the method each compiler supports is not supported by the other. +//===----------------------------------------------------------------------===// +#include "FuzzerDefs.h" +#if LIBFUZZER_WINDOWS + +#include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" + +using namespace fuzzer; + +// Intermediate macro to ensure the parameter is expanded before stringified. +#define STRINGIFY_(A) #A +#define STRINGIFY(A) STRINGIFY_(A) + +#if LIBFUZZER_MSVC +// Copied from compiler-rt/lib/sanitizer_common/sanitizer_win_defs.h +#if defined(_M_IX86) || defined(__i386__) +#define WIN_SYM_PREFIX "_" +#else +#define WIN_SYM_PREFIX +#endif + +// Declare external functions as having alternativenames, so that we can +// determine if they are not defined. +#define EXTERNAL_FUNC(Name, Default) \ + __pragma(comment(linker, "/alternatename:" WIN_SYM_PREFIX STRINGIFY( \ + Name) "=" WIN_SYM_PREFIX STRINGIFY(Default))) +#else +// Declare external functions as weak to allow them to default to a specified +// function if not defined explicitly. We must use weak symbols because clang's +// support for alternatename is not 100%, see +// https://bugs.llvm.org/show_bug.cgi?id=40218 for more details. +#define EXTERNAL_FUNC(Name, Default) \ + __attribute__((weak, alias(STRINGIFY(Default)))) +#endif // LIBFUZZER_MSVC + +extern "C" { +#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + RETURN_TYPE NAME##Def FUNC_SIG { \ + Printf("ERROR: Function \"%s\" not defined.\n", #NAME); \ + exit(1); \ + } \ + EXTERNAL_FUNC(NAME, NAME##Def) RETURN_TYPE NAME FUNC_SIG; + +#include "FuzzerExtFunctions.def" + +#undef EXT_FUNC +} + +template <typename T> +static T *GetFnPtr(T *Fun, T *FunDef, const char *FnName, bool WarnIfMissing) { + if (Fun == FunDef) { + if (WarnIfMissing) + Printf("WARNING: Failed to find function \"%s\".\n", FnName); + return nullptr; + } + return Fun; +} + +namespace fuzzer { + +ExternalFunctions::ExternalFunctions() { +#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + this->NAME = GetFnPtr<decltype(::NAME)>(::NAME, ::NAME##Def, #NAME, WARN); + +#include "FuzzerExtFunctions.def" + +#undef EXT_FUNC +} + +} // namespace fuzzer + +#endif // LIBFUZZER_WINDOWS diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerFlags.def b/contrib/compiler-rt/lib/fuzzer/FuzzerFlags.def index ba04bc25fd45..91281c979c50 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerFlags.def +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerFlags.def @@ -17,7 +17,7 @@ FUZZER_FLAG_INT(runs, -1, FUZZER_FLAG_INT(max_len, 0, "Maximum length of the test input. " "If 0, libFuzzer tries to guess a good value based on the corpus " "and reports it. ") -FUZZER_FLAG_INT(len_control, 1000, "Try generating small inputs first, " +FUZZER_FLAG_INT(len_control, 100, "Try generating small inputs first, " "then try larger inputs over time. Specifies the rate at which the length " "limit is increased (smaller == faster). If 0, immediately try inputs with " "size up to max_len.") @@ -110,15 +110,6 @@ FUZZER_FLAG_INT(print_coverage, 0, "If 1, print coverage information as text" FUZZER_FLAG_INT(dump_coverage, 0, "Deprecated." " If 1, dump coverage information as a" " .sancov file at exit.") -FUZZER_FLAG_INT(handle_unstable, 0, "Experimental." - " Executes every input 3 times in total if a unique feature" - " is found during the first execution." - " If 1, we only use the minimum hit count from the 3 runs" - " to determine whether an input is interesting." - " If 2, we disregard edges that are found unstable for" - " feature collection.") -FUZZER_FLAG_INT(print_unstable_stats, 0, "Experimental." - " If 1, print unstable statistics at exit.") FUZZER_FLAG_INT(handle_segv, 1, "If 1, try to intercept SIGSEGV.") FUZZER_FLAG_INT(handle_bus, 1, "If 1, try to intercept SIGBUS.") FUZZER_FLAG_INT(handle_abrt, 1, "If 1, try to intercept SIGABRT.") @@ -162,4 +153,3 @@ FUZZER_DEPRECATED_FLAG(use_equivalence_server) FUZZER_FLAG_INT(analyze_dict, 0, "Experimental") FUZZER_DEPRECATED_FLAG(use_clang_coverage) FUZZER_FLAG_STRING(data_flow_trace, "Experimental: use the data flow trace") -FUZZER_FLAG_INT(print_mutation_stats, 0, "Experimental") diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerIO.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerIO.cpp index f3ead0ec5357..c4c31e82471f 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerIO.cpp +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerIO.cpp @@ -31,7 +31,7 @@ long GetEpoch(const std::string &Path) { } Unit FileToVector(const std::string &Path, size_t MaxSize, bool ExitOnError) { - std::ifstream T(Path); + std::ifstream T(Path, std::ios::binary); if (ExitOnError && !T) { Printf("No such directory: %s; exiting\n", Path.c_str()); exit(1); @@ -51,7 +51,7 @@ Unit FileToVector(const std::string &Path, size_t MaxSize, bool ExitOnError) { } std::string FileToString(const std::string &Path) { - std::ifstream T(Path); + std::ifstream T(Path, std::ios::binary); return std::string((std::istreambuf_iterator<char>(T)), std::istreambuf_iterator<char>()); } @@ -100,14 +100,6 @@ std::string DirPlusFile(const std::string &DirPath, return DirPath + GetSeparator() + FileName; } -std::string Basename(const std::string &Path, char Separator) { - size_t Pos = Path.rfind(Separator); - if (Pos == std::string::npos) - return Path; - assert(Pos < Path.size()); - return Path.substr(Pos + 1); -} - void DupAndCloseStderr() { int OutputFd = DuplicateFile(2); if (OutputFd > 0) { diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerIO.h b/contrib/compiler-rt/lib/fuzzer/FuzzerIO.h index 6d7757435b7b..b4a68190e780 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerIO.h +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerIO.h @@ -68,7 +68,7 @@ void GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V); char GetSeparator(); // Similar to the basename utility: returns the file name w/o the dir prefix. -std::string Basename(const std::string &Path, char Separator = GetSeparator()); +std::string Basename(const std::string &Path); FILE* OpenFile(int Fd, const char *Mode); diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerIOPosix.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerIOPosix.cpp index 17e884d3c4c3..401b4cbbf74f 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerIOPosix.cpp +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerIOPosix.cpp @@ -46,6 +46,13 @@ size_t FileSize(const std::string &Path) { return St.st_size; } +std::string Basename(const std::string &Path) { + size_t Pos = Path.rfind(GetSeparator()); + if (Pos == std::string::npos) return Path; + assert(Pos < Path.size()); + return Path.substr(Pos + 1); +} + void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, Vector<std::string> *V, bool TopDir) { auto E = GetEpoch(Dir); diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerIOWindows.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerIOWindows.cpp index 74853646b213..75dcaf72a940 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerIOWindows.cpp +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerIOWindows.cpp @@ -72,6 +72,26 @@ bool IsFile(const std::string &Path) { return IsFile(Path, Att); } +std::string Basename(const std::string &Path) { + size_t Pos = Path.find_last_of("/\\"); + if (Pos == std::string::npos) return Path; + assert(Pos < Path.size()); + return Path.substr(Pos + 1); +} + +size_t FileSize(const std::string &Path) { + WIN32_FILE_ATTRIBUTE_DATA attr; + if (!GetFileAttributesExA(Path.c_str(), GetFileExInfoStandard, &attr)) { + Printf("GetFileAttributesExA() failed for \"%s\" (Error code: %lu).\n", + Path.c_str(), GetLastError()); + return 0; + } + ULARGE_INTEGER size; + size.HighPart = attr.nFileSizeHigh; + size.LowPart = attr.nFileSizeLow; + return size.QuadPart; +} + void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, Vector<std::string> *V, bool TopDir) { auto E = GetEpoch(Dir); @@ -91,7 +111,7 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, { if (GetLastError() == ERROR_FILE_NOT_FOUND) return; - Printf("No such directory: %s; exiting\n", Dir.c_str()); + Printf("No such file or directory: %s; exiting\n", Dir.c_str()); exit(1); } diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerInternal.h b/contrib/compiler-rt/lib/fuzzer/FuzzerInternal.h index bfc898248adb..a7fdc89cba5f 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerInternal.h +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerInternal.h @@ -67,7 +67,6 @@ public: static void StaticGracefulExitCallback(); void ExecuteCallback(const uint8_t *Data, size_t Size); - void CheckForUnstableCounters(const uint8_t *Data, size_t Size); bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false, InputInfo *II = nullptr, bool *FoundUniqFeatures = nullptr); diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerLoop.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerLoop.cpp index 4bc88365a0b9..a32a30723004 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerLoop.cpp +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerLoop.cpp @@ -275,7 +275,8 @@ NO_SANITIZE_MEMORY void Fuzzer::AlarmCallback() { assert(Options.UnitTimeoutSec > 0); // In Windows Alarm callback is executed by a different thread. -#if !LIBFUZZER_WINDOWS + // NetBSD's current behavior needs this change too. +#if !LIBFUZZER_WINDOWS && !LIBFUZZER_NETBSD if (!InFuzzingThread()) return; #endif @@ -354,13 +355,10 @@ void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units) { void Fuzzer::PrintFinalStats() { if (Options.PrintCoverage) TPC.PrintCoverage(); - if (Options.PrintUnstableStats) - TPC.PrintUnstableStats(); if (Options.DumpCoverage) TPC.DumpCoverage(); if (Options.PrintCorpusStats) Corpus.PrintStats(); - if (Options.PrintMutationStats) MD.PrintMutationStats(); if (!Options.PrintFinalStats) return; size_t ExecPerSec = execPerSec(); @@ -449,34 +447,6 @@ void Fuzzer::PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size) { } } -void Fuzzer::CheckForUnstableCounters(const uint8_t *Data, size_t Size) { - auto CBSetupAndRun = [&]() { - ScopedEnableMsanInterceptorChecks S; - UnitStartTime = system_clock::now(); - TPC.ResetMaps(); - RunningUserCallback = true; - CB(Data, Size); - RunningUserCallback = false; - UnitStopTime = system_clock::now(); - }; - - // Copy original run counters into our unstable counters - TPC.InitializeUnstableCounters(); - - // First Rerun - CBSetupAndRun(); - TPC.UpdateUnstableCounters(Options.HandleUnstable); - - // Second Rerun - CBSetupAndRun(); - TPC.UpdateUnstableCounters(Options.HandleUnstable); - - // Move minimum hit counts back to ModuleInline8bitCounters - if (Options.HandleUnstable == TracePC::MinUnstable || - Options.HandleUnstable == TracePC::ZeroUnstable) - TPC.ApplyUnstableCounters(); -} - bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, InputInfo *II, bool *FoundUniqFeatures) { if (!Size) @@ -487,17 +457,6 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, UniqFeatureSetTmp.clear(); size_t FoundUniqFeaturesOfII = 0; size_t NumUpdatesBefore = Corpus.NumFeatureUpdates(); - bool NewFeaturesUnstable = false; - - if (Options.HandleUnstable || Options.PrintUnstableStats) { - TPC.CollectFeatures([&](size_t Feature) { - if (Corpus.IsFeatureNew(Feature, Size, Options.Shrink)) - NewFeaturesUnstable = true; - }); - if (NewFeaturesUnstable) - CheckForUnstableCounters(Data, Size); - } - TPC.CollectFeatures([&](size_t Feature) { if (Corpus.AddFeature(Feature, Size, Options.Shrink)) UniqFeatureSetTmp.push_back(Feature); @@ -506,12 +465,10 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, II->UniqFeatureSet.end(), Feature)) FoundUniqFeaturesOfII++; }); - if (FoundUniqFeatures) *FoundUniqFeatures = FoundUniqFeaturesOfII; PrintPulseAndReportSlowInput(Data, Size); size_t NumNewFeatures = Corpus.NumFeatureUpdates() - NumUpdatesBefore; - if (NumNewFeatures) { TPC.UpdateObservedPCs(); Corpus.AddToCorpus({Data, Data + Size}, NumNewFeatures, MayDeleteFile, diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerMutate.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerMutate.cpp index ff076cca683f..142b2b0b0014 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerMutate.cpp +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerMutate.cpp @@ -30,36 +30,34 @@ MutationDispatcher::MutationDispatcher(Random &Rand, DefaultMutators.insert( DefaultMutators.begin(), { - {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes", 0, 0}, - {&MutationDispatcher::Mutate_InsertByte, "InsertByte", 0, 0}, + {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes"}, + {&MutationDispatcher::Mutate_InsertByte, "InsertByte"}, {&MutationDispatcher::Mutate_InsertRepeatedBytes, - "InsertRepeatedBytes", 0, 0}, - {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte", 0, 0}, - {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit", 0, 0}, - {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes", 0, 0}, - {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt", 0, - 0}, - {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt", 0, - 0}, - {&MutationDispatcher::Mutate_CopyPart, "CopyPart", 0, 0}, - {&MutationDispatcher::Mutate_CrossOver, "CrossOver", 0, 0}, + "InsertRepeatedBytes"}, + {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte"}, + {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit"}, + {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes"}, + {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt"}, + {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt"}, + {&MutationDispatcher::Mutate_CopyPart, "CopyPart"}, + {&MutationDispatcher::Mutate_CrossOver, "CrossOver"}, {&MutationDispatcher::Mutate_AddWordFromManualDictionary, - "ManualDict", 0, 0}, + "ManualDict"}, {&MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary, - "PersAutoDict", 0, 0}, + "PersAutoDict"}, }); if(Options.UseCmp) DefaultMutators.push_back( - {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP", 0, 0}); + {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP"}); if (EF->LLVMFuzzerCustomMutator) - Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom", 0, 0}); + Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom"}); else Mutators = DefaultMutators; if (EF->LLVMFuzzerCustomCrossOver) Mutators.push_back( - {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver", 0, 0}); + {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver"}); } static char RandCh(Random &Rand) { @@ -466,7 +464,6 @@ void MutationDispatcher::RecordSuccessfulMutationSequence() { if (!PersistentAutoDictionary.ContainsWord(DE->GetW())) PersistentAutoDictionary.push_back({DE->GetW(), 1}); } - RecordUsefulMutations(); } void MutationDispatcher::PrintRecommendedDictionary() { @@ -487,7 +484,8 @@ void MutationDispatcher::PrintRecommendedDictionary() { void MutationDispatcher::PrintMutationSequence() { Printf("MS: %zd ", CurrentMutatorSequence.size()); - for (auto M : CurrentMutatorSequence) Printf("%s-", M->Name); + for (auto M : CurrentMutatorSequence) + Printf("%s-", M.Name); if (!CurrentDictionaryEntrySequence.empty()) { Printf(" DE: "); for (auto DE : CurrentDictionaryEntrySequence) { @@ -515,13 +513,12 @@ size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size, // in which case they will return 0. // Try several times before returning un-mutated data. for (int Iter = 0; Iter < 100; Iter++) { - auto M = &Mutators[Rand(Mutators.size())]; - size_t NewSize = (this->*(M->Fn))(Data, Size, MaxSize); + auto M = Mutators[Rand(Mutators.size())]; + size_t NewSize = (this->*(M.Fn))(Data, Size, MaxSize); if (NewSize && NewSize <= MaxSize) { if (Options.OnlyASCII) ToASCII(Data, NewSize); CurrentMutatorSequence.push_back(M); - M->TotalCount++; return NewSize; } } @@ -562,21 +559,4 @@ void MutationDispatcher::AddWordToManualDictionary(const Word &W) { {W, std::numeric_limits<size_t>::max()}); } -void MutationDispatcher::RecordUsefulMutations() { - for (auto M : CurrentMutatorSequence) M->UsefulCount++; -} - -void MutationDispatcher::PrintMutationStats() { - Printf("\nstat::mutation_usefulness: "); - for (size_t i = 0; i < Mutators.size(); i++) { - double UsefulPercentage = - Mutators[i].TotalCount - ? (100.0 * Mutators[i].UsefulCount) / Mutators[i].TotalCount - : 0; - Printf("%.3f", UsefulPercentage); - if (i < Mutators.size() - 1) Printf(","); - } - Printf("\n"); -} - } // namespace fuzzer diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerMutate.h b/contrib/compiler-rt/lib/fuzzer/FuzzerMutate.h index 828ecc13d866..a51c7fb44d46 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerMutate.h +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerMutate.h @@ -93,16 +93,10 @@ public: Random &GetRand() { return Rand; } - void PrintMutationStats(); - - void RecordUsefulMutations(); - private: struct Mutator { size_t (MutationDispatcher::*Fn)(uint8_t *Data, size_t Size, size_t Max); const char *Name; - uint64_t UsefulCount; - uint64_t TotalCount; }; size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size, @@ -141,7 +135,6 @@ public: Dictionary PersistentAutoDictionary; Vector<DictionaryEntry *> CurrentDictionaryEntrySequence; - Vector<Mutator *> CurrentMutatorSequence; static const size_t kCmpDictionaryEntriesDequeSize = 16; DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize]; @@ -156,6 +149,7 @@ public: Vector<Mutator> Mutators; Vector<Mutator> DefaultMutators; + Vector<Mutator> CurrentMutatorSequence; }; } // namespace fuzzer diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerOptions.h b/contrib/compiler-rt/lib/fuzzer/FuzzerOptions.h index ce39c0876cd7..ab90df82a63d 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerOptions.h +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerOptions.h @@ -52,11 +52,8 @@ struct FuzzingOptions { bool PrintNewCovPcs = false; int PrintNewCovFuncs = 0; bool PrintFinalStats = false; - bool PrintMutationStats = false; bool PrintCorpusStats = false; bool PrintCoverage = false; - bool PrintUnstableStats = false; - int HandleUnstable = 0; bool DumpCoverage = false; bool DetectLeaks = true; int PurgeAllocatorIntervalSec = 1; diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp index 29ffc8e34fc0..80b33105bb22 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp @@ -13,6 +13,8 @@ //===----------------------------------------------------------------------===// #include "FuzzerTracePC.h" +#include "FuzzerBuiltins.h" +#include "FuzzerBuiltinsMsvc.h" #include "FuzzerCorpus.h" #include "FuzzerDefs.h" #include "FuzzerDictionary.h" @@ -32,8 +34,7 @@ ATTRIBUTE_INTERFACE uintptr_t __sancov_trace_pc_pcs[fuzzer::TracePC::kNumPCs]; // Used by -fsanitize-coverage=stack-depth to track stack depth -ATTRIBUTE_INTERFACE __attribute__((tls_model("initial-exec"))) -thread_local uintptr_t __sancov_lowest_stack; +ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC uintptr_t __sancov_lowest_stack; namespace fuzzer { @@ -57,49 +58,6 @@ size_t TracePC::GetTotalPCCoverage() { return Res; } -template<class CallBack> -void TracePC::IterateInline8bitCounters(CallBack CB) const { - if (NumInline8bitCounters && NumInline8bitCounters == NumPCsInPCTables) { - size_t CounterIdx = 0; - for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) { - uint8_t *Beg = ModuleCounters[i].Start; - size_t Size = ModuleCounters[i].Stop - Beg; - assert(Size == (size_t)(ModulePCTable[i].Stop - ModulePCTable[i].Start)); - for (size_t j = 0; j < Size; j++, CounterIdx++) - CB(i, j, CounterIdx); - } - } -} - -// Initializes unstable counters by copying Inline8bitCounters to unstable -// counters. -void TracePC::InitializeUnstableCounters() { - IterateInline8bitCounters([&](int i, int j, int UnstableIdx) { - UnstableCounters[UnstableIdx].Counter = ModuleCounters[i].Start[j]; - }); -} - -// Compares the current counters with counters from previous runs -// and records differences as unstable edges. -void TracePC::UpdateUnstableCounters(int UnstableMode) { - IterateInline8bitCounters([&](int i, int j, int UnstableIdx) { - if (ModuleCounters[i].Start[j] != UnstableCounters[UnstableIdx].Counter) { - UnstableCounters[UnstableIdx].IsUnstable = true; - if (UnstableMode == ZeroUnstable) - UnstableCounters[UnstableIdx].Counter = 0; - else if (UnstableMode == MinUnstable) - UnstableCounters[UnstableIdx].Counter = std::min( - ModuleCounters[i].Start[j], UnstableCounters[UnstableIdx].Counter); - } - }); -} - -// Moves the minimum hit counts to ModuleCounters. -void TracePC::ApplyUnstableCounters() { - IterateInline8bitCounters([&](int i, int j, int UnstableIdx) { - ModuleCounters[i].Start[j] = UnstableCounters[UnstableIdx].Counter; - }); -} void TracePC::HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop) { if (Start == Stop) return; @@ -185,11 +143,42 @@ void TracePC::HandleCallerCallee(uintptr_t Caller, uintptr_t Callee) { ValueProfileMap.AddValueModPrime(Idx); } +/// \return the address of the previous instruction. +/// Note: the logic is copied from `sanitizer_common/sanitizer_stacktrace.h` +inline ALWAYS_INLINE uintptr_t GetPreviousInstructionPc(uintptr_t PC) { +#if defined(__arm__) + // T32 (Thumb) branch instructions might be 16 or 32 bit long, + // so we return (pc-2) in that case in order to be safe. + // For A32 mode we return (pc-4) because all instructions are 32 bit long. + return (PC - 3) & (~1); +#elif defined(__powerpc__) || defined(__powerpc64__) || defined(__aarch64__) + // PCs are always 4 byte aligned. + return PC - 4; +#elif defined(__sparc__) || defined(__mips__) + return PC - 8; +#else + return PC - 1; +#endif +} + +/// \return the address of the next instruction. +/// Note: the logic is copied from `sanitizer_common/sanitizer_stacktrace.cc` +inline ALWAYS_INLINE uintptr_t GetNextInstructionPc(uintptr_t PC) { +#if defined(__mips__) + return PC + 8; +#elif defined(__powerpc__) || defined(__sparc__) || defined(__arm__) || \ + defined(__aarch64__) + return PC + 4; +#else + return PC + 1; +#endif +} + void TracePC::UpdateObservedPCs() { Vector<uintptr_t> CoveredFuncs; auto ObservePC = [&](uintptr_t PC) { if (ObservedPCs.insert(PC).second && DoPrintNewPCs) { - PrintPC("\tNEW_PC: %p %F %L", "\tNEW_PC: %p", PC + 1); + PrintPC("\tNEW_PC: %p %F %L", "\tNEW_PC: %p", GetNextInstructionPc(PC)); Printf("\n"); } }; @@ -203,10 +192,15 @@ void TracePC::UpdateObservedPCs() { if (NumPCsInPCTables) { if (NumInline8bitCounters == NumPCsInPCTables) { - IterateInline8bitCounters([&](int i, int j, int CounterIdx) { - if (ModuleCounters[i].Start[j]) - Observe(ModulePCTable[i].Start[j]); - }); + for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) { + uint8_t *Beg = ModuleCounters[i].Start; + size_t Size = ModuleCounters[i].Stop - Beg; + assert(Size == + (size_t)(ModulePCTable[i].Stop - ModulePCTable[i].Start)); + for (size_t j = 0; j < Size; j++) + if (Beg[j]) + Observe(ModulePCTable[i].Start[j]); + } } else if (NumGuards == NumPCsInPCTables) { size_t GuardIdx = 1; for (size_t i = 0; i < NumModules; i++) { @@ -224,22 +218,11 @@ void TracePC::UpdateObservedPCs() { for (size_t i = 0, N = Min(CoveredFuncs.size(), NumPrintNewFuncs); i < N; i++) { Printf("\tNEW_FUNC[%zd/%zd]: ", i + 1, CoveredFuncs.size()); - PrintPC("%p %F %L", "%p", CoveredFuncs[i] + 1); + PrintPC("%p %F %L", "%p", GetNextInstructionPc(CoveredFuncs[i])); Printf("\n"); } } -inline ALWAYS_INLINE uintptr_t GetPreviousInstructionPc(uintptr_t PC) { - // TODO: this implementation is x86 only. - // see sanitizer_common GetPreviousInstructionPc for full implementation. - return PC - 1; -} - -inline ALWAYS_INLINE uintptr_t GetNextInstructionPc(uintptr_t PC) { - // TODO: this implementation is x86 only. - // see sanitizer_common GetPreviousInstructionPc for full implementation. - return PC + 1; -} static std::string GetModuleName(uintptr_t PC) { char ModulePathRaw[4096] = ""; // What's PATH_MAX in portable C++? @@ -349,15 +332,6 @@ void TracePC::DumpCoverage() { } } -void TracePC::PrintUnstableStats() { - size_t count = 0; - for (size_t i = 0; i < NumInline8bitCounters; i++) - if (UnstableCounters[i].IsUnstable) - count++; - Printf("stat::stability_rate: %.2f\n", - 100 - static_cast<float>(count * 100) / NumInline8bitCounters); -} - // Value profile. // We keep track of various values that affect control flow. // These values are inserted into a bit-set-based hash map. @@ -401,20 +375,14 @@ ATTRIBUTE_TARGET_POPCNT ALWAYS_INLINE ATTRIBUTE_NO_SANITIZE_ALL void TracePC::HandleCmp(uintptr_t PC, T Arg1, T Arg2) { uint64_t ArgXor = Arg1 ^ Arg2; - uint64_t ArgDistance = __builtin_popcountll(ArgXor) + 1; // [1,65] - uintptr_t Idx = ((PC & 4095) + 1) * ArgDistance; if (sizeof(T) == 4) TORC4.Insert(ArgXor, Arg1, Arg2); else if (sizeof(T) == 8) TORC8.Insert(ArgXor, Arg1, Arg2); - // TODO: remove these flags and instead use all metrics at once. - if (UseValueProfileMask & 1) - ValueProfileMap.AddValue(Idx); - if (UseValueProfileMask & 2) - ValueProfileMap.AddValue( - PC * 64 + (Arg1 == Arg2 ? 0 : __builtin_clzll(Arg1 - Arg2) + 1)); - if (UseValueProfileMask & 4) // alternative way to use the hamming distance - ValueProfileMap.AddValue(PC * 64 + ArgDistance); + uint64_t HammingDistance = Popcountll(ArgXor); // [0,64] + uint64_t AbsoluteDistance = (Arg1 == Arg2 ? 0 : Clzll(Arg1 - Arg2) + 1); + ValueProfileMap.AddValue(PC * 128 + HammingDistance); + ValueProfileMap.AddValue(PC * 128 + 64 + AbsoluteDistance); } static size_t InternalStrnlen(const char *S, size_t MaxLen) { @@ -455,7 +423,7 @@ extern "C" { ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL void __sanitizer_cov_trace_pc_guard(uint32_t *Guard) { - uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); uint32_t Idx = *Guard; __sancov_trace_pc_pcs[Idx] = PC; __sancov_trace_pc_guard_8bit_counters[Idx]++; @@ -466,7 +434,7 @@ void __sanitizer_cov_trace_pc_guard(uint32_t *Guard) { ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL void __sanitizer_cov_trace_pc() { - uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); uintptr_t Idx = PC & (((uintptr_t)1 << fuzzer::TracePC::kTracePcBits) - 1); __sancov_trace_pc_pcs[Idx] = PC; __sancov_trace_pc_guard_8bit_counters[Idx]++; @@ -491,7 +459,7 @@ void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL void __sanitizer_cov_trace_pc_indir(uintptr_t Callee) { - uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); fuzzer::TPC.HandleCallerCallee(PC, Callee); } @@ -499,7 +467,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_cmp8(uint64_t Arg1, uint64_t Arg2) { - uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -510,7 +478,7 @@ ATTRIBUTE_TARGET_POPCNT // the behaviour of __sanitizer_cov_trace_cmp[1248] ones. This, however, // should be changed later to make full use of instrumentation. void __sanitizer_cov_trace_const_cmp8(uint64_t Arg1, uint64_t Arg2) { - uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -518,7 +486,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_cmp4(uint32_t Arg1, uint32_t Arg2) { - uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -526,7 +494,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_const_cmp4(uint32_t Arg1, uint32_t Arg2) { - uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -534,7 +502,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_cmp2(uint16_t Arg1, uint16_t Arg2) { - uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -542,7 +510,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_const_cmp2(uint16_t Arg1, uint16_t Arg2) { - uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -550,7 +518,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_cmp1(uint8_t Arg1, uint8_t Arg2) { - uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -558,7 +526,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_const_cmp1(uint8_t Arg1, uint8_t Arg2) { - uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } @@ -572,7 +540,7 @@ void __sanitizer_cov_trace_switch(uint64_t Val, uint64_t *Cases) { // Skip the most common and the most boring case. if (Vals[N - 1] < 256 && Val < 256) return; - uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); size_t i; uint64_t Token = 0; for (i = 0; i < N; i++) { @@ -593,7 +561,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_div4(uint32_t Val) { - uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Val, (uint32_t)0); } @@ -601,7 +569,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_div8(uint64_t Val) { - uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Val, (uint64_t)0); } @@ -609,7 +577,7 @@ ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_gep(uintptr_t Idx) { - uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); fuzzer::TPC.HandleCmp(PC, Idx, (uintptr_t)0); } diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerTracePC.h b/contrib/compiler-rt/lib/fuzzer/FuzzerTracePC.h index 097ba69bdc08..46d6c24887fd 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerTracePC.h +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerTracePC.h @@ -74,11 +74,6 @@ class TracePC { // How many bits of PC are used from __sanitizer_cov_trace_pc. static const size_t kTracePcBits = 18; - enum HandleUnstableOptions { - MinUnstable = 1, - ZeroUnstable = 2, - }; - void HandleInit(uint32_t *Start, uint32_t *Stop); void HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop); void HandlePCsInit(const uintptr_t *Start, const uintptr_t *Stop); @@ -109,7 +104,6 @@ class TracePC { void PrintCoverage(); void DumpCoverage(); - void PrintUnstableStats(); template<class CallBack> void IterateCoveredFunctions(CallBack CB); @@ -142,18 +136,7 @@ class TracePC { void SetFocusFunction(const std::string &FuncName); bool ObservedFocusFunction(); - void InitializeUnstableCounters(); - void UpdateUnstableCounters(int UnstableMode); - void ApplyUnstableCounters(); - private: - struct UnstableEdge { - uint8_t Counter; - bool IsUnstable; - }; - - UnstableEdge UnstableCounters[kNumPCs]; - bool UseCounters = false; uint32_t UseValueProfileMask = false; bool DoPrintNewPCs = false; @@ -185,9 +168,6 @@ private: Set<uintptr_t> ObservedPCs; std::unordered_map<uintptr_t, uintptr_t> ObservedFuncs; // PC => Counter. - template <class Callback> - void IterateInline8bitCounters(Callback CB) const; - std::pair<size_t, size_t> FocusFunction = {-1, -1}; // Module and PC IDs. ValueBitMap ValueProfileMap; diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerUtil.h b/contrib/compiler-rt/lib/fuzzer/FuzzerUtil.h index 8c5c57c3ab83..d2f1d5de4267 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerUtil.h +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerUtil.h @@ -12,8 +12,10 @@ #ifndef LLVM_FUZZER_UTIL_H #define LLVM_FUZZER_UTIL_H -#include "FuzzerDefs.h" +#include "FuzzerBuiltins.h" +#include "FuzzerBuiltinsMsvc.h" #include "FuzzerCommand.h" +#include "FuzzerDefs.h" namespace fuzzer { @@ -84,7 +86,7 @@ std::string SearchRegexCmd(const std::string &Regex); size_t SimpleFastHash(const uint8_t *Data, size_t Size); -inline uint32_t Log(uint32_t X) { return 32 - __builtin_clz(X) - 1; } +inline uint32_t Log(uint32_t X) { return 32 - Clz(X) - 1; } } // namespace fuzzer diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp index cd2bb7438e9d..cd48fefef352 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp @@ -49,9 +49,6 @@ void CrashTrampolineAsm() __asm__("CrashTrampolineAsm"); namespace { -// TODO(phosek): remove this and replace it with ZX_TIME_INFINITE -#define ZX_TIME_INFINITE_OLD INT64_MAX - // A magic value for the Zircon exception port, chosen to spell 'FUZZING' // when interpreted as a byte sequence on little-endian platforms. const uint64_t kFuzzingCrash = 0x474e495a5a5546; @@ -237,7 +234,7 @@ void CrashHandler(zx_handle_t *Event) { "_zx_object_signal"); zx_port_packet_t Packet; - ExitOnErr(_zx_port_wait(Port.Handle, ZX_TIME_INFINITE_OLD, &Packet), + ExitOnErr(_zx_port_wait(Port.Handle, ZX_TIME_INFINITE, &Packet), "_zx_port_wait"); // At this point, we want to get the state of the crashing thread, but @@ -315,8 +312,8 @@ void SetSignalHandler(const FuzzingOptions &Options) { ExitOnErr(_zx_event_create(0, &Event), "_zx_event_create"); std::thread T(CrashHandler, &Event); - zx_status_t Status = _zx_object_wait_one(Event, ZX_USER_SIGNAL_0, - ZX_TIME_INFINITE_OLD, nullptr); + zx_status_t Status = + _zx_object_wait_one(Event, ZX_USER_SIGNAL_0, ZX_TIME_INFINITE, nullptr); _zx_handle_close(Event); ExitOnErr(Status, "_zx_object_wait_one"); @@ -378,19 +375,28 @@ int ExecuteCommand(const Command &Cmd) { Argv[i] = Args[i].c_str(); Argv[Argc] = nullptr; - // Determine stdout + // Determine output. On Fuchsia, the fuzzer is typically run as a component + // that lacks a mutable working directory. Fortunately, when this is the case + // a mutable output directory must be specified using "-artifact_prefix=...", + // so write the log file(s) there. int FdOut = STDOUT_FILENO; - if (Cmd.hasOutputFile()) { - auto Filename = Cmd.getOutputFile(); - FdOut = open(Filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0); + std::string Path; + if (Cmd.hasFlag("artifact_prefix")) + Path = Cmd.getFlagValue("artifact_prefix") + "/" + Cmd.getOutputFile(); + else + Path = Cmd.getOutputFile(); + FdOut = open(Path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0); if (FdOut == -1) { - Printf("libFuzzer: failed to open %s: %s\n", Filename.c_str(), + Printf("libFuzzer: failed to open %s: %s\n", Path.c_str(), strerror(errno)); return ZX_ERR_IO; } } - auto CloseFdOut = at_scope_exit([&]() { close(FdOut); } ); + auto CloseFdOut = at_scope_exit([FdOut]() { + if (FdOut != STDOUT_FILENO) + close(FdOut); + }); // Determine stderr int FdErr = STDERR_FILENO; @@ -440,7 +446,7 @@ int ExecuteCommand(const Command &Cmd) { // Now join the process and return the exit status. if ((rc = _zx_object_wait_one(ProcessHandle, ZX_PROCESS_TERMINATED, - ZX_TIME_INFINITE_OLD, nullptr)) != ZX_OK) { + ZX_TIME_INFINITE, nullptr)) != ZX_OK) { Printf("libFuzzer: failed to join '%s': %s\n", Argv[0], _zx_status_get_string(rc)); return rc; diff --git a/contrib/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp b/contrib/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp index 8227e778ea0a..393b4768be7e 100644 --- a/contrib/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp +++ b/contrib/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp @@ -24,7 +24,7 @@ #include <windows.h> // This must be included after windows.h. -#include <Psapi.h> +#include <psapi.h> namespace fuzzer { @@ -179,7 +179,9 @@ const void *SearchMemory(const void *Data, size_t DataLen, const void *Patt, } std::string DisassembleCmd(const std::string &FileName) { - if (ExecuteCommand("dumpbin /summary > nul") == 0) + Vector<std::string> command_vector; + command_vector.push_back("dumpbin /summary > nul"); + if (ExecuteCommand(Command(command_vector)) == 0) return "dumpbin /disasm " + FileName; Printf("libFuzzer: couldn't find tool to disassemble (dumpbin)\n"); exit(1); diff --git a/contrib/compiler-rt/lib/hwasan/hwasan.cc b/contrib/compiler-rt/lib/hwasan/hwasan.cc index 7dab8249e3f7..e2bfea5e4226 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan.cc +++ b/contrib/compiler-rt/lib/hwasan/hwasan.cc @@ -13,19 +13,20 @@ //===----------------------------------------------------------------------===// #include "hwasan.h" -#include "hwasan_mapping.h" +#include "hwasan_checks.h" #include "hwasan_poisoning.h" #include "hwasan_report.h" #include "hwasan_thread.h" +#include "hwasan_thread_list.h" #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_flag_parser.h" +#include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_procmaps.h" +#include "sanitizer_common/sanitizer_stackdepot.h" #include "sanitizer_common/sanitizer_stacktrace.h" #include "sanitizer_common/sanitizer_symbolizer.h" -#include "sanitizer_common/sanitizer_stackdepot.h" #include "ubsan/ubsan_flags.h" #include "ubsan/ubsan_init.h" @@ -36,17 +37,17 @@ using namespace __sanitizer; namespace __hwasan { void EnterSymbolizer() { - HwasanThread *t = GetCurrentThread(); + Thread *t = GetCurrentThread(); CHECK(t); t->EnterSymbolizer(); } void ExitSymbolizer() { - HwasanThread *t = GetCurrentThread(); + Thread *t = GetCurrentThread(); CHECK(t); t->LeaveSymbolizer(); } bool IsInSymbolizer() { - HwasanThread *t = GetCurrentThread(); + Thread *t = GetCurrentThread(); return t && t->InSymbolizer(); } @@ -57,6 +58,7 @@ Flags *flags() { } int hwasan_inited = 0; +int hwasan_shadow_inited = 0; bool hwasan_init_is_running; int hwasan_report_count = 0; @@ -86,7 +88,18 @@ static void InitializeFlags() { cf.check_printf = false; cf.intercept_tls_get_addr = true; cf.exitcode = 99; + // Sigtrap is used in error reporting. cf.handle_sigtrap = kHandleSignalExclusive; + +#if SANITIZER_ANDROID + // Let platform handle other signals. It is better at reporting them then we + // are. + cf.handle_segv = kHandleSignalNo; + cf.handle_sigbus = kHandleSignalNo; + cf.handle_abort = kHandleSignalNo; + cf.handle_sigill = kHandleSignalNo; + cf.handle_sigfpe = kHandleSignalNo; +#endif OverrideCommonFlags(cf); } @@ -119,7 +132,8 @@ static void InitializeFlags() { #if HWASAN_CONTAINS_UBSAN ubsan_parser.ParseString(GetEnv("UBSAN_OPTIONS")); #endif - VPrintf(1, "HWASAN_OPTIONS: %s\n", hwasan_options ? hwasan_options : "<empty>"); + VPrintf(1, "HWASAN_OPTIONS: %s\n", + hwasan_options ? hwasan_options : "<empty>"); InitializeCommonFlags(); @@ -130,8 +144,13 @@ static void InitializeFlags() { void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, void *context, bool request_fast_unwind) { - HwasanThread *t = GetCurrentThread(); - if (!t || !StackTrace::WillUseFastUnwind(request_fast_unwind)) { + Thread *t = GetCurrentThread(); + if (!t) { + // the thread is still being created. + stack->size = 0; + return; + } + if (!StackTrace::WillUseFastUnwind(request_fast_unwind)) { // Block reports from our interceptors during _Unwind_Backtrace. SymbolizerScope sym_scope; return stack->Unwind(max_s, pc, bp, context, 0, 0, request_fast_unwind); @@ -140,11 +159,6 @@ void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, request_fast_unwind); } -void PrintWarning(uptr pc, uptr bp) { - GET_FATAL_STACK_TRACE_PC_BP(pc, bp); - ReportInvalidAccess(&stack, 0); -} - static void HWAsanCheckFailed(const char *file, int line, const char *cond, u64 v1, u64 v2) { Report("HWAddressSanitizer CHECK failed: %s:%d \"%s\" (0x%zx, 0x%zx)\n", file, @@ -153,6 +167,84 @@ static void HWAsanCheckFailed(const char *file, int line, const char *cond, Die(); } +static constexpr uptr kMemoryUsageBufferSize = 4096; + +static void HwasanFormatMemoryUsage(InternalScopedString &s) { + HwasanThreadList &thread_list = hwasanThreadList(); + auto thread_stats = thread_list.GetThreadStats(); + auto *sds = StackDepotGetStats(); + AllocatorStatCounters asc; + GetAllocatorStats(asc); + s.append( + "HWASAN pid: %d rss: %zd threads: %zd stacks: %zd" + " thr_aux: %zd stack_depot: %zd uniq_stacks: %zd" + " heap: %zd", + internal_getpid(), GetRSS(), thread_stats.n_live_threads, + thread_stats.total_stack_size, + thread_stats.n_live_threads * thread_list.MemoryUsedPerThread(), + sds->allocated, sds->n_uniq_ids, asc[AllocatorStatMapped]); +} + +#if SANITIZER_ANDROID +static char *memory_usage_buffer = nullptr; + +#define PR_SET_VMA 0x53564d41 +#define PR_SET_VMA_ANON_NAME 0 + +static void InitMemoryUsage() { + memory_usage_buffer = + (char *)MmapOrDie(kMemoryUsageBufferSize, "memory usage string"); + CHECK(memory_usage_buffer); + memory_usage_buffer[0] = '\0'; + CHECK(internal_prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, + (uptr)memory_usage_buffer, kMemoryUsageBufferSize, + (uptr)memory_usage_buffer) == 0); +} + +void UpdateMemoryUsage() { + if (!flags()->export_memory_stats) + return; + if (!memory_usage_buffer) + InitMemoryUsage(); + InternalScopedString s(kMemoryUsageBufferSize); + HwasanFormatMemoryUsage(s); + internal_strncpy(memory_usage_buffer, s.data(), kMemoryUsageBufferSize - 1); + memory_usage_buffer[kMemoryUsageBufferSize - 1] = '\0'; +} +#else +void UpdateMemoryUsage() {} +#endif + +struct FrameDescription { + uptr PC; + const char *Descr; +}; + +struct FrameDescriptionArray { + FrameDescription *beg, *end; +}; + +static InternalMmapVectorNoCtor<FrameDescriptionArray> AllFrames; + +void InitFrameDescriptors(uptr b, uptr e) { + FrameDescription *beg = reinterpret_cast<FrameDescription *>(b); + FrameDescription *end = reinterpret_cast<FrameDescription *>(e); + if (beg == end) + return; + AllFrames.push_back({beg, end}); + if (Verbosity()) + for (FrameDescription *frame_descr = beg; frame_descr < end; frame_descr++) + Printf("Frame: %p %s\n", frame_descr->PC, frame_descr->Descr); +} + +const char *GetStackFrameDescr(uptr pc) { + for (uptr i = 0, n = AllFrames.size(); i < n; i++) + for (auto p = AllFrames[i].beg; p < AllFrames[i].end; p++) + if (p->PC == pc) + return p->Descr; + return nullptr; +} + } // namespace __hwasan // Interface. @@ -161,6 +253,20 @@ using namespace __hwasan; uptr __hwasan_shadow_memory_dynamic_address; // Global interface symbol. +void __hwasan_shadow_init() { + if (hwasan_shadow_inited) return; + if (!InitShadow()) { + Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n"); + DumpProcessMap(); + Die(); + } + hwasan_shadow_inited = 1; +} + +void __hwasan_init_frames(uptr beg, uptr end) { + InitFrameDescriptors(beg, end); +} + void __hwasan_init() { CHECK(!hwasan_init_is_running); if (hwasan_inited) return; @@ -177,18 +283,20 @@ void __hwasan_init() { __sanitizer_set_report_path(common_flags()->log_path); + AndroidTestTlsSlot(); + DisableCoreDumperIfNecessary(); - if (!InitShadow()) { - Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n"); - if (HWASAN_FIXED_MAPPING) { - Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n"); - Printf("FATAL: Disabling ASLR is known to cause this error.\n"); - Printf("FATAL: If running under GDB, try " - "'set disable-randomization off'.\n"); - } - DumpProcessMap(); - Die(); - } + + __hwasan_shadow_init(); + + InitThreads(); + hwasanThreadList().CreateCurrentThread(); + + MadviseShadow(); + + SetPrintfAndReportCallback(AppendToErrorMessageBuffer); + // This may call libc -> needs initialized shadow. + AndroidLogInit(); InitializeInterceptors(); InstallDeadlySignalHandlers(HwasanOnDeadlySignal); @@ -198,14 +306,11 @@ void __hwasan_init() { InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir); - HwasanTSDInit(HwasanTSDDtor); + HwasanTSDInit(); + HwasanTSDThreadInit(); HwasanAllocatorInit(); - HwasanThread *main_thread = HwasanThread::Create(nullptr, nullptr); - SetCurrentThread(main_thread); - main_thread->ThreadStart(); - #if HWASAN_CONTAINS_UBSAN __ubsan::InitAsPlugin(); #endif @@ -216,9 +321,14 @@ void __hwasan_init() { hwasan_inited = 1; } -void __hwasan_print_shadow(const void *x, uptr size) { - // FIXME: - Printf("FIXME: __hwasan_print_shadow unimplemented\n"); +void __hwasan_print_shadow(const void *p, uptr sz) { + uptr ptr_raw = UntagAddr(reinterpret_cast<uptr>(p)); + uptr shadow_first = MemToShadow(ptr_raw); + uptr shadow_last = MemToShadow(ptr_raw + sz - 1); + Printf("HWASan shadow map for %zx .. %zx (pointer tag %x)\n", ptr_raw, + ptr_raw + sz, GetTagFromPointer((uptr)p)); + for (uptr s = shadow_first; s <= shadow_last; ++s) + Printf(" %zx: %x\n", ShadowToMem(s), *(tag_t *)s); } sptr __hwasan_test_shadow(const void *p, uptr sz) { @@ -227,12 +337,12 @@ sptr __hwasan_test_shadow(const void *p, uptr sz) { tag_t ptr_tag = GetTagFromPointer((uptr)p); if (ptr_tag == 0) return -1; - uptr ptr_raw = GetAddressFromPointer((uptr)p); - uptr shadow_first = MEM_TO_SHADOW(ptr_raw); - uptr shadow_last = MEM_TO_SHADOW(ptr_raw + sz - 1); + uptr ptr_raw = UntagAddr(reinterpret_cast<uptr>(p)); + uptr shadow_first = MemToShadow(ptr_raw); + uptr shadow_last = MemToShadow(ptr_raw + sz - 1); for (uptr s = shadow_first; s <= shadow_last; ++s) if (*(tag_t*)s != ptr_tag) - return SHADOW_TO_MEM(s) - ptr_raw; + return ShadowToMem(s) - ptr_raw; return -1; } @@ -255,63 +365,6 @@ void __sanitizer_unaligned_store64(uu64 *p, u64 x) { *p = x; } -template<unsigned X> -__attribute__((always_inline)) -static void SigTrap(uptr p) { -#if defined(__aarch64__) - (void)p; - // 0x900 is added to do not interfere with the kernel use of lower values of - // brk immediate. - // FIXME: Add a constraint to put the pointer into x0, the same as x86 branch. - asm("brk %0\n\t" ::"n"(0x900 + X)); -#elif defined(__x86_64__) - // INT3 + NOP DWORD ptr [EAX + X] to pass X to our signal handler, 5 bytes - // total. The pointer is passed via rdi. - // 0x40 is added as a safeguard, to help distinguish our trap from others and - // to avoid 0 offsets in the command (otherwise it'll be reduced to a - // different nop command, the three bytes one). - asm volatile( - "int3\n" - "nopl %c0(%%rax)\n" - :: "n"(0x40 + X), "D"(p)); -#else - // FIXME: not always sigill. - __builtin_trap(); -#endif - // __builtin_unreachable(); -} - -enum class ErrorAction { Abort, Recover }; -enum class AccessType { Load, Store }; - -template <ErrorAction EA, AccessType AT, unsigned LogSize> -__attribute__((always_inline, nodebug)) static void CheckAddress(uptr p) { - tag_t ptr_tag = GetTagFromPointer(p); - uptr ptr_raw = p & ~kAddressTagMask; - tag_t mem_tag = *(tag_t *)MEM_TO_SHADOW(ptr_raw); - if (UNLIKELY(ptr_tag != mem_tag)) { - SigTrap<0x20 * (EA == ErrorAction::Recover) + - 0x10 * (AT == AccessType::Store) + LogSize>(p); - if (EA == ErrorAction::Abort) __builtin_unreachable(); - } -} - -template <ErrorAction EA, AccessType AT> -__attribute__((always_inline, nodebug)) static void CheckAddressSized(uptr p, - uptr sz) { - CHECK_NE(0, sz); - tag_t ptr_tag = GetTagFromPointer(p); - uptr ptr_raw = p & ~kAddressTagMask; - tag_t *shadow_first = (tag_t *)MEM_TO_SHADOW(ptr_raw); - tag_t *shadow_last = (tag_t *)MEM_TO_SHADOW(ptr_raw + sz - 1); - for (tag_t *t = shadow_first; t <= shadow_last; ++t) - if (UNLIKELY(ptr_tag != *t)) { - SigTrap<0x20 * (EA == ErrorAction::Recover) + - 0x10 * (AT == AccessType::Store) + 0xf>(p); - if (EA == ErrorAction::Abort) __builtin_unreachable(); - } -} - void __hwasan_loadN(uptr p, uptr sz) { CheckAddressSized<ErrorAction::Abort, AccessType::Load>(p, sz); } @@ -392,10 +445,38 @@ void __hwasan_tag_memory(uptr p, u8 tag, uptr sz) { TagMemoryAligned(p, sz, tag); } +uptr __hwasan_tag_pointer(uptr p, u8 tag) { + return AddTagToPointer(p, tag); +} + +void __hwasan_handle_longjmp(const void *sp_dst) { + uptr dst = (uptr)sp_dst; + // HWASan does not support tagged SP. + CHECK(GetTagFromPointer(dst) == 0); + + uptr sp = (uptr)__builtin_frame_address(0); + static const uptr kMaxExpectedCleanupSize = 64 << 20; // 64M + if (dst < sp || dst - sp > kMaxExpectedCleanupSize) { + Report( + "WARNING: HWASan is ignoring requested __hwasan_handle_longjmp: " + "stack top: %p; target %p; distance: %p (%zd)\n" + "False positive error reports may follow\n", + (void *)sp, (void *)dst, dst - sp); + return; + } + TagMemory(sp, dst - sp, 0); +} + +void __hwasan_print_memory_usage() { + InternalScopedString s(kMemoryUsageBufferSize); + HwasanFormatMemoryUsage(s); + Printf("%s\n", s.data()); +} + static const u8 kFallbackTag = 0xBB; u8 __hwasan_generate_tag() { - HwasanThread *t = GetCurrentThread(); + Thread *t = GetCurrentThread(); if (!t) return kFallbackTag; return t->GenerateRandomTag(); } diff --git a/contrib/compiler-rt/lib/hwasan/hwasan.h b/contrib/compiler-rt/lib/hwasan/hwasan.h index 47d1d057a0dd..ce9e904c5c69 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan.h +++ b/contrib/compiler-rt/lib/hwasan/hwasan.h @@ -30,6 +30,10 @@ # define HWASAN_CONTAINS_UBSAN CAN_SANITIZE_UB #endif +#ifndef HWASAN_WITH_INTERCEPTORS +#define HWASAN_WITH_INTERCEPTORS 0 +#endif + typedef u8 tag_t; // TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address @@ -37,16 +41,21 @@ typedef u8 tag_t; const unsigned kAddressTagShift = 56; const uptr kAddressTagMask = 0xFFUL << kAddressTagShift; +// Minimal alignment of the shadow base address. Determines the space available +// for threads and stack histories. This is an ABI constant. +const unsigned kShadowBaseAlignment = 32; + static inline tag_t GetTagFromPointer(uptr p) { return p >> kAddressTagShift; } -static inline uptr GetAddressFromPointer(uptr p) { - return p & ~kAddressTagMask; +static inline uptr UntagAddr(uptr tagged_addr) { + return tagged_addr & ~kAddressTagMask; } -static inline void * GetAddressFromPointer(const void *p) { - return (void *)((uptr)p & ~kAddressTagMask); +static inline void *UntagPtr(const void *tagged_ptr) { + return reinterpret_cast<void *>( + UntagAddr(reinterpret_cast<uptr>(tagged_ptr))); } static inline uptr AddTagToPointer(uptr p, tag_t tag) { @@ -61,12 +70,13 @@ extern int hwasan_report_count; bool ProtectRange(uptr beg, uptr end); bool InitShadow(); +void InitThreads(); +void MadviseShadow(); char *GetProcSelfMaps(); void InitializeInterceptors(); void HwasanAllocatorInit(); void HwasanAllocatorThreadFinish(); -void HwasanDeallocate(StackTrace *stack, void *ptr); void *hwasan_malloc(uptr size, StackTrace *stack); void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack); @@ -77,11 +87,13 @@ void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack); void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack); int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size, StackTrace *stack); +void hwasan_free(void *ptr, StackTrace *stack); void InstallTrapHandler(); void InstallAtExitHandler(); const char *GetStackOriginDescr(u32 id, uptr *pc); +const char *GetStackFrameDescr(uptr pc); void EnterSymbolizer(); void ExitSymbolizer(); @@ -92,8 +104,6 @@ struct SymbolizerScope { ~SymbolizerScope() { ExitSymbolizer(); } }; -void PrintWarning(uptr pc, uptr bp); - void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, void *context, bool request_fast_unwind); @@ -135,13 +145,17 @@ class ScopedThreadLocalStateBackup { u64 va_arg_overflow_size_tls; }; -void HwasanTSDInit(void (*destructor)(void *tsd)); -void *HwasanTSDGet(); -void HwasanTSDSet(void *tsd); -void HwasanTSDDtor(void *tsd); +void HwasanTSDInit(); +void HwasanTSDThreadInit(); void HwasanOnDeadlySignal(int signo, void *info, void *context); +void UpdateMemoryUsage(); + +void AppendToErrorMessageBuffer(const char *buffer); + +void AndroidTestTlsSlot(); + } // namespace __hwasan #define HWASAN_MALLOC_HOOK(ptr, size) \ diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_allocator.cc b/contrib/compiler-rt/lib/hwasan/hwasan_allocator.cc index c2b9b0b69589..8487ed7e1e55 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_allocator.cc +++ b/contrib/compiler-rt/lib/hwasan/hwasan_allocator.cc @@ -12,10 +12,6 @@ // HWAddressSanitizer allocator. //===----------------------------------------------------------------------===// -#include "sanitizer_common/sanitizer_allocator.h" -#include "sanitizer_common/sanitizer_allocator_checks.h" -#include "sanitizer_common/sanitizer_allocator_interface.h" -#include "sanitizer_common/sanitizer_allocator_report.h" #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_errno.h" #include "sanitizer_common/sanitizer_stackdepot.h" @@ -23,30 +19,53 @@ #include "hwasan_allocator.h" #include "hwasan_mapping.h" #include "hwasan_thread.h" -#include "hwasan_poisoning.h" +#include "hwasan_report.h" + +#if HWASAN_WITH_INTERCEPTORS +DEFINE_REAL(void *, realloc, void *ptr, uptr size) +DEFINE_REAL(void, free, void *ptr) +#endif namespace __hwasan { -enum { - CHUNK_INVALID = 0, - CHUNK_FREE = 1, - CHUNK_ALLOCATED = 2 -}; +static Allocator allocator; +static AllocatorCache fallback_allocator_cache; +static SpinMutex fallback_mutex; +static atomic_uint8_t hwasan_allocator_tagging_enabled; + +static const tag_t kFallbackAllocTag = 0xBB; +static const tag_t kFallbackFreeTag = 0xBC; -struct Metadata { - u64 state : 2; - u64 requested_size : 62; - u32 alloc_context_id; - u32 free_context_id; +enum RightAlignMode { + kRightAlignNever, + kRightAlignSometimes, + kRightAlignAlways }; -bool HwasanChunkView::IsValid() const { - return metadata_ && metadata_->state != CHUNK_INVALID; -} +// These two variables are initialized from flags()->malloc_align_right +// in HwasanAllocatorInit and are never changed afterwards. +static RightAlignMode right_align_mode = kRightAlignNever; +static bool right_align_8 = false; + +// Initialized in HwasanAllocatorInit, an never changed. +static ALIGNED(16) u8 tail_magic[kShadowAlignment]; + bool HwasanChunkView::IsAllocated() const { - return metadata_ && metadata_->state == CHUNK_ALLOCATED; + return metadata_ && metadata_->alloc_context_id && metadata_->requested_size; +} + +// Aligns the 'addr' right to the granule boundary. +static uptr AlignRight(uptr addr, uptr requested_size) { + uptr tail_size = requested_size % kShadowAlignment; + if (!tail_size) return addr; + if (right_align_8) + return tail_size > 8 ? addr : addr + 8; + return addr + kShadowAlignment - tail_size; } + uptr HwasanChunkView::Beg() const { + if (metadata_ && metadata_->right_aligned) + return AlignRight(block_, metadata_->requested_size); return block_; } uptr HwasanChunkView::End() const { @@ -58,88 +77,79 @@ uptr HwasanChunkView::UsedSize() const { u32 HwasanChunkView::GetAllocStackId() const { return metadata_->alloc_context_id; } -u32 HwasanChunkView::GetFreeStackId() const { - return metadata_->free_context_id; -} - -struct HwasanMapUnmapCallback { - void OnMap(uptr p, uptr size) const {} - void OnUnmap(uptr p, uptr size) const { - // We are about to unmap a chunk of user memory. - // It can return as user-requested mmap() or another thread stack. - // Make it accessible with zero-tagged pointer. - TagMemory(p, size, 0); - } -}; -#if !defined(__aarch64__) && !defined(__x86_64__) -#error Unsupported platform -#endif - -static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G -static const uptr kRegionSizeLog = 20; -static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog; -typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap; - -struct AP32 { - static const uptr kSpaceBeg = 0; - static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; - static const uptr kMetadataSize = sizeof(Metadata); - typedef __sanitizer::CompactSizeClassMap SizeClassMap; - static const uptr kRegionSizeLog = __hwasan::kRegionSizeLog; - typedef __hwasan::ByteMap ByteMap; - typedef HwasanMapUnmapCallback MapUnmapCallback; - static const uptr kFlags = 0; -}; -typedef SizeClassAllocator32<AP32> PrimaryAllocator; -typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; -typedef LargeMmapAllocator<HwasanMapUnmapCallback> SecondaryAllocator; -typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, - SecondaryAllocator> Allocator; +uptr HwasanChunkView::ActualSize() const { + return allocator.GetActuallyAllocatedSize(reinterpret_cast<void *>(block_)); +} -static Allocator allocator; -static AllocatorCache fallback_allocator_cache; -static SpinMutex fallback_mutex; -static atomic_uint8_t hwasan_allocator_tagging_enabled; +bool HwasanChunkView::FromSmallHeap() const { + return allocator.FromPrimary(reinterpret_cast<void *>(block_)); +} -static const tag_t kFallbackAllocTag = 0xBB; -static const tag_t kFallbackFreeTag = 0xBC; +void GetAllocatorStats(AllocatorStatCounters s) { + allocator.GetStats(s); +} void HwasanAllocatorInit() { atomic_store_relaxed(&hwasan_allocator_tagging_enabled, !flags()->disable_allocator_tagging); SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); allocator.Init(common_flags()->allocator_release_to_os_interval_ms); + switch (flags()->malloc_align_right) { + case 0: break; + case 1: + right_align_mode = kRightAlignSometimes; + right_align_8 = false; + break; + case 2: + right_align_mode = kRightAlignAlways; + right_align_8 = false; + break; + case 8: + right_align_mode = kRightAlignSometimes; + right_align_8 = true; + break; + case 9: + right_align_mode = kRightAlignAlways; + right_align_8 = true; + break; + default: + Report("ERROR: unsupported value of malloc_align_right flag: %d\n", + flags()->malloc_align_right); + Die(); + } + for (uptr i = 0; i < kShadowAlignment; i++) + tail_magic[i] = GetCurrentThread()->GenerateRandomTag(); } -AllocatorCache *GetAllocatorCache(HwasanThreadLocalMallocStorage *ms) { - CHECK(ms); - CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache)); - return reinterpret_cast<AllocatorCache *>(ms->allocator_cache); +void AllocatorSwallowThreadLocalCache(AllocatorCache *cache) { + allocator.SwallowCache(cache); } -void HwasanThreadLocalMallocStorage::CommitBack() { - allocator.SwallowCache(GetAllocatorCache(this)); +static uptr TaggedSize(uptr size) { + if (!size) size = 1; + uptr new_size = RoundUpTo(size, kShadowAlignment); + CHECK_GE(new_size, size); + return new_size; } -static void *HwasanAllocate(StackTrace *stack, uptr size, uptr alignment, - bool zeroise) { - alignment = Max(alignment, kShadowAlignment); - size = RoundUpTo(size, kShadowAlignment); - - if (size > kMaxAllowedMallocSize) { +static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment, + bool zeroise) { + if (orig_size > kMaxAllowedMallocSize) { if (AllocatorMayReturnNull()) { Report("WARNING: HWAddressSanitizer failed to allocate 0x%zx bytes\n", - size); + orig_size); return nullptr; } - ReportAllocationSizeTooBig(size, kMaxAllowedMallocSize, stack); + ReportAllocationSizeTooBig(orig_size, kMaxAllowedMallocSize, stack); } - HwasanThread *t = GetCurrentThread(); + + alignment = Max(alignment, kShadowAlignment); + uptr size = TaggedSize(orig_size); + Thread *t = GetCurrentThread(); void *allocated; if (t) { - AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); - allocated = allocator.Allocate(cache, size, alignment); + allocated = allocator.Allocate(t->allocator_cache(), size, alignment); } else { SpinMutexLock l(&fallback_mutex); AllocatorCache *cache = &fallback_allocator_cache; @@ -153,11 +163,18 @@ static void *HwasanAllocate(StackTrace *stack, uptr size, uptr alignment, } Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated)); - meta->state = CHUNK_ALLOCATED; - meta->requested_size = size; + meta->requested_size = static_cast<u32>(orig_size); meta->alloc_context_id = StackDepotPut(*stack); - if (zeroise) + meta->right_aligned = false; + if (zeroise) { internal_memset(allocated, 0, size); + } else if (flags()->max_malloc_fill_size > 0) { + uptr fill_size = Min(size, (uptr)flags()->max_malloc_fill_size); + internal_memset(allocated, flags()->malloc_fill_byte, fill_size); + } + if (!right_align_mode) + internal_memcpy(reinterpret_cast<u8 *>(allocated) + orig_size, tail_magic, + size - orig_size); void *user_ptr = allocated; if (flags()->tag_in_malloc && @@ -165,74 +182,101 @@ static void *HwasanAllocate(StackTrace *stack, uptr size, uptr alignment, user_ptr = (void *)TagMemoryAligned( (uptr)user_ptr, size, t ? t->GenerateRandomTag() : kFallbackAllocTag); + if ((orig_size % kShadowAlignment) && (alignment <= kShadowAlignment) && + right_align_mode) { + uptr as_uptr = reinterpret_cast<uptr>(user_ptr); + if (right_align_mode == kRightAlignAlways || + GetTagFromPointer(as_uptr) & 1) { // use a tag bit as a random bit. + user_ptr = reinterpret_cast<void *>(AlignRight(as_uptr, orig_size)); + meta->right_aligned = 1; + } + } + HWASAN_MALLOC_HOOK(user_ptr, size); return user_ptr; } -void HwasanDeallocate(StackTrace *stack, void *user_ptr) { - CHECK(user_ptr); - HWASAN_FREE_HOOK(user_ptr); +static bool PointerAndMemoryTagsMatch(void *tagged_ptr) { + CHECK(tagged_ptr); + tag_t ptr_tag = GetTagFromPointer(reinterpret_cast<uptr>(tagged_ptr)); + tag_t mem_tag = *reinterpret_cast<tag_t *>( + MemToShadow(reinterpret_cast<uptr>(UntagPtr(tagged_ptr)))); + return ptr_tag == mem_tag; +} + +static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) { + CHECK(tagged_ptr); + HWASAN_FREE_HOOK(tagged_ptr); + + if (!PointerAndMemoryTagsMatch(tagged_ptr)) + ReportInvalidFree(stack, reinterpret_cast<uptr>(tagged_ptr)); + + void *untagged_ptr = UntagPtr(tagged_ptr); + void *aligned_ptr = reinterpret_cast<void *>( + RoundDownTo(reinterpret_cast<uptr>(untagged_ptr), kShadowAlignment)); + Metadata *meta = + reinterpret_cast<Metadata *>(allocator.GetMetaData(aligned_ptr)); + uptr orig_size = meta->requested_size; + u32 free_context_id = StackDepotPut(*stack); + u32 alloc_context_id = meta->alloc_context_id; + + // Check tail magic. + uptr tagged_size = TaggedSize(orig_size); + if (flags()->free_checks_tail_magic && !right_align_mode && orig_size) { + uptr tail_size = tagged_size - orig_size; + CHECK_LT(tail_size, kShadowAlignment); + void *tail_beg = reinterpret_cast<void *>( + reinterpret_cast<uptr>(aligned_ptr) + orig_size); + if (tail_size && internal_memcmp(tail_beg, tail_magic, tail_size)) + ReportTailOverwritten(stack, reinterpret_cast<uptr>(tagged_ptr), + orig_size, tail_size, tail_magic); + } - void *p = GetAddressFromPointer(user_ptr); - Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p)); - uptr size = meta->requested_size; - meta->state = CHUNK_FREE; meta->requested_size = 0; - meta->free_context_id = StackDepotPut(*stack); + meta->alloc_context_id = 0; // This memory will not be reused by anyone else, so we are free to keep it // poisoned. - HwasanThread *t = GetCurrentThread(); + Thread *t = GetCurrentThread(); + if (flags()->max_free_fill_size > 0) { + uptr fill_size = + Min(TaggedSize(orig_size), (uptr)flags()->max_free_fill_size); + internal_memset(aligned_ptr, flags()->free_fill_byte, fill_size); + } if (flags()->tag_in_free && atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) - TagMemoryAligned((uptr)p, size, + TagMemoryAligned(reinterpret_cast<uptr>(aligned_ptr), TaggedSize(orig_size), t ? t->GenerateRandomTag() : kFallbackFreeTag); if (t) { - AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); - allocator.Deallocate(cache, p); + allocator.Deallocate(t->allocator_cache(), aligned_ptr); + if (auto *ha = t->heap_allocations()) + ha->push({reinterpret_cast<uptr>(tagged_ptr), alloc_context_id, + free_context_id, static_cast<u32>(orig_size)}); } else { SpinMutexLock l(&fallback_mutex); AllocatorCache *cache = &fallback_allocator_cache; - allocator.Deallocate(cache, p); + allocator.Deallocate(cache, aligned_ptr); } } -void *HwasanReallocate(StackTrace *stack, void *user_old_p, uptr new_size, - uptr alignment) { - alignment = Max(alignment, kShadowAlignment); - new_size = RoundUpTo(new_size, kShadowAlignment); - - void *old_p = GetAddressFromPointer(user_old_p); - Metadata *meta = reinterpret_cast<Metadata*>(allocator.GetMetaData(old_p)); - uptr old_size = meta->requested_size; - uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(old_p); - if (new_size <= actually_allocated_size) { - // We are not reallocating here. - // FIXME: update stack trace for the allocation? - meta->requested_size = new_size; - if (!atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) - return user_old_p; - if (flags()->retag_in_realloc) { - HwasanThread *t = GetCurrentThread(); - return (void *)TagMemoryAligned( - (uptr)old_p, new_size, - t ? t->GenerateRandomTag() : kFallbackAllocTag); - } - if (new_size > old_size) { - tag_t tag = GetTagFromPointer((uptr)user_old_p); - TagMemoryAligned((uptr)old_p + old_size, new_size - old_size, tag); - } - return user_old_p; +static void *HwasanReallocate(StackTrace *stack, void *tagged_ptr_old, + uptr new_size, uptr alignment) { + if (!PointerAndMemoryTagsMatch(tagged_ptr_old)) + ReportInvalidFree(stack, reinterpret_cast<uptr>(tagged_ptr_old)); + + void *tagged_ptr_new = + HwasanAllocate(stack, new_size, alignment, false /*zeroise*/); + if (tagged_ptr_old && tagged_ptr_new) { + void *untagged_ptr_old = UntagPtr(tagged_ptr_old); + Metadata *meta = + reinterpret_cast<Metadata *>(allocator.GetMetaData(untagged_ptr_old)); + internal_memcpy(UntagPtr(tagged_ptr_new), untagged_ptr_old, + Min(new_size, static_cast<uptr>(meta->requested_size))); + HwasanDeallocate(stack, tagged_ptr_old); } - uptr memcpy_size = Min(new_size, old_size); - void *new_p = HwasanAllocate(stack, new_size, alignment, false /*zeroise*/); - if (new_p) { - internal_memcpy(new_p, old_p, memcpy_size); - HwasanDeallocate(stack, old_p); - } - return new_p; + return tagged_ptr_new; } -void *HwasanCalloc(StackTrace *stack, uptr nmemb, uptr size) { +static void *HwasanCalloc(StackTrace *stack, uptr nmemb, uptr size) { if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { if (AllocatorMayReturnNull()) return nullptr; @@ -250,12 +294,18 @@ HwasanChunkView FindHeapChunkByAddress(uptr address) { return HwasanChunkView(reinterpret_cast<uptr>(block), metadata); } -static uptr AllocationSize(const void *user_ptr) { - const void *p = GetAddressFromPointer(user_ptr); - if (!p) return 0; - const void *beg = allocator.GetBlockBegin(p); - if (beg != p) return 0; - Metadata *b = (Metadata *)allocator.GetMetaData(p); +static uptr AllocationSize(const void *tagged_ptr) { + const void *untagged_ptr = UntagPtr(tagged_ptr); + if (!untagged_ptr) return 0; + const void *beg = allocator.GetBlockBegin(untagged_ptr); + Metadata *b = (Metadata *)allocator.GetMetaData(untagged_ptr); + if (b->right_aligned) { + if (beg != reinterpret_cast<void *>(RoundDownTo( + reinterpret_cast<uptr>(untagged_ptr), kShadowAlignment))) + return 0; + } else { + if (beg != untagged_ptr) return 0; + } return b->requested_size; } @@ -270,6 +320,14 @@ void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack) { void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack) { if (!ptr) return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false)); + +#if HWASAN_WITH_INTERCEPTORS + // A tag of 0 means that this is a system allocator allocation, so we must use + // the system allocator to realloc it. + if (!flags()->disable_allocator_tagging && GetTagFromPointer((uptr)ptr) == 0) + return REAL(realloc)(ptr, size); +#endif + if (size == 0) { HwasanDeallocate(stack, ptr); return nullptr; @@ -331,6 +389,17 @@ int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size, return 0; } +void hwasan_free(void *ptr, StackTrace *stack) { +#if HWASAN_WITH_INTERCEPTORS + // A tag of 0 means that this is a system allocator allocation, so we must use + // the system allocator to free it. + if (!flags()->disable_allocator_tagging && GetTagFromPointer((uptr)ptr) == 0) + return REAL(free)(ptr); +#endif + + return HwasanDeallocate(stack, ptr); +} + } // namespace __hwasan using namespace __hwasan; @@ -340,6 +409,15 @@ void __hwasan_enable_allocator_tagging() { } void __hwasan_disable_allocator_tagging() { +#if HWASAN_WITH_INTERCEPTORS + // Allocator tagging must be enabled for the system allocator fallback to work + // correctly. This means that we can't disable it at runtime if it was enabled + // at startup since that might result in our deallocations going to the system + // allocator. If tagging was disabled at startup we avoid this problem by + // disabling the fallback altogether. + CHECK(flags()->disable_allocator_tagging); +#endif + atomic_store_relaxed(&hwasan_allocator_tagging_enabled, 0); } diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_allocator.h b/contrib/compiler-rt/lib/hwasan/hwasan_allocator.h index d025112e9773..6ab722fa6bef 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_allocator.h +++ b/contrib/compiler-rt/lib/hwasan/hwasan_allocator.h @@ -1,4 +1,4 @@ -//===-- hwasan_allocator.h ----------------------------------------*- C++ -*-===// +//===-- hwasan_allocator.h --------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -14,35 +14,73 @@ #ifndef HWASAN_ALLOCATOR_H #define HWASAN_ALLOCATOR_H +#include "interception/interception.h" +#include "sanitizer_common/sanitizer_allocator.h" +#include "sanitizer_common/sanitizer_allocator_checks.h" +#include "sanitizer_common/sanitizer_allocator_interface.h" +#include "sanitizer_common/sanitizer_allocator_report.h" #include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_ring_buffer.h" +#include "hwasan_poisoning.h" + +#if !defined(__aarch64__) && !defined(__x86_64__) +#error Unsupported platform +#endif + +#if HWASAN_WITH_INTERCEPTORS +DECLARE_REAL(void *, realloc, void *ptr, uptr size) +DECLARE_REAL(void, free, void *ptr) +#endif namespace __hwasan { -struct HwasanThreadLocalMallocStorage { - uptr quarantine_cache[16]; - // Allocator cache contains atomic_uint64_t which must be 8-byte aligned. - ALIGNED(8) uptr allocator_cache[96 * (512 * 8 + 16)]; // Opaque. - void CommitBack(); +struct Metadata { + u32 requested_size : 31; // sizes are < 2G. + u32 right_aligned : 1; + u32 alloc_context_id; +}; - private: - // These objects are allocated via mmap() and are zero-initialized. - HwasanThreadLocalMallocStorage() {} +struct HwasanMapUnmapCallback { + void OnMap(uptr p, uptr size) const { UpdateMemoryUsage(); } + void OnUnmap(uptr p, uptr size) const { + // We are about to unmap a chunk of user memory. + // It can return as user-requested mmap() or another thread stack. + // Make it accessible with zero-tagged pointer. + TagMemory(p, size, 0); + } }; -struct Metadata; +static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G + +struct AP64 { + static const uptr kSpaceBeg = ~0ULL; + static const uptr kSpaceSize = 0x2000000000ULL; + static const uptr kMetadataSize = sizeof(Metadata); + typedef __sanitizer::VeryDenseSizeClassMap SizeClassMap; + using AddressSpaceView = LocalAddressSpaceView; + typedef HwasanMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; +}; +typedef SizeClassAllocator64<AP64> PrimaryAllocator; +typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; +typedef LargeMmapAllocator<HwasanMapUnmapCallback> SecondaryAllocator; +typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, + SecondaryAllocator> Allocator; + +void AllocatorSwallowThreadLocalCache(AllocatorCache *cache); class HwasanChunkView { public: HwasanChunkView() : block_(0), metadata_(nullptr) {} HwasanChunkView(uptr block, Metadata *metadata) : block_(block), metadata_(metadata) {} - bool IsValid() const; // Checks if it points to a valid allocated chunk bool IsAllocated() const; // Checks if the memory is currently allocated uptr Beg() const; // First byte of user memory uptr End() const; // Last byte of user memory uptr UsedSize() const; // Size requested by the user + uptr ActualSize() const; // Size allocated by the allocator. u32 GetAllocStackId() const; - u32 GetFreeStackId() const; + bool FromSmallHeap() const; private: uptr block_; Metadata *const metadata_; @@ -50,6 +88,21 @@ class HwasanChunkView { HwasanChunkView FindHeapChunkByAddress(uptr address); +// Information about one (de)allocation that happened in the past. +// These are recorded in a thread-local ring buffer. +// TODO: this is currently 24 bytes (20 bytes + alignment). +// Compress it to 16 bytes or extend it to be more useful. +struct HeapAllocationRecord { + uptr tagged_addr; + u32 alloc_context_id; + u32 free_context_id; + u32 requested_size; +}; + +typedef RingBuffer<HeapAllocationRecord> HeapAllocationsRingBuffer; + +void GetAllocatorStats(AllocatorStatCounters s); + } // namespace __hwasan #endif // HWASAN_ALLOCATOR_H diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_checks.h b/contrib/compiler-rt/lib/hwasan/hwasan_checks.h new file mode 100644 index 000000000000..688b5e2bed82 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_checks.h @@ -0,0 +1,80 @@ +//===-- hwasan_checks.h -----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +//===----------------------------------------------------------------------===// + +#ifndef HWASAN_CHECKS_H +#define HWASAN_CHECKS_H + +#include "hwasan_mapping.h" + +namespace __hwasan { +template <unsigned X> +__attribute__((always_inline)) static void SigTrap(uptr p) { +#if defined(__aarch64__) + (void)p; + // 0x900 is added to do not interfere with the kernel use of lower values of + // brk immediate. + // FIXME: Add a constraint to put the pointer into x0, the same as x86 branch. + asm("brk %0\n\t" ::"n"(0x900 + X)); +#elif defined(__x86_64__) + // INT3 + NOP DWORD ptr [EAX + X] to pass X to our signal handler, 5 bytes + // total. The pointer is passed via rdi. + // 0x40 is added as a safeguard, to help distinguish our trap from others and + // to avoid 0 offsets in the command (otherwise it'll be reduced to a + // different nop command, the three bytes one). + asm volatile( + "int3\n" + "nopl %c0(%%rax)\n" ::"n"(0x40 + X), + "D"(p)); +#else + // FIXME: not always sigill. + __builtin_trap(); +#endif + // __builtin_unreachable(); +} + +enum class ErrorAction { Abort, Recover }; +enum class AccessType { Load, Store }; + +template <ErrorAction EA, AccessType AT, unsigned LogSize> +__attribute__((always_inline, nodebug)) static void CheckAddress(uptr p) { + tag_t ptr_tag = GetTagFromPointer(p); + uptr ptr_raw = p & ~kAddressTagMask; + tag_t mem_tag = *(tag_t *)MemToShadow(ptr_raw); + if (UNLIKELY(ptr_tag != mem_tag)) { + SigTrap<0x20 * (EA == ErrorAction::Recover) + + 0x10 * (AT == AccessType::Store) + LogSize>(p); + if (EA == ErrorAction::Abort) + __builtin_unreachable(); + } +} + +template <ErrorAction EA, AccessType AT> +__attribute__((always_inline, nodebug)) static void CheckAddressSized(uptr p, + uptr sz) { + if (sz == 0) + return; + tag_t ptr_tag = GetTagFromPointer(p); + uptr ptr_raw = p & ~kAddressTagMask; + tag_t *shadow_first = (tag_t *)MemToShadow(ptr_raw); + tag_t *shadow_last = (tag_t *)MemToShadow(ptr_raw + sz - 1); + for (tag_t *t = shadow_first; t <= shadow_last; ++t) + if (UNLIKELY(ptr_tag != *t)) { + SigTrap<0x20 * (EA == ErrorAction::Recover) + + 0x10 * (AT == AccessType::Store) + 0xf>(p); + if (EA == ErrorAction::Abort) + __builtin_unreachable(); + } +} +} // end namespace __hwasan + +#endif // HWASAN_CHECKS_H diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cc b/contrib/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cc index 17338003aa65..87670f508283 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cc +++ b/contrib/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cc @@ -13,6 +13,7 @@ /// //===----------------------------------------------------------------------===// +#include "hwasan.h" #include "hwasan_dynamic_shadow.h" #include "hwasan_mapping.h" #include "sanitizer_common/sanitizer_common.h" @@ -35,12 +36,16 @@ static void UnmapFromTo(uptr from, uptr to) { } } -// Returns an address aligned to 8 pages, such that one page on the left and -// shadow_size_bytes bytes on the right of it are mapped r/o. +// Returns an address aligned to kShadowBaseAlignment, such that +// 2**kShadowBaseAlingment on the left and shadow_size_bytes bytes on the right +// of it are mapped no access. static uptr MapDynamicShadow(uptr shadow_size_bytes) { const uptr granularity = GetMmapGranularity(); - const uptr alignment = granularity * SHADOW_GRANULARITY; - const uptr left_padding = granularity; + const uptr min_alignment = granularity << kShadowScale; + const uptr alignment = 1ULL << kShadowBaseAlignment; + CHECK_GE(alignment, min_alignment); + + const uptr left_padding = 1ULL << kShadowBaseAlignment; const uptr shadow_size = RoundUpTo(shadow_size_bytes, granularity); const uptr map_size = shadow_size + left_padding + alignment; @@ -58,8 +63,7 @@ static uptr MapDynamicShadow(uptr shadow_size_bytes) { } // namespace __hwasan -#if HWASAN_PREMAP_SHADOW - +#if SANITIZER_ANDROID extern "C" { INTERFACE_ATTRIBUTE void __hwasan_shadow(); @@ -117,16 +121,22 @@ void __hwasan_shadow(); } // extern "C" -#endif // HWASAN_PREMAP_SHADOW - namespace __hwasan { uptr FindDynamicShadowStart(uptr shadow_size_bytes) { -#if HWASAN_PREMAP_SHADOW if (IsPremapShadowAvailable()) return FindPremappedShadowStart(shadow_size_bytes); -#endif return MapDynamicShadow(shadow_size_bytes); } } // namespace __hwasan +#else +namespace __hwasan { + +uptr FindDynamicShadowStart(uptr shadow_size_bytes) { + return MapDynamicShadow(shadow_size_bytes); +} + +} // namespace __hwasan +# +#endif // SANITIZER_ANDROID diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_flags.h b/contrib/compiler-rt/lib/hwasan/hwasan_flags.h index 16d60c4d8ba5..492d5bb98c27 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_flags.h +++ b/contrib/compiler-rt/lib/hwasan/hwasan_flags.h @@ -1,4 +1,4 @@ -//===-- hwasan_flags.h --------------------------------------------*- C++ -*-===// +//===-- hwasan_flags.h ------------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_flags.inc b/contrib/compiler-rt/lib/hwasan/hwasan_flags.inc index c45781168d6c..b450ab9503f9 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_flags.inc +++ b/contrib/compiler-rt/lib/hwasan/hwasan_flags.inc @@ -17,9 +17,10 @@ // HWASAN_FLAG(Type, Name, DefaultValue, Description) // See COMMON_FLAG in sanitizer_flags.inc for more details. +HWASAN_FLAG(bool, verbose_threads, false, + "inform on thread creation/destruction") HWASAN_FLAG(bool, tag_in_malloc, true, "") HWASAN_FLAG(bool, tag_in_free, true, "") -HWASAN_FLAG(bool, retag_in_realloc, true, "") HWASAN_FLAG(bool, print_stats, false, "") HWASAN_FLAG(bool, halt_on_error, true, "") HWASAN_FLAG(bool, atexit, false, "") @@ -31,3 +32,57 @@ HWASAN_FLAG(bool, disable_allocator_tagging, false, "") // If false, use simple increment of a thread local counter to generate new // tags. HWASAN_FLAG(bool, random_tags, true, "") + +HWASAN_FLAG( + int, max_malloc_fill_size, 0x1000, // By default, fill only the first 4K. + "HWASan allocator flag. max_malloc_fill_size is the maximal amount of " + "bytes that will be filled with malloc_fill_byte on malloc.") + +// Rules for malloc alignment on aarch64: +// * If the size is 16-aligned, then malloc should return 16-aligned memory. +// * Otherwise, malloc should return 8-alignment memory. +// So, +// * If the size is 16-aligned, we don't need to do anything. +// * Otherwise we don't have to obey 16-alignment, just the 8-alignment. +// * We may want to break the 8-alignment rule to catch more buffer overflows +// but this will break valid code in some rare cases, like this: +// struct Foo { +// // accessed via atomic instructions that require 8-alignment. +// std::atomic<int64_t> atomic_stuff; +// ... +// char vla[1]; // the actual size of vla could be anything. +// } +// Which means that the safe values for malloc_align_right are 0, 8, 9, +// and the values 1 and 2 may require changes in otherwise valid code. + +HWASAN_FLAG( + int, malloc_align_right, 0, // off by default + "HWASan allocator flag. " + "0 (default): allocations are always aligned left to 16-byte boundary; " + "1: allocations are sometimes aligned right to 1-byte boundary (risky); " + "2: allocations are always aligned right to 1-byte boundary (risky); " + "8: allocations are sometimes aligned right to 8-byte boundary; " + "9: allocations are always aligned right to 8-byte boundary." + ) +HWASAN_FLAG(bool, free_checks_tail_magic, 1, + "If set, free() will check the magic values " + "to the right of the allocated object " + "if the allocation size is not a divident of the granule size") +HWASAN_FLAG( + int, max_free_fill_size, 0, + "HWASan allocator flag. max_free_fill_size is the maximal amount of " + "bytes that will be filled with free_fill_byte during free.") +HWASAN_FLAG(int, malloc_fill_byte, 0xbe, + "Value used to fill the newly allocated memory.") +HWASAN_FLAG(int, free_fill_byte, 0x55, + "Value used to fill deallocated memory.") +HWASAN_FLAG(int, heap_history_size, 1023, + "The number of heap (de)allocations remembered per thread. " + "Affects the quality of heap-related reports, but not the ability " + "to find bugs.") +HWASAN_FLAG(bool, export_memory_stats, true, + "Export up-to-date memory stats through /proc") +HWASAN_FLAG(int, stack_history_size, 1024, + "The number of stack frames remembered per thread. " + "Affects the quality of stack-related reports, but not the ability " + "to find bugs.") diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_interceptors.cc b/contrib/compiler-rt/lib/hwasan/hwasan_interceptors.cc index 66aab95db56f..fb0dcb8905c0 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_interceptors.cc +++ b/contrib/compiler-rt/lib/hwasan/hwasan_interceptors.cc @@ -1,4 +1,4 @@ -//===-- hwasan_interceptors.cc ----------------------------------------------===// +//===-- hwasan_interceptors.cc --------------------------------------------===// // // The LLVM Compiler Infrastructure // @@ -17,6 +17,7 @@ #include "interception/interception.h" #include "hwasan.h" +#include "hwasan_allocator.h" #include "hwasan_mapping.h" #include "hwasan_thread.h" #include "hwasan_poisoning.h" @@ -44,24 +45,19 @@ using __sanitizer::atomic_load; using __sanitizer::atomic_store; using __sanitizer::atomic_uintptr_t; -DECLARE_REAL(SIZE_T, strlen, const char *s) -DECLARE_REAL(SIZE_T, strnlen, const char *s, SIZE_T maxlen) -DECLARE_REAL(void *, memcpy, void *dest, const void *src, uptr n) -DECLARE_REAL(void *, memset, void *dest, int c, uptr n) - bool IsInInterceptorScope() { - HwasanThread *t = GetCurrentThread(); + Thread *t = GetCurrentThread(); return t && t->InInterceptorScope(); } struct InterceptorScope { InterceptorScope() { - HwasanThread *t = GetCurrentThread(); + Thread *t = GetCurrentThread(); if (t) t->EnterInterceptorScope(); } ~InterceptorScope() { - HwasanThread *t = GetCurrentThread(); + Thread *t = GetCurrentThread(); if (t) t->LeaveInterceptorScope(); } @@ -92,66 +88,24 @@ static void *AllocateFromLocalPool(uptr size_in_bytes) { } while (0) - -#define HWASAN_READ_RANGE(ctx, offset, size) \ - CHECK_UNPOISONED(offset, size) -#define HWASAN_WRITE_RANGE(ctx, offset, size) \ - CHECK_UNPOISONED(offset, size) - - - -// Check that [x, x+n) range is unpoisoned. -#define CHECK_UNPOISONED_0(x, n) \ - do { \ - sptr __offset = __hwasan_test_shadow(x, n); \ - if (__hwasan::IsInSymbolizer()) break; \ - if (__offset >= 0) { \ - GET_CALLER_PC_BP_SP; \ - (void)sp; \ - ReportInvalidAccessInsideAddressRange(__func__, x, n, __offset); \ - __hwasan::PrintWarning(pc, bp); \ - if (__hwasan::flags()->halt_on_error) { \ - Printf("Exiting\n"); \ - Die(); \ - } \ - } \ - } while (0) - -// Check that [x, x+n) range is unpoisoned unless we are in a nested -// interceptor. -#define CHECK_UNPOISONED(x, n) \ - do { \ - if (!IsInInterceptorScope()) CHECK_UNPOISONED_0(x, n); \ - } while (0) - -#define CHECK_UNPOISONED_STRING_OF_LEN(x, len, n) \ - CHECK_UNPOISONED((x), \ - common_flags()->strict_string_checks ? (len) + 1 : (n) ) - - -INTERCEPTOR(int, posix_memalign, void **memptr, SIZE_T alignment, SIZE_T size) { +int __sanitizer_posix_memalign(void **memptr, uptr alignment, uptr size) { GET_MALLOC_STACK_TRACE; CHECK_NE(memptr, 0); int res = hwasan_posix_memalign(memptr, alignment, size, &stack); return res; } -#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD -INTERCEPTOR(void *, memalign, SIZE_T alignment, SIZE_T size) { +void * __sanitizer_memalign(uptr alignment, uptr size) { GET_MALLOC_STACK_TRACE; return hwasan_memalign(alignment, size, &stack); } -#define HWASAN_MAYBE_INTERCEPT_MEMALIGN INTERCEPT_FUNCTION(memalign) -#else -#define HWASAN_MAYBE_INTERCEPT_MEMALIGN -#endif -INTERCEPTOR(void *, aligned_alloc, SIZE_T alignment, SIZE_T size) { +void * __sanitizer_aligned_alloc(uptr alignment, uptr size) { GET_MALLOC_STACK_TRACE; return hwasan_aligned_alloc(alignment, size, &stack); } -INTERCEPTOR(void *, __libc_memalign, SIZE_T alignment, SIZE_T size) { +void * __sanitizer___libc_memalign(uptr alignment, uptr size) { GET_MALLOC_STACK_TRACE; void *ptr = hwasan_memalign(alignment, size, &stack); if (ptr) @@ -159,80 +113,47 @@ INTERCEPTOR(void *, __libc_memalign, SIZE_T alignment, SIZE_T size) { return ptr; } -INTERCEPTOR(void *, valloc, SIZE_T size) { +void * __sanitizer_valloc(uptr size) { GET_MALLOC_STACK_TRACE; return hwasan_valloc(size, &stack); } -#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD -INTERCEPTOR(void *, pvalloc, SIZE_T size) { +void * __sanitizer_pvalloc(uptr size) { GET_MALLOC_STACK_TRACE; return hwasan_pvalloc(size, &stack); } -#define HWASAN_MAYBE_INTERCEPT_PVALLOC INTERCEPT_FUNCTION(pvalloc) -#else -#define HWASAN_MAYBE_INTERCEPT_PVALLOC -#endif -INTERCEPTOR(void, free, void *ptr) { +void __sanitizer_free(void *ptr) { GET_MALLOC_STACK_TRACE; if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return; - HwasanDeallocate(&stack, ptr); + hwasan_free(ptr, &stack); } -#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD -INTERCEPTOR(void, cfree, void *ptr) { +void __sanitizer_cfree(void *ptr) { GET_MALLOC_STACK_TRACE; if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return; - HwasanDeallocate(&stack, ptr); + hwasan_free(ptr, &stack); } -#define HWASAN_MAYBE_INTERCEPT_CFREE INTERCEPT_FUNCTION(cfree) -#else -#define HWASAN_MAYBE_INTERCEPT_CFREE -#endif -INTERCEPTOR(uptr, malloc_usable_size, void *ptr) { +uptr __sanitizer_malloc_usable_size(const void *ptr) { return __sanitizer_get_allocated_size(ptr); } -#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD -// This function actually returns a struct by value, but we can't unpoison a -// temporary! The following is equivalent on all supported platforms but -// aarch64 (which uses a different register for sret value). We have a test -// to confirm that. -INTERCEPTOR(void, mallinfo, __sanitizer_mallinfo *sret) { -#ifdef __aarch64__ - uptr r8; - asm volatile("mov %0,x8" : "=r" (r8)); - sret = reinterpret_cast<__sanitizer_mallinfo*>(r8); -#endif - REAL(memset)(sret, 0, sizeof(*sret)); +struct __sanitizer_struct_mallinfo __sanitizer_mallinfo() { + __sanitizer_struct_mallinfo sret; + internal_memset(&sret, 0, sizeof(sret)); + return sret; } -#define HWASAN_MAYBE_INTERCEPT_MALLINFO INTERCEPT_FUNCTION(mallinfo) -#else -#define HWASAN_MAYBE_INTERCEPT_MALLINFO -#endif -#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD -INTERCEPTOR(int, mallopt, int cmd, int value) { - return -1; +int __sanitizer_mallopt(int cmd, int value) { + return 0; } -#define HWASAN_MAYBE_INTERCEPT_MALLOPT INTERCEPT_FUNCTION(mallopt) -#else -#define HWASAN_MAYBE_INTERCEPT_MALLOPT -#endif -#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD -INTERCEPTOR(void, malloc_stats, void) { +void __sanitizer_malloc_stats(void) { // FIXME: implement, but don't call REAL(malloc_stats)! } -#define HWASAN_MAYBE_INTERCEPT_MALLOC_STATS INTERCEPT_FUNCTION(malloc_stats) -#else -#define HWASAN_MAYBE_INTERCEPT_MALLOC_STATS -#endif - -INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) { +void * __sanitizer_calloc(uptr nmemb, uptr size) { GET_MALLOC_STACK_TRACE; if (UNLIKELY(!hwasan_inited)) // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. @@ -240,7 +161,7 @@ INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) { return hwasan_calloc(nmemb, size, &stack); } -INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) { +void * __sanitizer_realloc(void *ptr, uptr size) { GET_MALLOC_STACK_TRACE; if (UNLIKELY(IsInDlsymAllocPool(ptr))) { uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; @@ -258,7 +179,7 @@ INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) { return hwasan_realloc(ptr, size, &stack); } -INTERCEPTOR(void *, malloc, SIZE_T size) { +void * __sanitizer_malloc(uptr size) { GET_MALLOC_STACK_TRACE; if (UNLIKELY(!hwasan_init_is_running)) ENSURE_HWASAN_INITED(); @@ -268,48 +189,44 @@ INTERCEPTOR(void *, malloc, SIZE_T size) { return hwasan_malloc(size, &stack); } -template <class Mmap> -static void *mmap_interceptor(Mmap real_mmap, void *addr, SIZE_T sz, int prot, - int flags, int fd, OFF64_T off) { - if (addr && !MEM_IS_APP(addr)) { - if (flags & map_fixed) { - errno = errno_EINVAL; - return (void *)-1; - } else { - addr = nullptr; - } - } - return real_mmap(addr, sz, prot, flags, fd, off); -} - -extern "C" int pthread_attr_init(void *attr); -extern "C" int pthread_attr_destroy(void *attr); - -static void *HwasanThreadStartFunc(void *arg) { - HwasanThread *t = (HwasanThread *)arg; - SetCurrentThread(t); - return t->ThreadStart(); -} - -INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*), - void * param) { - ENSURE_HWASAN_INITED(); // for GetTlsSize() - __sanitizer_pthread_attr_t myattr; - if (!attr) { - pthread_attr_init(&myattr); - attr = &myattr; - } +#if HWASAN_WITH_INTERCEPTORS +#define INTERCEPTOR_ALIAS(RET, FN, ARGS...) \ + extern "C" SANITIZER_INTERFACE_ATTRIBUTE RET WRAP(FN)(ARGS) \ + ALIAS("__sanitizer_" #FN); \ + extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE RET FN( \ + ARGS) ALIAS("__sanitizer_" #FN) + +INTERCEPTOR_ALIAS(int, posix_memalign, void **memptr, SIZE_T alignment, + SIZE_T size); +INTERCEPTOR_ALIAS(void *, aligned_alloc, SIZE_T alignment, SIZE_T size); +INTERCEPTOR_ALIAS(void *, __libc_memalign, SIZE_T alignment, SIZE_T size); +INTERCEPTOR_ALIAS(void *, valloc, SIZE_T size); +INTERCEPTOR_ALIAS(void, free, void *ptr); +INTERCEPTOR_ALIAS(uptr, malloc_usable_size, const void *ptr); +INTERCEPTOR_ALIAS(void *, calloc, SIZE_T nmemb, SIZE_T size); +INTERCEPTOR_ALIAS(void *, realloc, void *ptr, SIZE_T size); +INTERCEPTOR_ALIAS(void *, malloc, SIZE_T size); - AdjustStackSize(attr); - - HwasanThread *t = HwasanThread::Create(callback, param); +#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD +INTERCEPTOR_ALIAS(void *, memalign, SIZE_T alignment, SIZE_T size); +INTERCEPTOR_ALIAS(void *, pvalloc, SIZE_T size); +INTERCEPTOR_ALIAS(void, cfree, void *ptr); +INTERCEPTOR_ALIAS(__sanitizer_struct_mallinfo, mallinfo); +INTERCEPTOR_ALIAS(int, mallopt, int cmd, int value); +INTERCEPTOR_ALIAS(void, malloc_stats, void); +#endif +#endif // HWASAN_WITH_INTERCEPTORS - int res = REAL(pthread_create)(th, attr, HwasanThreadStartFunc, t); - if (attr == &myattr) - pthread_attr_destroy(&myattr); +#if HWASAN_WITH_INTERCEPTORS && !defined(__aarch64__) +INTERCEPTOR(int, pthread_create, void *th, void *attr, + void *(*callback)(void *), void *param) { + ScopedTaggingDisabler disabler; + int res = REAL(pthread_create)(UntagPtr(th), UntagPtr(attr), + callback, param); return res; } +#endif static void BeforeFork() { StackDepotLockAll(); @@ -341,137 +258,21 @@ int OnExit() { } // namespace __hwasan -// A version of CHECK_UNPOISONED using a saved scope value. Used in common -// interceptors. -#define CHECK_UNPOISONED_CTX(ctx, x, n) \ - do { \ - if (!((HwasanInterceptorContext *)ctx)->in_interceptor_scope) \ - CHECK_UNPOISONED_0(x, n); \ - } while (0) - -#define HWASAN_INTERCEPT_FUNC(name) \ - do { \ - if ((!INTERCEPT_FUNCTION(name) || !REAL(name))) \ - VReport(1, "HWAddressSanitizer: failed to intercept '" #name "'\n"); \ - } while (0) - -#define HWASAN_INTERCEPT_FUNC_VER(name, ver) \ - do { \ - if ((!INTERCEPT_FUNCTION_VER(name, ver) || !REAL(name))) \ - VReport( \ - 1, "HWAddressSanitizer: failed to intercept '" #name "@@" #ver "'\n"); \ - } while (0) - -#define COMMON_INTERCEPT_FUNCTION(name) HWASAN_INTERCEPT_FUNC(name) -#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \ - HWASAN_INTERCEPT_FUNC_VER(name, ver) -#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ - CHECK_UNPOISONED_CTX(ctx, ptr, size) -#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ - CHECK_UNPOISONED_CTX(ctx, ptr, size) -#define COMMON_INTERCEPTOR_INITIALIZE_RANGE(ptr, size) \ - HWASAN_WRITE_RANGE(ctx, ptr, size) -#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ - if (hwasan_init_is_running) return REAL(func)(__VA_ARGS__); \ - ENSURE_HWASAN_INITED(); \ - HwasanInterceptorContext hwasan_ctx = {IsInInterceptorScope()}; \ - ctx = (void *)&hwasan_ctx; \ - (void)ctx; \ - InterceptorScope interceptor_scope; -#define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \ - do { \ - } while (false) -#define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ - do { \ - } while (false) -#define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \ - do { \ - } while (false) -#define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \ - do { \ - } while (false) -#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \ - do { \ - } while (false) // FIXME -#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \ - do { \ - } while (false) // FIXME -#define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name) -#define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit() - -#define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \ - if (HwasanThread *t = GetCurrentThread()) { \ - *begin = t->tls_begin(); \ - *end = t->tls_end(); \ - } else { \ - *begin = *end = 0; \ - } - -#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size) \ - { \ - COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size); \ - if (common_flags()->intercept_intrin && \ - MEM_IS_APP(GetAddressFromPointer(dst))) \ - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \ - return REAL(memset)(dst, v, size); \ - } - -#define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, length, prot, flags, fd, \ - offset) \ - do { \ - return mmap_interceptor(REAL(mmap), addr, length, prot, flags, fd, \ - offset); \ - } while (false) - -#include "sanitizer_common/sanitizer_platform_interceptors.h" -#include "sanitizer_common/sanitizer_common_interceptors.inc" -#include "sanitizer_common/sanitizer_signal_interceptors.inc" - -#define COMMON_SYSCALL_PRE_READ_RANGE(p, s) CHECK_UNPOISONED(p, s) -#define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \ - do { \ - (void)(p); \ - (void)(s); \ - } while (false) -#define COMMON_SYSCALL_POST_READ_RANGE(p, s) \ - do { \ - (void)(p); \ - (void)(s); \ - } while (false) -#define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \ - do { \ - (void)(p); \ - (void)(s); \ - } while (false) -#include "sanitizer_common/sanitizer_common_syscalls.inc" -#include "sanitizer_common/sanitizer_syscalls_netbsd.inc" - - - namespace __hwasan { void InitializeInterceptors() { static int inited = 0; CHECK_EQ(inited, 0); - InitializeCommonInterceptors(); - InitializeSignalInterceptors(); - - INTERCEPT_FUNCTION(posix_memalign); - HWASAN_MAYBE_INTERCEPT_MEMALIGN; - INTERCEPT_FUNCTION(__libc_memalign); - INTERCEPT_FUNCTION(valloc); - HWASAN_MAYBE_INTERCEPT_PVALLOC; - INTERCEPT_FUNCTION(malloc); - INTERCEPT_FUNCTION(calloc); + + INTERCEPT_FUNCTION(fork); + +#if HWASAN_WITH_INTERCEPTORS +#if !defined(__aarch64__) + INTERCEPT_FUNCTION(pthread_create); +#endif INTERCEPT_FUNCTION(realloc); INTERCEPT_FUNCTION(free); - HWASAN_MAYBE_INTERCEPT_CFREE; - INTERCEPT_FUNCTION(malloc_usable_size); - HWASAN_MAYBE_INTERCEPT_MALLINFO; - HWASAN_MAYBE_INTERCEPT_MALLOPT; - HWASAN_MAYBE_INTERCEPT_MALLOC_STATS; - INTERCEPT_FUNCTION(pthread_create); - INTERCEPT_FUNCTION(fork); +#endif inited = 1; } diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_interface_internal.h b/contrib/compiler-rt/lib/hwasan/hwasan_interface_internal.h index b4e5c80904df..d3b2087d03c6 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_interface_internal.h +++ b/contrib/compiler-rt/lib/hwasan/hwasan_interface_internal.h @@ -1,4 +1,4 @@ -//===-- hwasan_interface_internal.h -------------------------------*- C++ -*-===// +//===-- hwasan_interface_internal.h -----------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -16,10 +16,14 @@ #define HWASAN_INTERFACE_INTERNAL_H #include "sanitizer_common/sanitizer_internal_defs.h" +#include "sanitizer_common/sanitizer_platform_limits_posix.h" extern "C" { SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_shadow_init(); + +SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_init(); using __sanitizer::uptr; @@ -33,6 +37,9 @@ using __sanitizer::u16; using __sanitizer::u8; SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_init_frames(uptr, uptr); + +SANITIZER_INTERFACE_ATTRIBUTE extern uptr __hwasan_shadow_memory_dynamic_address; SANITIZER_INTERFACE_ATTRIBUTE @@ -91,6 +98,9 @@ SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_tag_memory(uptr p, u8 tag, uptr sz); SANITIZER_INTERFACE_ATTRIBUTE +uptr __hwasan_tag_pointer(uptr p, u8 tag); + +SANITIZER_INTERFACE_ATTRIBUTE u8 __hwasan_generate_tag(); // Returns the offset of the first tag mismatch or -1 if the whole range is @@ -105,6 +115,9 @@ SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_print_shadow(const void *x, uptr size); SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_handle_longjmp(const void *sp_dst); + +SANITIZER_INTERFACE_ATTRIBUTE u16 __sanitizer_unaligned_load16(const uu16 *p); SANITIZER_INTERFACE_ATTRIBUTE @@ -128,6 +141,66 @@ void __hwasan_enable_allocator_tagging(); SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_disable_allocator_tagging(); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_thread_enter(); + +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_thread_exit(); + +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_print_memory_usage(); + +SANITIZER_INTERFACE_ATTRIBUTE +int __sanitizer_posix_memalign(void **memptr, uptr alignment, uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE +void * __sanitizer_memalign(uptr alignment, uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE +void * __sanitizer_aligned_alloc(uptr alignment, uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE +void * __sanitizer___libc_memalign(uptr alignment, uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE +void * __sanitizer_valloc(uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE +void * __sanitizer_pvalloc(uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_free(void *ptr); + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_cfree(void *ptr); + +SANITIZER_INTERFACE_ATTRIBUTE +uptr __sanitizer_malloc_usable_size(const void *ptr); + +SANITIZER_INTERFACE_ATTRIBUTE +__hwasan::__sanitizer_struct_mallinfo __sanitizer_mallinfo(); + +SANITIZER_INTERFACE_ATTRIBUTE +int __sanitizer_mallopt(int cmd, int value); + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_malloc_stats(void); + +SANITIZER_INTERFACE_ATTRIBUTE +void * __sanitizer_calloc(uptr nmemb, uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE +void * __sanitizer_realloc(void *ptr, uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE +void * __sanitizer_malloc(uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE +void *__hwasan_memcpy(void *dst, const void *src, uptr size); +SANITIZER_INTERFACE_ATTRIBUTE +void *__hwasan_memset(void *s, int c, uptr n); +SANITIZER_INTERFACE_ATTRIBUTE +void *__hwasan_memmove(void *dest, const void *src, uptr n); } // extern "C" #endif // HWASAN_INTERFACE_INTERNAL_H diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_linux.cc b/contrib/compiler-rt/lib/hwasan/hwasan_linux.cc index 5ab98dca594d..5b0a8b4ac98f 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_linux.cc +++ b/contrib/compiler-rt/lib/hwasan/hwasan_linux.cc @@ -22,7 +22,9 @@ #include "hwasan_mapping.h" #include "hwasan_report.h" #include "hwasan_thread.h" +#include "hwasan_thread_list.h" +#include <dlfcn.h> #include <elf.h> #include <link.h> #include <pthread.h> @@ -37,6 +39,11 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_procmaps.h" +#if HWASAN_WITH_INTERCEPTORS && !SANITIZER_ANDROID +SANITIZER_INTERFACE_ATTRIBUTE +THREADLOCAL uptr __hwasan_tls; +#endif + namespace __hwasan { static void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) { @@ -51,8 +58,6 @@ static void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) { size); Abort(); } - if (common_flags()->no_huge_pages_for_shadow) NoHugePagesInRegion(beg, size); - if (common_flags()->use_madv_dontdump) DontDumpShadowMemory(beg, size); } static void ProtectGap(uptr addr, uptr size) { @@ -103,55 +108,31 @@ static void PrintAddressSpaceLayout() { else CHECK_EQ(kHighShadowEnd + 1, kHighMemStart); PrintRange(kHighShadowStart, kHighShadowEnd, "HighShadow"); - if (SHADOW_OFFSET) { - if (kLowShadowEnd + 1 < kHighShadowStart) - PrintRange(kLowShadowEnd + 1, kHighShadowStart - 1, "ShadowGap"); - else - CHECK_EQ(kLowMemEnd + 1, kHighShadowStart); - PrintRange(kLowShadowStart, kLowShadowEnd, "LowShadow"); - if (kLowMemEnd + 1 < kLowShadowStart) - PrintRange(kLowMemEnd + 1, kLowShadowStart - 1, "ShadowGap"); - else - CHECK_EQ(kLowMemEnd + 1, kLowShadowStart); - PrintRange(kLowMemStart, kLowMemEnd, "LowMem"); - CHECK_EQ(0, kLowMemStart); - } else { - if (kLowMemEnd + 1 < kHighShadowStart) - PrintRange(kLowMemEnd + 1, kHighShadowStart - 1, "ShadowGap"); - else - CHECK_EQ(kLowMemEnd + 1, kHighShadowStart); - PrintRange(kLowMemStart, kLowMemEnd, "LowMem"); - CHECK_EQ(kLowShadowEnd + 1, kLowMemStart); - PrintRange(kLowShadowStart, kLowShadowEnd, "LowShadow"); - PrintRange(0, kLowShadowStart - 1, "ShadowGap"); - } + if (kLowShadowEnd + 1 < kHighShadowStart) + PrintRange(kLowShadowEnd + 1, kHighShadowStart - 1, "ShadowGap"); + else + CHECK_EQ(kLowMemEnd + 1, kHighShadowStart); + PrintRange(kLowShadowStart, kLowShadowEnd, "LowShadow"); + if (kLowMemEnd + 1 < kLowShadowStart) + PrintRange(kLowMemEnd + 1, kLowShadowStart - 1, "ShadowGap"); + else + CHECK_EQ(kLowMemEnd + 1, kLowShadowStart); + PrintRange(kLowMemStart, kLowMemEnd, "LowMem"); + CHECK_EQ(0, kLowMemStart); } static uptr GetHighMemEnd() { // HighMem covers the upper part of the address space. uptr max_address = GetMaxUserVirtualAddress(); - if (SHADOW_OFFSET) - // Adjust max address to make sure that kHighMemEnd and kHighMemStart are - // properly aligned: - max_address |= SHADOW_GRANULARITY * GetMmapGranularity() - 1; + // Adjust max address to make sure that kHighMemEnd and kHighMemStart are + // properly aligned: + max_address |= (GetMmapGranularity() << kShadowScale) - 1; return max_address; } static void InitializeShadowBaseAddress(uptr shadow_size_bytes) { - // Set the shadow memory address to uninitialized. - __hwasan_shadow_memory_dynamic_address = kDefaultShadowSentinel; - uptr shadow_start = SHADOW_OFFSET; - // Detect if a dynamic shadow address must be used and find the available - // location when necessary. When dynamic address is used, the macro - // kLowShadowBeg expands to __hwasan_shadow_memory_dynamic_address which - // was just set to kDefaultShadowSentinel. - if (shadow_start == kDefaultShadowSentinel) { - __hwasan_shadow_memory_dynamic_address = 0; - CHECK_EQ(0, SHADOW_OFFSET); - shadow_start = FindDynamicShadowStart(shadow_size_bytes); - } - // Update the shadow memory address (potentially) used by instrumentation. - __hwasan_shadow_memory_dynamic_address = shadow_start; + __hwasan_shadow_memory_dynamic_address = + FindDynamicShadowStart(shadow_size_bytes); } bool InitShadow() { @@ -159,29 +140,23 @@ bool InitShadow() { kHighMemEnd = GetHighMemEnd(); // Determine shadow memory base offset. - InitializeShadowBaseAddress(MEM_TO_SHADOW_SIZE(kHighMemEnd)); + InitializeShadowBaseAddress(MemToShadowSize(kHighMemEnd)); // Place the low memory first. - if (SHADOW_OFFSET) { - kLowMemEnd = SHADOW_OFFSET - 1; - kLowMemStart = 0; - } else { - // LowMem covers as much of the first 4GB as possible. - kLowMemEnd = (1UL << 32) - 1; - kLowMemStart = MEM_TO_SHADOW(kLowMemEnd) + 1; - } + kLowMemEnd = __hwasan_shadow_memory_dynamic_address - 1; + kLowMemStart = 0; // Define the low shadow based on the already placed low memory. - kLowShadowEnd = MEM_TO_SHADOW(kLowMemEnd); - kLowShadowStart = SHADOW_OFFSET ? SHADOW_OFFSET : MEM_TO_SHADOW(kLowMemStart); + kLowShadowEnd = MemToShadow(kLowMemEnd); + kLowShadowStart = __hwasan_shadow_memory_dynamic_address; // High shadow takes whatever memory is left up there (making sure it is not // interfering with low memory in the fixed case). - kHighShadowEnd = MEM_TO_SHADOW(kHighMemEnd); - kHighShadowStart = Max(kLowMemEnd, MEM_TO_SHADOW(kHighShadowEnd)) + 1; + kHighShadowEnd = MemToShadow(kHighMemEnd); + kHighShadowStart = Max(kLowMemEnd, MemToShadow(kHighShadowEnd)) + 1; // High memory starts where allocated shadow allows. - kHighMemStart = SHADOW_TO_MEM(kHighShadowStart); + kHighMemStart = ShadowToMem(kHighShadowStart); // Check the sanity of the defined memory ranges (there might be gaps). CHECK_EQ(kHighMemStart % GetMmapGranularity(), 0); @@ -190,10 +165,7 @@ bool InitShadow() { CHECK_GT(kHighShadowStart, kLowMemEnd); CHECK_GT(kLowMemEnd, kLowMemStart); CHECK_GT(kLowShadowEnd, kLowShadowStart); - if (SHADOW_OFFSET) - CHECK_GT(kLowShadowStart, kLowMemEnd); - else - CHECK_GT(kLowMemEnd, kLowShadowStart); + CHECK_GT(kLowShadowStart, kLowMemEnd); if (Verbosity()) PrintAddressSpaceLayout(); @@ -204,21 +176,43 @@ bool InitShadow() { // Protect all the gaps. ProtectGap(0, Min(kLowMemStart, kLowShadowStart)); - if (SHADOW_OFFSET) { - if (kLowMemEnd + 1 < kLowShadowStart) - ProtectGap(kLowMemEnd + 1, kLowShadowStart - kLowMemEnd - 1); - if (kLowShadowEnd + 1 < kHighShadowStart) - ProtectGap(kLowShadowEnd + 1, kHighShadowStart - kLowShadowEnd - 1); - } else { - if (kLowMemEnd + 1 < kHighShadowStart) - ProtectGap(kLowMemEnd + 1, kHighShadowStart - kLowMemEnd - 1); - } + if (kLowMemEnd + 1 < kLowShadowStart) + ProtectGap(kLowMemEnd + 1, kLowShadowStart - kLowMemEnd - 1); + if (kLowShadowEnd + 1 < kHighShadowStart) + ProtectGap(kLowShadowEnd + 1, kHighShadowStart - kLowShadowEnd - 1); if (kHighShadowEnd + 1 < kHighMemStart) ProtectGap(kHighShadowEnd + 1, kHighMemStart - kHighShadowEnd - 1); return true; } +void InitThreads() { + CHECK(__hwasan_shadow_memory_dynamic_address); + uptr guard_page_size = GetMmapGranularity(); + uptr thread_space_start = + __hwasan_shadow_memory_dynamic_address - (1ULL << kShadowBaseAlignment); + uptr thread_space_end = + __hwasan_shadow_memory_dynamic_address - guard_page_size; + ReserveShadowMemoryRange(thread_space_start, thread_space_end - 1, + "hwasan threads"); + ProtectGap(thread_space_end, + __hwasan_shadow_memory_dynamic_address - thread_space_end); + InitThreadList(thread_space_start, thread_space_end - thread_space_start); +} + +static void MadviseShadowRegion(uptr beg, uptr end) { + uptr size = end - beg + 1; + if (common_flags()->no_huge_pages_for_shadow) + NoHugePagesInRegion(beg, size); + if (common_flags()->use_madv_dontdump) + DontDumpShadowMemory(beg, size); +} + +void MadviseShadow() { + MadviseShadowRegion(kLowShadowStart, kLowShadowEnd); + MadviseShadowRegion(kHighShadowStart, kHighShadowEnd); +} + bool MemIsApp(uptr p) { CHECK(GetTagFromPointer(p) == 0); return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd); @@ -240,37 +234,81 @@ void InstallAtExitHandler() { // ---------------------- TSD ---------------- {{{1 +extern "C" void __hwasan_thread_enter() { + hwasanThreadList().CreateCurrentThread(); +} + +extern "C" void __hwasan_thread_exit() { + Thread *t = GetCurrentThread(); + // Make sure that signal handler can not see a stale current thread pointer. + atomic_signal_fence(memory_order_seq_cst); + if (t) + hwasanThreadList().ReleaseThread(t); +} + +#if HWASAN_WITH_INTERCEPTORS static pthread_key_t tsd_key; static bool tsd_key_inited = false; -void HwasanTSDInit(void (*destructor)(void *tsd)) { +void HwasanTSDThreadInit() { + if (tsd_key_inited) + CHECK_EQ(0, pthread_setspecific(tsd_key, + (void *)GetPthreadDestructorIterations())); +} + +void HwasanTSDDtor(void *tsd) { + uptr iterations = (uptr)tsd; + if (iterations > 1) { + CHECK_EQ(0, pthread_setspecific(tsd_key, (void *)(iterations - 1))); + return; + } + __hwasan_thread_exit(); +} + +void HwasanTSDInit() { CHECK(!tsd_key_inited); tsd_key_inited = true; - CHECK_EQ(0, pthread_key_create(&tsd_key, destructor)); + CHECK_EQ(0, pthread_key_create(&tsd_key, HwasanTSDDtor)); } +#else +void HwasanTSDInit() {} +void HwasanTSDThreadInit() {} +#endif -HwasanThread *GetCurrentThread() { - return (HwasanThread*)pthread_getspecific(tsd_key); +#if SANITIZER_ANDROID +uptr *GetCurrentThreadLongPtr() { + return (uptr *)get_android_tls_ptr(); } - -void SetCurrentThread(HwasanThread *t) { - // Make sure that HwasanTSDDtor gets called at the end. - CHECK(tsd_key_inited); - // Make sure we do not reset the current HwasanThread. - CHECK_EQ(0, pthread_getspecific(tsd_key)); - pthread_setspecific(tsd_key, (void *)t); +#else +uptr *GetCurrentThreadLongPtr() { + return &__hwasan_tls; } +#endif -void HwasanTSDDtor(void *tsd) { - HwasanThread *t = (HwasanThread*)tsd; - if (t->destructor_iterations_ > 1) { - t->destructor_iterations_--; - CHECK_EQ(0, pthread_setspecific(tsd_key, tsd)); - return; +#if SANITIZER_ANDROID +void AndroidTestTlsSlot() { + uptr kMagicValue = 0x010203040A0B0C0D; + *(uptr *)get_android_tls_ptr() = kMagicValue; + dlerror(); + if (*(uptr *)get_android_tls_ptr() != kMagicValue) { + Printf( + "ERROR: Incompatible version of Android: TLS_SLOT_SANITIZER(6) is used " + "for dlerror().\n"); + Die(); } - // Make sure that signal handler can not see a stale current thread pointer. - atomic_signal_fence(memory_order_seq_cst); - HwasanThread::TSDDtor(tsd); +} +#else +void AndroidTestTlsSlot() {} +#endif + +Thread *GetCurrentThread() { + uptr *ThreadLong = GetCurrentThreadLongPtr(); +#if HWASAN_WITH_INTERCEPTORS + if (!*ThreadLong) + __hwasan_thread_enter(); +#endif + auto *R = (StackAllocationsRingBuffer *)ThreadLong; + return hwasanThreadList().GetThreadByBufferAddress((uptr)(R->Next())); } struct AccessInfo { @@ -340,14 +378,13 @@ static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) { BufferedStackTrace *stack = stack_buffer.data(); stack->Reset(); SignalContext sig{info, uc}; - GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, uc, - common_flags()->fast_unwind_on_fatal); - - ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store); + GetStackTrace(stack, kStackTraceMax, StackTrace::GetNextInstructionPc(sig.pc), + sig.bp, uc, common_flags()->fast_unwind_on_fatal); ++hwasan_report_count; - if (flags()->halt_on_error || !ai.recover) - Die(); + + bool fatal = flags()->halt_on_error || !ai.recover; + ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal); #if defined(__aarch64__) uc->uc_mcontext.pc += 4; @@ -360,8 +397,8 @@ static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) { static void OnStackUnwind(const SignalContext &sig, const void *, BufferedStackTrace *stack) { - GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, - common_flags()->fast_unwind_on_fatal); + GetStackTrace(stack, kStackTraceMax, StackTrace::GetNextInstructionPc(sig.pc), + sig.bp, sig.context, common_flags()->fast_unwind_on_fatal); } void HwasanOnDeadlySignal(int signo, void *info, void *context) { diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_mapping.h b/contrib/compiler-rt/lib/hwasan/hwasan_mapping.h index 650a5aefcb2d..e5e23dc60337 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_mapping.h +++ b/contrib/compiler-rt/lib/hwasan/hwasan_mapping.h @@ -16,68 +16,41 @@ #define HWASAN_MAPPING_H #include "sanitizer_common/sanitizer_internal_defs.h" +#include "hwasan_interface_internal.h" -// Typical mapping on Linux/x86_64 with fixed shadow mapping: -// || [0x080000000000, 0x7fffffffffff] || HighMem || -// || [0x008000000000, 0x07ffffffffff] || HighShadow || -// || [0x000100000000, 0x007fffffffff] || ShadowGap || -// || [0x000010000000, 0x0000ffffffff] || LowMem || -// || [0x000001000000, 0x00000fffffff] || LowShadow || -// || [0x000000000000, 0x000000ffffff] || ShadowGap || -// -// and with dynamic shadow mapped at [0x770d59f40000, 0x7f0d59f40000]: +// Typical mapping on Linux/x86_64: +// with dynamic shadow mapped at [0x770d59f40000, 0x7f0d59f40000]: // || [0x7f0d59f40000, 0x7fffffffffff] || HighMem || // || [0x7efe2f934000, 0x7f0d59f3ffff] || HighShadow || // || [0x7e7e2f934000, 0x7efe2f933fff] || ShadowGap || // || [0x770d59f40000, 0x7e7e2f933fff] || LowShadow || // || [0x000000000000, 0x770d59f3ffff] || LowMem || -// Typical mapping on Android/AArch64 (39-bit VMA): -// || [0x001000000000, 0x007fffffffff] || HighMem || -// || [0x000800000000, 0x000fffffffff] || ShadowGap || -// || [0x000100000000, 0x0007ffffffff] || HighShadow || -// || [0x000010000000, 0x0000ffffffff] || LowMem || -// || [0x000001000000, 0x00000fffffff] || LowShadow || -// || [0x000000000000, 0x000000ffffff] || ShadowGap || -// -// and with dynamic shadow mapped: [0x007477480000, 0x007c77480000]: +// Typical mapping on Android/AArch64 +// with dynamic shadow mapped: [0x007477480000, 0x007c77480000]: // || [0x007c77480000, 0x007fffffffff] || HighMem || // || [0x007c3ebc8000, 0x007c7747ffff] || HighShadow || // || [0x007bbebc8000, 0x007c3ebc7fff] || ShadowGap || // || [0x007477480000, 0x007bbebc7fff] || LowShadow || // || [0x000000000000, 0x00747747ffff] || LowMem || -static constexpr __sanitizer::u64 kDefaultShadowSentinel = ~(__sanitizer::u64)0; - // Reasonable values are 4 (for 1/16th shadow) and 6 (for 1/64th). -constexpr __sanitizer::uptr kShadowScale = 4; -constexpr __sanitizer::uptr kShadowAlignment = 1ULL << kShadowScale; - -#if SANITIZER_ANDROID -# define HWASAN_FIXED_MAPPING 0 -#else -# define HWASAN_FIXED_MAPPING 1 -#endif - -#if HWASAN_FIXED_MAPPING -# define SHADOW_OFFSET (0) -# define HWASAN_PREMAP_SHADOW 0 -#else -# define SHADOW_OFFSET (__hwasan_shadow_memory_dynamic_address) -# define HWASAN_PREMAP_SHADOW 1 -#endif - -#define SHADOW_GRANULARITY (1ULL << kShadowScale) - -#define MEM_TO_SHADOW(mem) (((uptr)(mem) >> kShadowScale) + SHADOW_OFFSET) -#define SHADOW_TO_MEM(shadow) (((uptr)(shadow) - SHADOW_OFFSET) << kShadowScale) - -#define MEM_TO_SHADOW_SIZE(size) ((uptr)(size) >> kShadowScale) - -#define MEM_IS_APP(mem) MemIsApp((uptr)(mem)) +constexpr uptr kShadowScale = 4; +constexpr uptr kShadowAlignment = 1ULL << kShadowScale; namespace __hwasan { +inline uptr MemToShadow(uptr untagged_addr) { + return (untagged_addr >> kShadowScale) + + __hwasan_shadow_memory_dynamic_address; +} +inline uptr ShadowToMem(uptr shadow_addr) { + return (shadow_addr - __hwasan_shadow_memory_dynamic_address) << kShadowScale; +} +inline uptr MemToShadowSize(uptr size) { + return size >> kShadowScale; +} + bool MemIsApp(uptr p); } // namespace __hwasan diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_memintrinsics.cc b/contrib/compiler-rt/lib/hwasan/hwasan_memintrinsics.cc new file mode 100644 index 000000000000..9cb844e45c40 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_memintrinsics.cc @@ -0,0 +1,45 @@ +//===-- hwasan_memintrinsics.cc ---------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file is a part of HWAddressSanitizer and contains HWASAN versions of +/// memset, memcpy and memmove +/// +//===----------------------------------------------------------------------===// + +#include <string.h> +#include "hwasan.h" +#include "hwasan_checks.h" +#include "hwasan_flags.h" +#include "hwasan_interface_internal.h" +#include "sanitizer_common/sanitizer_libc.h" + +using namespace __hwasan; + +void *__hwasan_memset(void *block, int c, uptr size) { + CheckAddressSized<ErrorAction::Recover, AccessType::Store>( + reinterpret_cast<uptr>(block), size); + return memset(UntagPtr(block), c, size); +} + +void *__hwasan_memcpy(void *to, const void *from, uptr size) { + CheckAddressSized<ErrorAction::Recover, AccessType::Store>( + reinterpret_cast<uptr>(to), size); + CheckAddressSized<ErrorAction::Recover, AccessType::Load>( + reinterpret_cast<uptr>(from), size); + return memcpy(UntagPtr(to), UntagPtr(from), size); +} + +void *__hwasan_memmove(void *to, const void *from, uptr size) { + CheckAddressSized<ErrorAction::Recover, AccessType::Store>( + reinterpret_cast<uptr>(to), size); + CheckAddressSized<ErrorAction::Recover, AccessType::Load>( + reinterpret_cast<uptr>(from), size); + return memmove(UntagPtr(to), UntagPtr(from), size); +} diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_new_delete.cc b/contrib/compiler-rt/lib/hwasan/hwasan_new_delete.cc index 63ca74edd481..f2e8faf5da73 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_new_delete.cc +++ b/contrib/compiler-rt/lib/hwasan/hwasan_new_delete.cc @@ -51,7 +51,7 @@ void *operator new[](size_t size, std::nothrow_t const&) { #define OPERATOR_DELETE_BODY \ GET_MALLOC_STACK_TRACE; \ - if (ptr) HwasanDeallocate(&stack, ptr) + if (ptr) hwasan_free(ptr, &stack) INTERCEPTOR_ATTRIBUTE void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; } diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.cc b/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.cc index b99d8ed0be79..9c8e16b12ad5 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.cc +++ b/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.cc @@ -1,4 +1,4 @@ -//===-- hwasan_poisoning.cc ---------------------------------------*- C++ -*-===// +//===-- hwasan_poisoning.cc -------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -22,8 +22,8 @@ namespace __hwasan { uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) { CHECK(IsAligned(p, kShadowAlignment)); CHECK(IsAligned(size, kShadowAlignment)); - uptr shadow_start = MEM_TO_SHADOW(p); - uptr shadow_size = MEM_TO_SHADOW_SIZE(size); + uptr shadow_start = MemToShadow(p); + uptr shadow_size = MemToShadowSize(size); internal_memset((void *)shadow_start, tag, shadow_size); return AddTagToPointer(p, tag); } diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.h b/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.h index b44a91f975f5..0dbf9d8ed4ef 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.h +++ b/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.h @@ -1,4 +1,4 @@ -//===-- hwasan_poisoning.h ----------------------------------------*- C++ -*-===// +//===-- hwasan_poisoning.h --------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_report.cc b/contrib/compiler-rt/lib/hwasan/hwasan_report.cc index 16e9016ea35b..ea3e4096daef 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_report.cc +++ b/contrib/compiler-rt/lib/hwasan/hwasan_report.cc @@ -15,18 +15,64 @@ #include "hwasan.h" #include "hwasan_allocator.h" #include "hwasan_mapping.h" +#include "hwasan_thread.h" +#include "hwasan_thread_list.h" #include "sanitizer_common/sanitizer_allocator_internal.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_mutex.h" #include "sanitizer_common/sanitizer_report_decorator.h" #include "sanitizer_common/sanitizer_stackdepot.h" +#include "sanitizer_common/sanitizer_stacktrace_printer.h" #include "sanitizer_common/sanitizer_symbolizer.h" using namespace __sanitizer; namespace __hwasan { +class ScopedReport { + public: + ScopedReport(bool fatal = false) : error_message_(1), fatal(fatal) { + BlockingMutexLock lock(&error_message_lock_); + error_message_ptr_ = fatal ? &error_message_ : nullptr; + } + + ~ScopedReport() { + BlockingMutexLock lock(&error_message_lock_); + if (fatal) { + SetAbortMessage(error_message_.data()); + Die(); + } + error_message_ptr_ = nullptr; + } + + static void MaybeAppendToErrorMessage(const char *msg) { + BlockingMutexLock lock(&error_message_lock_); + if (!error_message_ptr_) + return; + uptr len = internal_strlen(msg); + uptr old_size = error_message_ptr_->size(); + error_message_ptr_->resize(old_size + len); + // overwrite old trailing '\0', keep new trailing '\0' untouched. + internal_memcpy(&(*error_message_ptr_)[old_size - 1], msg, len); + } + private: + ScopedErrorReportLock error_report_lock_; + InternalMmapVector<char> error_message_; + bool fatal; + + static InternalMmapVector<char> *error_message_ptr_; + static BlockingMutex error_message_lock_; +}; + +InternalMmapVector<char> *ScopedReport::error_message_ptr_; +BlockingMutex ScopedReport::error_message_lock_; + +// If there is an active ScopedReport, append to its error message. +void AppendToErrorMessageBuffer(const char *buffer) { + ScopedReport::MaybeAppendToErrorMessage(buffer); +} + static StackTrace GetStackTraceFromId(u32 id) { CHECK(id); StackTrace res = StackDepotGet(id); @@ -34,100 +80,344 @@ static StackTrace GetStackTraceFromId(u32 id) { return res; } +// A RAII object that holds a copy of the current thread stack ring buffer. +// The actual stack buffer may change while we are iterating over it (for +// example, Printf may call syslog() which can itself be built with hwasan). +class SavedStackAllocations { + public: + SavedStackAllocations(StackAllocationsRingBuffer *rb) { + uptr size = rb->size() * sizeof(uptr); + void *storage = + MmapAlignedOrDieOnFatalError(size, size * 2, "saved stack allocations"); + new (&rb_) StackAllocationsRingBuffer(*rb, storage); + } + + ~SavedStackAllocations() { + StackAllocationsRingBuffer *rb = get(); + UnmapOrDie(rb->StartOfStorage(), rb->size() * sizeof(uptr)); + } + + StackAllocationsRingBuffer *get() { + return (StackAllocationsRingBuffer *)&rb_; + } + + private: + uptr rb_; +}; + class Decorator: public __sanitizer::SanitizerCommonDecorator { public: Decorator() : SanitizerCommonDecorator() { } + const char *Access() { return Blue(); } const char *Allocation() const { return Magenta(); } const char *Origin() const { return Magenta(); } const char *Name() const { return Green(); } + const char *Location() { return Green(); } + const char *Thread() { return Green(); } }; -struct HeapAddressDescription { - uptr addr; - u32 alloc_stack_id; - u32 free_stack_id; - - void Print() const { - Decorator d; - if (free_stack_id) { - Printf("%sfreed here:%s\n", d.Allocation(), d.Default()); - GetStackTraceFromId(free_stack_id).Print(); - Printf("%spreviously allocated here:%s\n", d.Allocation(), d.Default()); - } else { - Printf("%sallocated here:%s\n", d.Allocation(), d.Default()); +// Returns the index of the rb element that matches tagged_addr (plus one), +// or zero if found nothing. +uptr FindHeapAllocation(HeapAllocationsRingBuffer *rb, + uptr tagged_addr, + HeapAllocationRecord *har) { + if (!rb) return 0; + for (uptr i = 0, size = rb->size(); i < size; i++) { + auto h = (*rb)[i]; + if (h.tagged_addr <= tagged_addr && + h.tagged_addr + h.requested_size > tagged_addr) { + *har = h; + return i + 1; } - GetStackTraceFromId(alloc_stack_id).Print(); } -}; + return 0; +} + +void PrintAddressDescription( + uptr tagged_addr, uptr access_size, + StackAllocationsRingBuffer *current_stack_allocations) { + Decorator d; + int num_descriptions_printed = 0; + uptr untagged_addr = UntagAddr(tagged_addr); + + // Print some very basic information about the address, if it's a heap. + HwasanChunkView chunk = FindHeapChunkByAddress(untagged_addr); + if (uptr beg = chunk.Beg()) { + uptr size = chunk.ActualSize(); + Printf("%s[%p,%p) is a %s %s heap chunk; " + "size: %zd offset: %zd\n%s", + d.Location(), + beg, beg + size, + chunk.FromSmallHeap() ? "small" : "large", + chunk.IsAllocated() ? "allocated" : "unallocated", + size, untagged_addr - beg, + d.Default()); + } + + // Check if this looks like a heap buffer overflow by scanning + // the shadow left and right and looking for the first adjacent + // object with a different memory tag. If that tag matches addr_tag, + // check the allocator if it has a live chunk there. + tag_t addr_tag = GetTagFromPointer(tagged_addr); + tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr)); + if (*tag_ptr != addr_tag) { // should be true usually. + tag_t *left = tag_ptr, *right = tag_ptr; + // scan left. + for (int i = 0; i < 1000 && *left == *tag_ptr; i++, left--){} + // scan right. + for (int i = 0; i < 1000 && *right == *tag_ptr; i++, right++){} + // Chose the object that has addr_tag and that is closer to addr. + tag_t *candidate = nullptr; + if (*right == addr_tag && *left == addr_tag) + candidate = right - tag_ptr < tag_ptr - left ? right : left; + else if (*right == addr_tag) + candidate = right; + else if (*left == addr_tag) + candidate = left; + + if (candidate) { + uptr mem = ShadowToMem(reinterpret_cast<uptr>(candidate)); + HwasanChunkView chunk = FindHeapChunkByAddress(mem); + if (chunk.IsAllocated()) { + Printf("%s", d.Location()); + Printf( + "%p is located %zd bytes to the %s of %zd-byte region [%p,%p)\n", + untagged_addr, + candidate == left ? untagged_addr - chunk.End() + : chunk.Beg() - untagged_addr, + candidate == right ? "left" : "right", chunk.UsedSize(), + chunk.Beg(), chunk.End()); + Printf("%s", d.Allocation()); + Printf("allocated here:\n"); + Printf("%s", d.Default()); + GetStackTraceFromId(chunk.GetAllocStackId()).Print(); + num_descriptions_printed++; + } + } + } + + hwasanThreadList().VisitAllLiveThreads([&](Thread *t) { + // Scan all threads' ring buffers to find if it's a heap-use-after-free. + HeapAllocationRecord har; + if (uptr D = FindHeapAllocation(t->heap_allocations(), tagged_addr, &har)) { + Printf("%s", d.Location()); + Printf("%p is located %zd bytes inside of %zd-byte region [%p,%p)\n", + untagged_addr, untagged_addr - UntagAddr(har.tagged_addr), + har.requested_size, UntagAddr(har.tagged_addr), + UntagAddr(har.tagged_addr) + har.requested_size); + Printf("%s", d.Allocation()); + Printf("freed by thread T%zd here:\n", t->unique_id()); + Printf("%s", d.Default()); + GetStackTraceFromId(har.free_context_id).Print(); + + Printf("%s", d.Allocation()); + Printf("previously allocated here:\n", t); + Printf("%s", d.Default()); + GetStackTraceFromId(har.alloc_context_id).Print(); + + // Print a developer note: the index of this heap object + // in the thread's deallocation ring buffer. + Printf("hwasan_dev_note_heap_rb_distance: %zd %zd\n", D, + flags()->heap_history_size); + + t->Announce(); + num_descriptions_printed++; + } + + // Very basic check for stack memory. + if (t->AddrIsInStack(untagged_addr)) { + Printf("%s", d.Location()); + Printf("Address %p is located in stack of thread T%zd\n", untagged_addr, + t->unique_id()); + Printf("%s", d.Default()); + t->Announce(); -bool GetHeapAddressInformation(uptr addr, uptr access_size, - HeapAddressDescription *description) { - HwasanChunkView chunk = FindHeapChunkByAddress(addr); - if (!chunk.IsValid()) - return false; - description->addr = addr; - description->alloc_stack_id = chunk.GetAllocStackId(); - description->free_stack_id = chunk.GetFreeStackId(); - return true; + // Temporary report section, needs to be improved. + Printf("Previously allocated frames:\n"); + auto *sa = (t == GetCurrentThread() && current_stack_allocations) + ? current_stack_allocations + : t->stack_allocations(); + uptr frames = Min((uptr)flags()->stack_history_size, sa->size()); + InternalScopedString frame_desc(GetPageSizeCached() * 2); + for (uptr i = 0; i < frames; i++) { + uptr record = (*sa)[i]; + if (!record) + break; + uptr sp = (record >> 48) << 4; + uptr pc_mask = (1ULL << 48) - 1; + uptr pc = record & pc_mask; + if (SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc)) { + frame_desc.append(" sp: 0x%zx pc: %p ", sp, pc); + RenderFrame(&frame_desc, "in %f %s:%l\n", 0, frame->info, + common_flags()->symbolize_vs_style, + common_flags()->strip_path_prefix); + frame->ClearAll(); + if (auto Descr = GetStackFrameDescr(pc)) + frame_desc.append(" %s\n", Descr); + } + Printf("%s", frame_desc.data()); + frame_desc.clear(); + } + + num_descriptions_printed++; + } + }); + + // Print the remaining threads, as an extra information, 1 line per thread. + hwasanThreadList().VisitAllLiveThreads([&](Thread *t) { t->Announce(); }); + + if (!num_descriptions_printed) + // We exhausted our possibilities. Bail out. + Printf("HWAddressSanitizer can not describe address in more detail.\n"); } -void PrintAddressDescription(uptr addr, uptr access_size) { - HeapAddressDescription heap_description; - if (GetHeapAddressInformation(addr, access_size, &heap_description)) { - heap_description.Print(); - return; +void ReportStats() {} + +static void PrintTagsAroundAddr(tag_t *tag_ptr) { + Printf( + "Memory tags around the buggy address (one tag corresponds to %zd " + "bytes):\n", kShadowAlignment); + + const uptr row_len = 16; // better be power of two. + const uptr num_rows = 17; + tag_t *center_row_beg = reinterpret_cast<tag_t *>( + RoundDownTo(reinterpret_cast<uptr>(tag_ptr), row_len)); + tag_t *beg_row = center_row_beg - row_len * (num_rows / 2); + tag_t *end_row = center_row_beg + row_len * (num_rows / 2); + InternalScopedString s(GetPageSizeCached() * 8); + for (tag_t *row = beg_row; row < end_row; row += row_len) { + s.append("%s", row == center_row_beg ? "=>" : " "); + for (uptr i = 0; i < row_len; i++) { + s.append("%s", row + i == tag_ptr ? "[" : " "); + s.append("%02x", row[i]); + s.append("%s", row + i == tag_ptr ? "]" : " "); + } + s.append("%s\n", row == center_row_beg ? "<=" : " "); } - // We exhausted our possibilities. Bail out. - Printf("HWAddressSanitizer can not describe address in more detail.\n"); + Printf("%s", s.data()); } -void ReportInvalidAccess(StackTrace *stack, u32 origin) { - ScopedErrorReportLock l; +void ReportInvalidFree(StackTrace *stack, uptr tagged_addr) { + ScopedReport R(flags()->halt_on_error); + uptr untagged_addr = UntagAddr(tagged_addr); + tag_t ptr_tag = GetTagFromPointer(tagged_addr); + tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr)); + tag_t mem_tag = *tag_ptr; Decorator d; - Printf("%s", d.Warning()); - Report("WARNING: HWAddressSanitizer: invalid access\n"); + Printf("%s", d.Error()); + uptr pc = stack->size ? stack->trace[0] : 0; + const char *bug_type = "invalid-free"; + Report("ERROR: %s: %s on address %p at pc %p\n", SanitizerToolName, bug_type, + untagged_addr, pc); + Printf("%s", d.Access()); + Printf("tags: %02x/%02x (ptr/mem)\n", ptr_tag, mem_tag); Printf("%s", d.Default()); + stack->Print(); - ReportErrorSummary("invalid-access", stack); -} -void ReportStats() {} + PrintAddressDescription(tagged_addr, 0, nullptr); -void ReportInvalidAccessInsideAddressRange(const char *what, const void *start, - uptr size, uptr offset) { - ScopedErrorReportLock l; + PrintTagsAroundAddr(tag_ptr); + ReportErrorSummary(bug_type, stack); +} + +void ReportTailOverwritten(StackTrace *stack, uptr tagged_addr, uptr orig_size, + uptr tail_size, const u8 *expected) { + ScopedReport R(flags()->halt_on_error); Decorator d; - Printf("%s", d.Warning()); - Printf("%sTag mismatch in %s%s%s at offset %zu inside [%p, %zu)%s\n", - d.Warning(), d.Name(), what, d.Warning(), offset, start, size, - d.Default()); - PrintAddressDescription((uptr)start + offset, 1); - // if (__sanitizer::Verbosity()) - // DescribeMemoryRange(start, size); + uptr untagged_addr = UntagAddr(tagged_addr); + Printf("%s", d.Error()); + const char *bug_type = "alocation-tail-overwritten"; + Report("ERROR: %s: %s; heap object [%p,%p) of size %zd\n", SanitizerToolName, + bug_type, untagged_addr, untagged_addr + orig_size, orig_size); + Printf("\n%s", d.Default()); + stack->Print(); + HwasanChunkView chunk = FindHeapChunkByAddress(untagged_addr); + if (chunk.Beg()) { + Printf("%s", d.Allocation()); + Printf("allocated here:\n"); + Printf("%s", d.Default()); + GetStackTraceFromId(chunk.GetAllocStackId()).Print(); + } + + InternalScopedString s(GetPageSizeCached() * 8); + CHECK_GT(tail_size, 0U); + CHECK_LT(tail_size, kShadowAlignment); + u8 *tail = reinterpret_cast<u8*>(untagged_addr + orig_size); + s.append("Tail contains: "); + for (uptr i = 0; i < kShadowAlignment - tail_size; i++) + s.append(".. "); + for (uptr i = 0; i < tail_size; i++) + s.append("%02x ", tail[i]); + s.append("\n"); + s.append("Expected: "); + for (uptr i = 0; i < kShadowAlignment - tail_size; i++) + s.append(".. "); + for (uptr i = 0; i < tail_size; i++) + s.append("%02x ", expected[i]); + s.append("\n"); + s.append(" "); + for (uptr i = 0; i < kShadowAlignment - tail_size; i++) + s.append(" "); + for (uptr i = 0; i < tail_size; i++) + s.append("%s ", expected[i] != tail[i] ? "^^" : " "); + + s.append("\nThis error occurs when a buffer overflow overwrites memory\n" + "to the right of a heap object, but within the %zd-byte granule, e.g.\n" + " char *x = new char[20];\n" + " x[25] = 42;\n" + "By default %s does not detect such bugs at the time of write,\n" + "but can detect them at the time of free/delete.\n" + "To disable this feature set HWASAN_OPTIONS=free_checks_tail_magic=0;\n" + "To enable checking at the time of access, set " + "HWASAN_OPTIONS=malloc_align_right to non-zero\n\n", + kShadowAlignment, SanitizerToolName); + Printf("%s", s.data()); + GetCurrentThread()->Announce(); + + tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr)); + PrintTagsAroundAddr(tag_ptr); + + ReportErrorSummary(bug_type, stack); } -void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size, - bool is_store) { - ScopedErrorReportLock l; +void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size, + bool is_store, bool fatal) { + ScopedReport R(fatal); + SavedStackAllocations current_stack_allocations( + GetCurrentThread()->stack_allocations()); Decorator d; - Printf("%s", d.Warning()); - uptr address = GetAddressFromPointer(addr); - Printf("%s of size %zu at %p\n", is_store ? "WRITE" : "READ", access_size, - address); - - tag_t ptr_tag = GetTagFromPointer(addr); - tag_t mem_tag = *(tag_t *)MEM_TO_SHADOW(address); - Printf("pointer tag 0x%x\nmemory tag 0x%x\n", ptr_tag, mem_tag); + Printf("%s", d.Error()); + uptr untagged_addr = UntagAddr(tagged_addr); + // TODO: when possible, try to print heap-use-after-free, etc. + const char *bug_type = "tag-mismatch"; + uptr pc = stack->size ? stack->trace[0] : 0; + Report("ERROR: %s: %s on address %p at pc %p\n", SanitizerToolName, bug_type, + untagged_addr, pc); + + Thread *t = GetCurrentThread(); + + tag_t ptr_tag = GetTagFromPointer(tagged_addr); + tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr)); + tag_t mem_tag = *tag_ptr; + Printf("%s", d.Access()); + Printf("%s of size %zu at %p tags: %02x/%02x (ptr/mem) in thread T%zd\n", + is_store ? "WRITE" : "READ", access_size, untagged_addr, ptr_tag, + mem_tag, t->unique_id()); Printf("%s", d.Default()); stack->Print(); - PrintAddressDescription(address, access_size); + PrintAddressDescription(tagged_addr, access_size, + current_stack_allocations.get()); + t->Announce(); + + PrintTagsAroundAddr(tag_ptr); - ReportErrorSummary("tag-mismatch", stack); + ReportErrorSummary(bug_type, stack); } } // namespace __hwasan diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_report.h b/contrib/compiler-rt/lib/hwasan/hwasan_report.h index bb33f1a87308..10fb20cc5be1 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_report.h +++ b/contrib/compiler-rt/lib/hwasan/hwasan_report.h @@ -21,12 +21,12 @@ namespace __hwasan { -void ReportInvalidAccess(StackTrace *stack, u32 origin); void ReportStats(); -void ReportInvalidAccessInsideAddressRange(const char *what, const void *start, - uptr size, uptr offset); void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size, - bool is_store); + bool is_store, bool fatal); +void ReportInvalidFree(StackTrace *stack, uptr addr); +void ReportTailOverwritten(StackTrace *stack, uptr addr, uptr orig_size, + uptr tail_size, const u8 *expected); void ReportAtExitStatistics(); diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_thread.cc b/contrib/compiler-rt/lib/hwasan/hwasan_thread.cc index b50c0fc76be4..631c2813eb06 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_thread.cc +++ b/contrib/compiler-rt/lib/hwasan/hwasan_thread.cc @@ -5,8 +5,11 @@ #include "hwasan_poisoning.h" #include "hwasan_interface_internal.h" +#include "sanitizer_common/sanitizer_file.h" +#include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_tls_get_addr.h" + namespace __hwasan { static u32 RandomSeed() { @@ -22,69 +25,70 @@ static u32 RandomSeed() { return seed; } -HwasanThread *HwasanThread::Create(thread_callback_t start_routine, - void *arg) { - uptr PageSize = GetPageSizeCached(); - uptr size = RoundUpTo(sizeof(HwasanThread), PageSize); - HwasanThread *thread = (HwasanThread*)MmapOrDie(size, __func__); - thread->start_routine_ = start_routine; - thread->arg_ = arg; - thread->destructor_iterations_ = GetPthreadDestructorIterations(); - thread->random_state_ = flags()->random_tags ? RandomSeed() : 0; - - return thread; -} - -void HwasanThread::SetThreadStackAndTls() { - uptr tls_size = 0; - uptr stack_size = 0; - GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, - &tls_begin_, &tls_size); +void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size) { + static u64 unique_id; + unique_id_ = unique_id++; + random_state_ = flags()->random_tags ? RandomSeed() : unique_id_; + if (auto sz = flags()->heap_history_size) + heap_allocations_ = HeapAllocationsRingBuffer::New(sz); + + HwasanTSDThreadInit(); // Only needed with interceptors. + uptr *ThreadLong = GetCurrentThreadLongPtr(); + // The following implicitly sets (this) as the current thread. + stack_allocations_ = new (ThreadLong) + StackAllocationsRingBuffer((void *)stack_buffer_start, stack_buffer_size); + // Check that it worked. + CHECK_EQ(GetCurrentThread(), this); + + // ScopedTaggingDisable needs GetCurrentThread to be set up. + ScopedTaggingDisabler disabler; + + uptr tls_size; + uptr stack_size; + GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, &tls_begin_, + &tls_size); stack_top_ = stack_bottom_ + stack_size; tls_end_ = tls_begin_ + tls_size; - int local; - CHECK(AddrIsInStack((uptr)&local)); -} - -void HwasanThread::Init() { - SetThreadStackAndTls(); - CHECK(MEM_IS_APP(stack_bottom_)); - CHECK(MEM_IS_APP(stack_top_ - 1)); -} + if (stack_bottom_) { + int local; + CHECK(AddrIsInStack((uptr)&local)); + CHECK(MemIsApp(stack_bottom_)); + CHECK(MemIsApp(stack_top_ - 1)); + } -void HwasanThread::TSDDtor(void *tsd) { - HwasanThread *t = (HwasanThread*)tsd; - t->Destroy(); + if (flags()->verbose_threads) { + if (IsMainThread()) { + Printf("sizeof(Thread): %zd sizeof(HeapRB): %zd sizeof(StackRB): %zd\n", + sizeof(Thread), heap_allocations_->SizeInBytes(), + stack_allocations_->size() * sizeof(uptr)); + } + Print("Creating : "); + } } -void HwasanThread::ClearShadowForThreadStackAndTLS() { - TagMemory(stack_bottom_, stack_top_ - stack_bottom_, 0); +void Thread::ClearShadowForThreadStackAndTLS() { + if (stack_top_ != stack_bottom_) + TagMemory(stack_bottom_, stack_top_ - stack_bottom_, 0); if (tls_begin_ != tls_end_) TagMemory(tls_begin_, tls_end_ - tls_begin_, 0); } -void HwasanThread::Destroy() { - malloc_storage().CommitBack(); +void Thread::Destroy() { + if (flags()->verbose_threads) + Print("Destroying: "); + AllocatorSwallowThreadLocalCache(allocator_cache()); ClearShadowForThreadStackAndTLS(); - uptr size = RoundUpTo(sizeof(HwasanThread), GetPageSizeCached()); - UnmapOrDie(this, size); + if (heap_allocations_) + heap_allocations_->Delete(); DTLS_Destroy(); } -thread_return_t HwasanThread::ThreadStart() { - Init(); - - if (!start_routine_) { - // start_routine_ == 0 if we're on the main thread or on one of the - // OS X libdispatch worker threads. But nobody is supposed to call - // ThreadStart() for the worker threads. - return 0; - } - - thread_return_t res = start_routine_(arg_); - - return res; +void Thread::Print(const char *Prefix) { + Printf("%sT%zd %p stack: [%p,%p) sz: %zd tls: [%p,%p)\n", Prefix, + unique_id_, this, stack_bottom(), stack_top(), + stack_top() - stack_bottom(), + tls_begin(), tls_end()); } static u32 xorshift(u32 state) { @@ -95,7 +99,8 @@ static u32 xorshift(u32 state) { } // Generate a (pseudo-)random non-zero tag. -tag_t HwasanThread::GenerateRandomTag() { +tag_t Thread::GenerateRandomTag() { + if (tagging_disabled_) return 0; tag_t tag; do { if (flags()->random_tags) { diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_thread.h b/contrib/compiler-rt/lib/hwasan/hwasan_thread.h index 1e482adeac84..4830473f4aab 100644 --- a/contrib/compiler-rt/lib/hwasan/hwasan_thread.h +++ b/contrib/compiler-rt/lib/hwasan/hwasan_thread.h @@ -1,4 +1,4 @@ -//===-- hwasan_thread.h -------------------------------------------*- C++ -*-===// +//===-- hwasan_thread.h -----------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -16,23 +16,23 @@ #include "hwasan_allocator.h" #include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_ring_buffer.h" namespace __hwasan { -class HwasanThread { +typedef __sanitizer::CompactRingBuffer<uptr> StackAllocationsRingBuffer; + +class Thread { public: - static HwasanThread *Create(thread_callback_t start_routine, void *arg); - static void TSDDtor(void *tsd); + void Init(uptr stack_buffer_start, uptr stack_buffer_size); // Must be called from the thread itself. void Destroy(); - void Init(); // Should be called from the thread itself. - thread_return_t ThreadStart(); - uptr stack_top() { return stack_top_; } uptr stack_bottom() { return stack_bottom_; } + uptr stack_size() { return stack_top() - stack_bottom(); } uptr tls_begin() { return tls_begin_; } uptr tls_end() { return tls_end_; } - bool IsMainThread() { return start_routine_ == nullptr; } + bool IsMainThread() { return unique_id_ == 0; } bool AddrIsInStack(uptr addr) { return addr >= stack_bottom_ && addr < stack_top_; @@ -50,19 +50,28 @@ class HwasanThread { void EnterInterceptorScope() { in_interceptor_scope_++; } void LeaveInterceptorScope() { in_interceptor_scope_--; } - HwasanThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; } + AllocatorCache *allocator_cache() { return &allocator_cache_; } + HeapAllocationsRingBuffer *heap_allocations() { return heap_allocations_; } + StackAllocationsRingBuffer *stack_allocations() { return stack_allocations_; } tag_t GenerateRandomTag(); - int destructor_iterations_; + void DisableTagging() { tagging_disabled_++; } + void EnableTagging() { tagging_disabled_--; } + bool TaggingIsDisabled() const { return tagging_disabled_; } + + u64 unique_id() const { return unique_id_; } + void Announce() { + if (announced_) return; + announced_ = true; + Print("Thread: "); + } private: - // NOTE: There is no HwasanThread constructor. It is allocated + // NOTE: There is no Thread constructor. It is allocated // via mmap() and *must* be valid in zero-initialized state. - void SetThreadStackAndTls(); void ClearShadowForThreadStackAndTLS(); - thread_callback_t start_routine_; - void *arg_; + void Print(const char *prefix); uptr stack_top_; uptr stack_bottom_; uptr tls_begin_; @@ -75,11 +84,30 @@ class HwasanThread { u32 random_state_; u32 random_buffer_; - HwasanThreadLocalMallocStorage malloc_storage_; + AllocatorCache allocator_cache_; + HeapAllocationsRingBuffer *heap_allocations_; + StackAllocationsRingBuffer *stack_allocations_; + + static void InsertIntoThreadList(Thread *t); + static void RemoveFromThreadList(Thread *t); + Thread *next_; // All live threads form a linked list. + + u64 unique_id_; // counting from zero. + + u32 tagging_disabled_; // if non-zero, malloc uses zero tag in this thread. + + bool announced_; + + friend struct ThreadListHead; }; -HwasanThread *GetCurrentThread(); -void SetCurrentThread(HwasanThread *t); +Thread *GetCurrentThread(); +uptr *GetCurrentThreadLongPtr(); + +struct ScopedTaggingDisabler { + ScopedTaggingDisabler() { GetCurrentThread()->DisableTagging(); } + ~ScopedTaggingDisabler() { GetCurrentThread()->EnableTagging(); } +}; } // namespace __hwasan diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_thread_list.cc b/contrib/compiler-rt/lib/hwasan/hwasan_thread_list.cc new file mode 100644 index 000000000000..a31eee84ed93 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_thread_list.cc @@ -0,0 +1,15 @@ +#include "hwasan_thread_list.h" + +namespace __hwasan { +static ALIGNED(16) char thread_list_placeholder[sizeof(HwasanThreadList)]; +static HwasanThreadList *hwasan_thread_list; + +HwasanThreadList &hwasanThreadList() { return *hwasan_thread_list; } + +void InitThreadList(uptr storage, uptr size) { + CHECK(hwasan_thread_list == nullptr); + hwasan_thread_list = + new (thread_list_placeholder) HwasanThreadList(storage, size); +} + +} // namespace diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_thread_list.h b/contrib/compiler-rt/lib/hwasan/hwasan_thread_list.h new file mode 100644 index 000000000000..53747b51fd65 --- /dev/null +++ b/contrib/compiler-rt/lib/hwasan/hwasan_thread_list.h @@ -0,0 +1,200 @@ +//===-- hwasan_thread_list.h ------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +//===----------------------------------------------------------------------===// + +// HwasanThreadList is a registry for live threads, as well as an allocator for +// HwasanThread objects and their stack history ring buffers. There are +// constraints on memory layout of the shadow region and CompactRingBuffer that +// are part of the ABI contract between compiler-rt and llvm. +// +// * Start of the shadow memory region is aligned to 2**kShadowBaseAlignment. +// * All stack ring buffers are located within (2**kShadowBaseAlignment) +// sized region below and adjacent to the shadow region. +// * Each ring buffer has a size of (2**N)*4096 where N is in [0, 8), and is +// aligned to twice its size. The value of N can be different for each buffer. +// +// These constrains guarantee that, given an address A of any element of the +// ring buffer, +// A_next = (A + sizeof(uptr)) & ~((1 << (N + 13)) - 1) +// is the address of the next element of that ring buffer (with wrap-around). +// And, with K = kShadowBaseAlignment, +// S = (A | ((1 << K) - 1)) + 1 +// (align up to kShadowBaseAlignment) is the start of the shadow region. +// +// These calculations are used in compiler instrumentation to update the ring +// buffer and obtain the base address of shadow using only two inputs: address +// of the current element of the ring buffer, and N (i.e. size of the ring +// buffer). Since the value of N is very limited, we pack both inputs into a +// single thread-local word as +// (1 << (N + 56)) | A +// See the implementation of class CompactRingBuffer, which is what is stored in +// said thread-local word. +// +// Note the unusual way of aligning up the address of the shadow: +// (A | ((1 << K) - 1)) + 1 +// It is only correct if A is not already equal to the shadow base address, but +// it saves 2 instructions on AArch64. + +#include "hwasan.h" +#include "hwasan_allocator.h" +#include "hwasan_flags.h" +#include "hwasan_thread.h" + +#include "sanitizer_common/sanitizer_placement_new.h" + +namespace __hwasan { + +static uptr RingBufferSize() { + uptr desired_bytes = flags()->stack_history_size * sizeof(uptr); + // FIXME: increase the limit to 8 once this bug is fixed: + // https://bugs.llvm.org/show_bug.cgi?id=39030 + for (int shift = 1; shift < 7; ++shift) { + uptr size = 4096 * (1ULL << shift); + if (size >= desired_bytes) + return size; + } + Printf("stack history size too large: %d\n", flags()->stack_history_size); + CHECK(0); + return 0; +} + +struct ThreadListHead { + Thread *list_; + + ThreadListHead() : list_(nullptr) {} + + void Push(Thread *t) { + t->next_ = list_; + list_ = t; + } + + Thread *Pop() { + Thread *t = list_; + if (t) + list_ = t->next_; + return t; + } + + void Remove(Thread *t) { + Thread **cur = &list_; + while (*cur != t) cur = &(*cur)->next_; + CHECK(*cur && "thread not found"); + *cur = (*cur)->next_; + } + + template <class CB> + void ForEach(CB cb) { + Thread *t = list_; + while (t) { + cb(t); + t = t->next_; + } + } +}; + +struct ThreadStats { + uptr n_live_threads; + uptr total_stack_size; +}; + +class HwasanThreadList { + public: + HwasanThreadList(uptr storage, uptr size) + : free_space_(storage), + free_space_end_(storage + size), + ring_buffer_size_(RingBufferSize()) {} + + Thread *CreateCurrentThread() { + Thread *t; + { + SpinMutexLock l(&list_mutex_); + t = free_list_.Pop(); + if (t) + internal_memset((void *)t, 0, sizeof(Thread) + ring_buffer_size_); + else + t = AllocThread(); + live_list_.Push(t); + } + t->Init((uptr)(t + 1), ring_buffer_size_); + AddThreadStats(t); + return t; + } + + void ReleaseThread(Thread *t) { + // FIXME: madvise away the ring buffer? + RemoveThreadStats(t); + t->Destroy(); + SpinMutexLock l(&list_mutex_); + live_list_.Remove(t); + free_list_.Push(t); + } + + Thread *GetThreadByBufferAddress(uptr p) { + uptr align = ring_buffer_size_ * 2; + return (Thread *)(RoundDownTo(p, align) - sizeof(Thread)); + } + + uptr MemoryUsedPerThread() { + uptr res = sizeof(Thread) + ring_buffer_size_; + if (auto sz = flags()->heap_history_size) + res += HeapAllocationsRingBuffer::SizeInBytes(sz); + return res; + } + + template <class CB> + void VisitAllLiveThreads(CB cb) { + SpinMutexLock l(&list_mutex_); + live_list_.ForEach(cb); + } + + void AddThreadStats(Thread *t) { + SpinMutexLock l(&stats_mutex_); + stats_.n_live_threads++; + stats_.total_stack_size += t->stack_size(); + } + + void RemoveThreadStats(Thread *t) { + SpinMutexLock l(&stats_mutex_); + stats_.n_live_threads--; + stats_.total_stack_size -= t->stack_size(); + } + + ThreadStats GetThreadStats() { + SpinMutexLock l(&stats_mutex_); + return stats_; + } + + private: + Thread *AllocThread() { + uptr align = ring_buffer_size_ * 2; + uptr ring_buffer_start = RoundUpTo(free_space_ + sizeof(Thread), align); + free_space_ = ring_buffer_start + ring_buffer_size_; + CHECK(free_space_ <= free_space_end_ && "out of thread memory"); + return (Thread *)(ring_buffer_start - sizeof(Thread)); + } + + uptr free_space_; + uptr free_space_end_; + uptr ring_buffer_size_; + + ThreadListHead free_list_; + ThreadListHead live_list_; + SpinMutex list_mutex_; + + ThreadStats stats_; + SpinMutex stats_mutex_; +}; + +void InitThreadList(uptr storage, uptr size); +HwasanThreadList &hwasanThreadList(); + +} // namespace diff --git a/contrib/compiler-rt/lib/interception/interception.h b/contrib/compiler-rt/lib/interception/interception.h index ddd6ec20979b..87b2365fd767 100644 --- a/contrib/compiler-rt/lib/interception/interception.h +++ b/contrib/compiler-rt/lib/interception/interception.h @@ -29,6 +29,7 @@ typedef __sanitizer::uptr SIZE_T; typedef __sanitizer::sptr SSIZE_T; typedef __sanitizer::sptr PTRDIFF_T; typedef __sanitizer::s64 INTMAX_T; +typedef __sanitizer::u64 UINTMAX_T; typedef __sanitizer::OFF_T OFF_T; typedef __sanitizer::OFF64_T OFF64_T; @@ -169,7 +170,7 @@ const interpose_substitution substitution_##func_name[] \ #elif !SANITIZER_MAC # define PTR_TO_REAL(x) real_##x # define REAL(x) __interception::PTR_TO_REAL(x) -# define FUNC_TYPE(x) x##_f +# define FUNC_TYPE(x) x##_type # define DECLARE_REAL(ret_type, func, ...) \ typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__); \ diff --git a/contrib/compiler-rt/lib/interception/interception_linux.h b/contrib/compiler-rt/lib/interception/interception_linux.h index 942c25609ccb..765a186e5823 100644 --- a/contrib/compiler-rt/lib/interception/interception_linux.h +++ b/contrib/compiler-rt/lib/interception/interception_linux.h @@ -38,7 +38,7 @@ void *GetFuncAddrVer(const char *func_name, const char *ver); // Android, Solaris and OpenBSD do not have dlvsym #if !SANITIZER_ANDROID && !SANITIZER_SOLARIS && !SANITIZER_OPENBSD #define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \ - (::__interception::real_##func = (func##_f)( \ + (::__interception::real_##func = (func##_type)( \ unsigned long)::__interception::GetFuncAddrVer(#func, symver)) #else #define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \ diff --git a/contrib/compiler-rt/lib/interception/interception_win.cc b/contrib/compiler-rt/lib/interception/interception_win.cc index bd4ad7274dde..cd13827e5857 100644 --- a/contrib/compiler-rt/lib/interception/interception_win.cc +++ b/contrib/compiler-rt/lib/interception/interception_win.cc @@ -223,8 +223,8 @@ static bool IsMemoryPadding(uptr address, uptr size) { return true; } -static const u8 kHintNop9Bytes[] = { - 0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 +static const u8 kHintNop8Bytes[] = { + 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 }; template<class T> @@ -239,8 +239,8 @@ static bool FunctionHasPrefix(uptr address, const T &pattern) { static bool FunctionHasPadding(uptr address, uptr size) { if (IsMemoryPadding(address - size, size)) return true; - if (size <= sizeof(kHintNop9Bytes) && - FunctionHasPrefix(address, kHintNop9Bytes)) + if (size <= sizeof(kHintNop8Bytes) && + FunctionHasPrefix(address, kHintNop8Bytes)) return true; return false; } diff --git a/contrib/compiler-rt/lib/lsan/lsan_allocator.cc b/contrib/compiler-rt/lib/lsan/lsan_allocator.cc index c58c3548002f..1b338bd5973e 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_allocator.cc +++ b/contrib/compiler-rt/lib/lsan/lsan_allocator.cc @@ -34,9 +34,6 @@ static const uptr kMaxAllowedMallocSize = 4UL << 30; #else static const uptr kMaxAllowedMallocSize = 8UL << 30; #endif -typedef LargeMmapAllocator<> SecondaryAllocator; -typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, - SecondaryAllocator> Allocator; static Allocator allocator; diff --git a/contrib/compiler-rt/lib/lsan/lsan_allocator.h b/contrib/compiler-rt/lib/lsan/lsan_allocator.h index 7c70bb6d9766..4c4e02fc0902 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_allocator.h +++ b/contrib/compiler-rt/lib/lsan/lsan_allocator.h @@ -54,19 +54,25 @@ struct ChunkMetadata { defined(__arm__) static const uptr kRegionSizeLog = 20; static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog; -typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap; +template <typename AddressSpaceView> +using ByteMapASVT = + TwoLevelByteMap<(kNumRegions >> 12), 1 << 12, AddressSpaceView>; +template <typename AddressSpaceViewTy> struct AP32 { static const uptr kSpaceBeg = 0; static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; static const uptr kMetadataSize = sizeof(ChunkMetadata); typedef __sanitizer::CompactSizeClassMap SizeClassMap; static const uptr kRegionSizeLog = __lsan::kRegionSizeLog; - typedef __lsan::ByteMap ByteMap; + using AddressSpaceView = AddressSpaceViewTy; + using ByteMap = __lsan::ByteMapASVT<AddressSpaceView>; typedef NoOpMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; -typedef SizeClassAllocator32<AP32> PrimaryAllocator; +template <typename AddressSpaceView> +using PrimaryAllocatorASVT = SizeClassAllocator32<AP32<AddressSpaceView>>; +using PrimaryAllocator = PrimaryAllocatorASVT<LocalAddressSpaceView>; #elif defined(__x86_64__) || defined(__powerpc64__) # if defined(__powerpc64__) const uptr kAllocatorSpace = 0xa0000000000ULL; @@ -75,6 +81,7 @@ const uptr kAllocatorSize = 0x20000000000ULL; // 2T. const uptr kAllocatorSpace = 0x600000000000ULL; const uptr kAllocatorSize = 0x40000000000ULL; // 4T. # endif +template <typename AddressSpaceViewTy> struct AP64 { // Allocator64 parameters. Deliberately using a short name. static const uptr kSpaceBeg = kAllocatorSpace; static const uptr kSpaceSize = kAllocatorSize; @@ -82,11 +89,30 @@ struct AP64 { // Allocator64 parameters. Deliberately using a short name. typedef DefaultSizeClassMap SizeClassMap; typedef NoOpMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; + using AddressSpaceView = AddressSpaceViewTy; }; -typedef SizeClassAllocator64<AP64> PrimaryAllocator; +template <typename AddressSpaceView> +using PrimaryAllocatorASVT = SizeClassAllocator64<AP64<AddressSpaceView>>; +using PrimaryAllocator = PrimaryAllocatorASVT<LocalAddressSpaceView>; #endif -typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; + +template <typename AddressSpaceView> +using AllocatorCacheASVT = + SizeClassAllocatorLocalCache<PrimaryAllocatorASVT<AddressSpaceView>>; +using AllocatorCache = AllocatorCacheASVT<LocalAddressSpaceView>; + +template <typename AddressSpaceView> +using SecondaryAllocatorASVT = + LargeMmapAllocator<NoOpMapUnmapCallback, DefaultLargeMmapAllocatorPtrArray, + AddressSpaceView>; + +template <typename AddressSpaceView> +using AllocatorASVT = + CombinedAllocator<PrimaryAllocatorASVT<AddressSpaceView>, + AllocatorCacheASVT<AddressSpaceView>, + SecondaryAllocatorASVT<AddressSpaceView>>; +using Allocator = AllocatorASVT<LocalAddressSpaceView>; AllocatorCache *GetAllocatorCache(); diff --git a/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc b/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc index 2508c1dbd873..a355cea96c43 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc +++ b/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_platform.h" +#include "sanitizer_common/sanitizer_libc.h" #include "lsan_common.h" #if CAN_SANITIZE_LEAKS && SANITIZER_MAC @@ -116,7 +117,8 @@ static const char *kSkippedSecNames[] = { // Scans global variables for heap pointers. void ProcessGlobalRegions(Frontier *frontier) { - for (auto name : kSkippedSecNames) CHECK(ARRAY_SIZE(name) < kMaxSegName); + for (auto name : kSkippedSecNames) + CHECK(internal_strnlen(name, kMaxSegName + 1) <= kMaxSegName); MemoryMappingLayout memory_mapping(false); InternalMmapVector<LoadedModule> modules; @@ -142,12 +144,6 @@ void ProcessGlobalRegions(Frontier *frontier) { } void ProcessPlatformSpecificAllocations(Frontier *frontier) { - mach_port_name_t port; - if (task_for_pid(mach_task_self(), internal_getpid(), &port) - != KERN_SUCCESS) { - return; - } - unsigned depth = 1; vm_size_t size = 0; vm_address_t address = 0; @@ -158,7 +154,7 @@ void ProcessPlatformSpecificAllocations(Frontier *frontier) { while (err == KERN_SUCCESS) { struct vm_region_submap_info_64 info; - err = vm_region_recurse_64(port, &address, &size, &depth, + err = vm_region_recurse_64(mach_task_self(), &address, &size, &depth, (vm_region_info_t)&info, &count); uptr end_address = address + size; diff --git a/contrib/compiler-rt/lib/lsan/lsan_interceptors.cc b/contrib/compiler-rt/lib/lsan/lsan_interceptors.cc index fde52e496164..a9bd2ba42319 100644 --- a/contrib/compiler-rt/lib/lsan/lsan_interceptors.cc +++ b/contrib/compiler-rt/lib/lsan/lsan_interceptors.cc @@ -153,7 +153,7 @@ INTERCEPTOR(struct fake_mallinfo, mallinfo, void) { #define LSAN_MAYBE_INTERCEPT_MALLINFO INTERCEPT_FUNCTION(mallinfo) INTERCEPTOR(int, mallopt, int cmd, int value) { - return -1; + return 0; } #define LSAN_MAYBE_INTERCEPT_MALLOPT INTERCEPT_FUNCTION(mallopt) #else diff --git a/contrib/compiler-rt/lib/msan/msan.cc b/contrib/compiler-rt/lib/msan/msan.cc index 06bcbdf88691..ba2d5d593303 100644 --- a/contrib/compiler-rt/lib/msan/msan.cc +++ b/contrib/compiler-rt/lib/msan/msan.cc @@ -59,6 +59,10 @@ SANITIZER_INTERFACE_ATTRIBUTE ALIGNED(16) THREADLOCAL u64 __msan_va_arg_tls[kMsanParamTlsSize / sizeof(u64)]; SANITIZER_INTERFACE_ATTRIBUTE +ALIGNED(16) +THREADLOCAL u32 __msan_va_arg_origin_tls[kMsanParamTlsSize / sizeof(u32)]; + +SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL u64 __msan_va_arg_overflow_size_tls; SANITIZER_INTERFACE_ATTRIBUTE @@ -277,6 +281,8 @@ void ScopedThreadLocalStateBackup::Restore() { internal_memset(__msan_param_tls, 0, sizeof(__msan_param_tls)); internal_memset(__msan_retval_tls, 0, sizeof(__msan_retval_tls)); internal_memset(__msan_va_arg_tls, 0, sizeof(__msan_va_arg_tls)); + internal_memset(__msan_va_arg_origin_tls, 0, + sizeof(__msan_va_arg_origin_tls)); if (__msan_get_track_origins()) { internal_memset(&__msan_retval_origin_tls, 0, diff --git a/contrib/compiler-rt/lib/msan/msan_allocator.cc b/contrib/compiler-rt/lib/msan/msan_allocator.cc index 36f0497a9d83..053ab0280397 100644 --- a/contrib/compiler-rt/lib/msan/msan_allocator.cc +++ b/contrib/compiler-rt/lib/msan/msan_allocator.cc @@ -57,7 +57,8 @@ struct MsanMapUnmapCallback { static const uptr kMetadataSize = sizeof(Metadata); typedef __sanitizer::CompactSizeClassMap SizeClassMap; static const uptr kRegionSizeLog = __msan::kRegionSizeLog; - typedef __msan::ByteMap ByteMap; + using AddressSpaceView = LocalAddressSpaceView; + using ByteMap = __msan::ByteMap; typedef MsanMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; @@ -78,6 +79,7 @@ struct MsanMapUnmapCallback { typedef DefaultSizeClassMap SizeClassMap; typedef MsanMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; + using AddressSpaceView = LocalAddressSpaceView; }; typedef SizeClassAllocator64<AP64> PrimaryAllocator; @@ -92,6 +94,7 @@ struct MsanMapUnmapCallback { typedef DefaultSizeClassMap SizeClassMap; typedef MsanMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; + using AddressSpaceView = LocalAddressSpaceView; }; typedef SizeClassAllocator64<AP64> PrimaryAllocator; @@ -107,7 +110,8 @@ struct MsanMapUnmapCallback { static const uptr kMetadataSize = sizeof(Metadata); typedef __sanitizer::CompactSizeClassMap SizeClassMap; static const uptr kRegionSizeLog = __msan::kRegionSizeLog; - typedef __msan::ByteMap ByteMap; + using AddressSpaceView = LocalAddressSpaceView; + using ByteMap = __msan::ByteMap; typedef MsanMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; diff --git a/contrib/compiler-rt/lib/msan/msan_interceptors.cc b/contrib/compiler-rt/lib/msan/msan_interceptors.cc index b3429bcf06b5..497f943a8a0e 100644 --- a/contrib/compiler-rt/lib/msan/msan_interceptors.cc +++ b/contrib/compiler-rt/lib/msan/msan_interceptors.cc @@ -34,11 +34,13 @@ #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_linux.h" #include "sanitizer_common/sanitizer_tls_get_addr.h" +#include "sanitizer_common/sanitizer_vector.h" #if SANITIZER_NETBSD #define fstat __fstat50 #define gettimeofday __gettimeofday50 #define getrusage __getrusage50 +#define tzset __tzset50 #endif #include <stdarg.h> @@ -249,11 +251,11 @@ INTERCEPTOR(uptr, malloc_usable_size, void *ptr) { // temporary! The following is equivalent on all supported platforms but // aarch64 (which uses a different register for sret value). We have a test // to confirm that. -INTERCEPTOR(void, mallinfo, __sanitizer_mallinfo *sret) { +INTERCEPTOR(void, mallinfo, __sanitizer_struct_mallinfo *sret) { #ifdef __aarch64__ uptr r8; asm volatile("mov %0,x8" : "=r" (r8)); - sret = reinterpret_cast<__sanitizer_mallinfo*>(r8); + sret = reinterpret_cast<__sanitizer_struct_mallinfo*>(r8); #endif REAL(memset)(sret, 0, sizeof(*sret)); __msan_unpoison(sret, sizeof(*sret)); @@ -265,7 +267,7 @@ INTERCEPTOR(void, mallinfo, __sanitizer_mallinfo *sret) { #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(int, mallopt, int cmd, int value) { - return -1; + return 0; } #define MSAN_MAYBE_INTERCEPT_MALLOPT INTERCEPT_FUNCTION(mallopt) #else @@ -1071,6 +1073,7 @@ extern char *tzname[2]; INTERCEPTOR(void, tzset, int fake) { ENSURE_MSAN_INITED(); + InterceptorScope interceptor_scope; REAL(tzset)(fake); if (tzname[0]) __msan_unpoison(tzname[0], REAL(strlen)(tzname[0]) + 1); @@ -1084,23 +1087,80 @@ struct MSanAtExitRecord { void *arg; }; -void MSanAtExitWrapper(void *arg) { +struct InterceptorContext { + BlockingMutex atexit_mu; + Vector<struct MSanAtExitRecord *> AtExitStack; + + InterceptorContext() + : AtExitStack() { + } +}; + +static ALIGNED(64) char interceptor_placeholder[sizeof(InterceptorContext)]; +InterceptorContext *interceptor_ctx() { + return reinterpret_cast<InterceptorContext*>(&interceptor_placeholder[0]); +} + +void MSanAtExitWrapper() { + MSanAtExitRecord *r; + { + BlockingMutexLock l(&interceptor_ctx()->atexit_mu); + + uptr element = interceptor_ctx()->AtExitStack.Size() - 1; + r = interceptor_ctx()->AtExitStack[element]; + interceptor_ctx()->AtExitStack.PopBack(); + } + + UnpoisonParam(1); + ((void(*)())r->func)(); + InternalFree(r); +} + +void MSanCxaAtExitWrapper(void *arg) { UnpoisonParam(1); MSanAtExitRecord *r = (MSanAtExitRecord *)arg; r->func(r->arg); InternalFree(r); } +static int setup_at_exit_wrapper(void(*f)(), void *arg, void *dso); + // Unpoison argument shadow for C++ module destructors. INTERCEPTOR(int, __cxa_atexit, void (*func)(void *), void *arg, void *dso_handle) { if (msan_init_is_running) return REAL(__cxa_atexit)(func, arg, dso_handle); + return setup_at_exit_wrapper((void(*)())func, arg, dso_handle); +} + +// Unpoison argument shadow for C++ module destructors. +INTERCEPTOR(int, atexit, void (*func)()) { + // Avoid calling real atexit as it is unrechable on at least on Linux. + if (msan_init_is_running) + return REAL(__cxa_atexit)((void (*)(void *a))func, 0, 0); + return setup_at_exit_wrapper((void(*)())func, 0, 0); +} + +static int setup_at_exit_wrapper(void(*f)(), void *arg, void *dso) { ENSURE_MSAN_INITED(); MSanAtExitRecord *r = (MSanAtExitRecord *)InternalAlloc(sizeof(MSanAtExitRecord)); - r->func = func; + r->func = (void(*)(void *a))f; r->arg = arg; - return REAL(__cxa_atexit)(MSanAtExitWrapper, r, dso_handle); + int res; + if (!dso) { + // NetBSD does not preserve the 2nd argument if dso is equal to 0 + // Store ctx in a local stack-like structure + + BlockingMutexLock l(&interceptor_ctx()->atexit_mu); + + res = REAL(__cxa_atexit)((void (*)(void *a))MSanAtExitWrapper, 0, 0); + if (!res) { + interceptor_ctx()->AtExitStack.PushBack(r); + } + } else { + res = REAL(__cxa_atexit)(MSanCxaAtExitWrapper, r, dso); + } + return res; } static void BeforeFork() { @@ -1520,6 +1580,9 @@ namespace __msan { void InitializeInterceptors() { static int inited = 0; CHECK_EQ(inited, 0); + + new(interceptor_ctx()) InterceptorContext(); + InitializeCommonInterceptors(); InitializeSignalInterceptors(); @@ -1629,6 +1692,7 @@ void InitializeInterceptors() { INTERCEPT_FUNCTION(pthread_join); INTERCEPT_FUNCTION(tzset); + INTERCEPT_FUNCTION(atexit); INTERCEPT_FUNCTION(__cxa_atexit); INTERCEPT_FUNCTION(shmat); INTERCEPT_FUNCTION(fork); diff --git a/contrib/compiler-rt/lib/msan/msan_linux.cc b/contrib/compiler-rt/lib/msan/msan_linux.cc index 385a650c4afc..0b0208884d2f 100644 --- a/contrib/compiler-rt/lib/msan/msan_linux.cc +++ b/contrib/compiler-rt/lib/msan/msan_linux.cc @@ -175,6 +175,51 @@ void InstallAtExitHandler() { // ---------------------- TSD ---------------- {{{1 +#if SANITIZER_NETBSD || SANITIZER_FREEBSD +// Thread Static Data cannot be used in early init on NetBSD and FreeBSD. +// Reuse the MSan TSD API for compatibility with existing code +// with an alternative implementation. + +static void (*tsd_destructor)(void *tsd) = nullptr; + +struct tsd_key { + tsd_key() : key(nullptr) {} + ~tsd_key() { + CHECK(tsd_destructor); + if (key) + (*tsd_destructor)(key); + } + MsanThread *key; +}; + +static thread_local struct tsd_key key; + +void MsanTSDInit(void (*destructor)(void *tsd)) { + CHECK(!tsd_destructor); + tsd_destructor = destructor; +} + +MsanThread *GetCurrentThread() { + CHECK(tsd_destructor); + return key.key; +} + +void SetCurrentThread(MsanThread *tsd) { + CHECK(tsd_destructor); + CHECK(tsd); + CHECK(!key.key); + key.key = tsd; +} + +void MsanTSDDtor(void *tsd) { + CHECK(tsd_destructor); + CHECK_EQ(key.key, tsd); + key.key = nullptr; + // Make sure that signal handler can not see a stale current thread pointer. + atomic_signal_fence(memory_order_seq_cst); + MsanThread::TSDDtor(tsd); +} +#else static pthread_key_t tsd_key; static bool tsd_key_inited = false; @@ -211,6 +256,7 @@ void MsanTSDDtor(void *tsd) { atomic_signal_fence(memory_order_seq_cst); MsanThread::TSDDtor(tsd); } +#endif } // namespace __msan diff --git a/contrib/compiler-rt/lib/profile/GCDAProfiling.c b/contrib/compiler-rt/lib/profile/GCDAProfiling.c index cbca365510b5..0665a680cf01 100644 --- a/contrib/compiler-rt/lib/profile/GCDAProfiling.c +++ b/contrib/compiler-rt/lib/profile/GCDAProfiling.c @@ -29,6 +29,8 @@ #include <string.h> #if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#include <windows.h> #include "WindowsMMap.h" #else #include <sys/mman.h> @@ -86,6 +88,9 @@ static uint64_t cur_buffer_size = 0; static uint64_t cur_pos = 0; static uint64_t file_size = 0; static int new_file = 0; +#if defined(_WIN32) +static HANDLE mmap_handle = NULL; +#endif static int fd = -1; typedef void (*fn_ptr)(); @@ -255,6 +260,28 @@ static int map_file() { if (file_size == 0) return -1; +#if defined(_WIN32) + HANDLE mmap_fd; + if (fd == -1) + mmap_fd = INVALID_HANDLE_VALUE; + else + mmap_fd = (HANDLE)_get_osfhandle(fd); + + mmap_handle = CreateFileMapping(mmap_fd, NULL, PAGE_READWRITE, DWORD_HI(file_size), DWORD_LO(file_size), NULL); + if (mmap_handle == NULL) { + fprintf(stderr, "profiling: %s: cannot create file mapping: %d\n", filename, + GetLastError()); + return -1; + } + + write_buffer = MapViewOfFile(mmap_handle, FILE_MAP_WRITE, 0, 0, file_size); + if (write_buffer == NULL) { + fprintf(stderr, "profiling: %s: cannot map: %d\n", filename, + GetLastError()); + CloseHandle(mmap_handle); + return -1; + } +#else write_buffer = mmap(0, file_size, PROT_READ | PROT_WRITE, MAP_FILE | MAP_SHARED, fd, 0); if (write_buffer == (void *)-1) { @@ -263,10 +290,30 @@ static int map_file() { strerror(errnum)); return -1; } +#endif + return 0; } static void unmap_file() { +#if defined(_WIN32) + if (!FlushViewOfFile(write_buffer, file_size)) { + fprintf(stderr, "profiling: %s: cannot flush mapped view: %d\n", filename, + GetLastError()); + } + + if (!UnmapViewOfFile(write_buffer)) { + fprintf(stderr, "profiling: %s: cannot unmap mapped view: %d\n", filename, + GetLastError()); + } + + if (!CloseHandle(mmap_handle)) { + fprintf(stderr, "profiling: %s: cannot close file mapping handle: %d\n", filename, + GetLastError()); + } + + mmap_handle = NULL; +#else if (msync(write_buffer, file_size, MS_SYNC) == -1) { int errnum = errno; fprintf(stderr, "profiling: %s: cannot msync: %s\n", filename, @@ -277,6 +324,8 @@ static void unmap_file() { * is written and we don't care. */ (void)munmap(write_buffer, file_size); +#endif + write_buffer = NULL; file_size = 0; } diff --git a/contrib/compiler-rt/lib/profile/InstrProfData.inc b/contrib/compiler-rt/lib/profile/InstrProfData.inc index eb4a792ce82d..454620ed997a 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfData.inc +++ b/contrib/compiler-rt/lib/profile/InstrProfData.inc @@ -308,14 +308,14 @@ typedef struct ValueProfRecord { #ifdef __cplusplus /*! - * \brief Return the number of value sites. + * Return the number of value sites. */ uint32_t getNumValueSites() const { return NumValueSites; } /*! - * \brief Read data from this record and save it to Record. + * Read data from this record and save it to Record. */ void deserializeTo(InstrProfRecord &Record, - InstrProfRecord::ValueMapType *VMap); + InstrProfSymtab *SymTab); /* * In-place byte swap: * Do byte swap for this instance. \c Old is the original order before @@ -393,7 +393,7 @@ typedef struct ValueProfData { * Read data from this data and save it to \c Record. */ void deserializeTo(InstrProfRecord &Record, - InstrProfRecord::ValueMapType *VMap); + InstrProfSymtab *SymTab); void operator delete(void *ptr) { ::operator delete(ptr); } #endif } ValueProfData; @@ -458,7 +458,7 @@ getValueProfRecordHeaderSize(uint32_t NumValueSites); #endif /*! - * \brief Return the \c ValueProfRecord header size including the + * Return the \c ValueProfRecord header size including the * padding bytes. */ INSTR_PROF_VISIBILITY INSTR_PROF_INLINE @@ -471,7 +471,7 @@ uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { } /*! - * \brief Return the total size of the value profile record including the + * Return the total size of the value profile record including the * header and the value data. */ INSTR_PROF_VISIBILITY INSTR_PROF_INLINE @@ -482,7 +482,7 @@ uint32_t getValueProfRecordSize(uint32_t NumValueSites, } /*! - * \brief Return the pointer to the start of value data array. + * Return the pointer to the start of value data array. */ INSTR_PROF_VISIBILITY INSTR_PROF_INLINE InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { @@ -491,7 +491,7 @@ InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { } /*! - * \brief Return the total number of value data for \c This record. + * Return the total number of value data for \c This record. */ INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { @@ -503,7 +503,7 @@ uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { } /*! - * \brief Use this method to advance to the next \c This \c ValueProfRecord. + * Use this method to advance to the next \c This \c ValueProfRecord. */ INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { @@ -514,7 +514,7 @@ ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { } /*! - * \brief Return the first \c ValueProfRecord instance. + * Return the first \c ValueProfRecord instance. */ INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/contrib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c index a517821a2fbd..3764df1d8079 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c +++ b/contrib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c @@ -8,7 +8,7 @@ \*===----------------------------------------------------------------------===*/ #if defined(__linux__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \ - (defined(__sun__) && defined(__svr4__)) + (defined(__sun__) && defined(__svr4__)) || defined(__NetBSD__) #include <stdlib.h> diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c b/contrib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c index a339abc7f883..7c2f14cfce16 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c +++ b/contrib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c @@ -8,7 +8,7 @@ \*===----------------------------------------------------------------------===*/ #if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__) && \ - !(defined(__sun__) && defined(__svr4__)) + !(defined(__sun__) && defined(__svr4__)) && !defined(__NetBSD__) #include <stdlib.h> diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingValue.c b/contrib/compiler-rt/lib/profile/InstrProfilingValue.c index 674a48609878..c7b01a570649 100644 --- a/contrib/compiler-rt/lib/profile/InstrProfilingValue.c +++ b/contrib/compiler-rt/lib/profile/InstrProfilingValue.c @@ -105,8 +105,7 @@ static int allocateValueProfileCounters(__llvm_profile_data *Data) { return 1; } -static ValueProfNode *allocateOneNode(__llvm_profile_data *Data, uint32_t Index, - uint64_t Value) { +static ValueProfNode *allocateOneNode(void) { ValueProfNode *Node; if (!hasStaticCounters) @@ -205,7 +204,7 @@ instrumentTargetValueImpl(uint64_t TargetValue, void *Data, return; } - CurVNode = allocateOneNode(PData, CounterIndex, TargetValue); + CurVNode = allocateOneNode(); if (!CurVNode) return; CurVNode->Value = TargetValue; diff --git a/contrib/compiler-rt/lib/profile/WindowsMMap.c b/contrib/compiler-rt/lib/profile/WindowsMMap.c index dc87a888ae7b..41cc67f41f1f 100644 --- a/contrib/compiler-rt/lib/profile/WindowsMMap.c +++ b/contrib/compiler-rt/lib/profile/WindowsMMap.c @@ -24,14 +24,6 @@ #include "InstrProfiling.h" -#ifdef __USE_FILE_OFFSET64 -# define DWORD_HI(x) (x >> 32) -# define DWORD_LO(x) ((x) & 0xffffffff) -#else -# define DWORD_HI(x) (0) -# define DWORD_LO(x) (x) -#endif - COMPILER_RT_VISIBILITY void *mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset) { diff --git a/contrib/compiler-rt/lib/profile/WindowsMMap.h b/contrib/compiler-rt/lib/profile/WindowsMMap.h index 271619aea09a..51a130b31977 100644 --- a/contrib/compiler-rt/lib/profile/WindowsMMap.h +++ b/contrib/compiler-rt/lib/profile/WindowsMMap.h @@ -12,7 +12,7 @@ #if defined(_WIN32) -#include <BaseTsd.h> +#include <basetsd.h> #include <io.h> #include <sys/types.h> @@ -45,6 +45,14 @@ #define LOCK_NB 4 /* don't block when locking */ #define LOCK_UN 8 /* unlock */ +#ifdef __USE_FILE_OFFSET64 +# define DWORD_HI(x) (x >> 32) +# define DWORD_LO(x) ((x) & 0xffffffff) +#else +# define DWORD_HI(x) (0) +# define DWORD_LO(x) (x) +#endif + void *mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset); diff --git a/contrib/compiler-rt/lib/safestack/safestack.cc b/contrib/compiler-rt/lib/safestack/safestack.cc index 8af93624b991..e68208015fb9 100644 --- a/contrib/compiler-rt/lib/safestack/safestack.cc +++ b/contrib/compiler-rt/lib/safestack/safestack.cc @@ -14,11 +14,13 @@ // //===----------------------------------------------------------------------===// +#include <errno.h> #include <limits.h> #include <pthread.h> #include <stddef.h> #include <stdint.h> #include <unistd.h> +#include <stdlib.h> #include <sys/resource.h> #include <sys/types.h> #if !defined(__NetBSD__) @@ -115,14 +117,6 @@ static inline void unsafe_stack_setup(void *start, size_t size, size_t guard) { unsafe_stack_guard = guard; } -static void unsafe_stack_free() { - if (unsafe_stack_start) { - UnmapOrDie((char *)unsafe_stack_start - unsafe_stack_guard, - unsafe_stack_size + unsafe_stack_guard); - } - unsafe_stack_start = nullptr; -} - /// Thread data for the cleanup handler static pthread_key_t thread_cleanup_key; @@ -149,26 +143,73 @@ static void *thread_start(void *arg) { tinfo->unsafe_stack_guard); // Make sure out thread-specific destructor will be called - // FIXME: we can do this only any other specific key is set by - // intercepting the pthread_setspecific function itself pthread_setspecific(thread_cleanup_key, (void *)1); return start_routine(start_routine_arg); } -/// Thread-specific data destructor +/// Linked list used to store exiting threads stack/thread information. +struct thread_stack_ll { + struct thread_stack_ll *next; + void *stack_base; + size_t size; + pid_t pid; + tid_t tid; +}; + +/// Linked list of unsafe stacks for threads that are exiting. We delay +/// unmapping them until the thread exits. +static thread_stack_ll *thread_stacks = nullptr; +static pthread_mutex_t thread_stacks_mutex = PTHREAD_MUTEX_INITIALIZER; + +/// Thread-specific data destructor. We want to free the unsafe stack only after +/// this thread is terminated. libc can call functions in safestack-instrumented +/// code (like free) after thread-specific data destructors have run. static void thread_cleanup_handler(void *_iter) { - // We want to free the unsafe stack only after all other destructors - // have already run. We force this function to be called multiple times. - // User destructors that might run more then PTHREAD_DESTRUCTOR_ITERATIONS-1 - // times might still end up executing after the unsafe stack is deallocated. - size_t iter = (size_t)_iter; - if (iter < PTHREAD_DESTRUCTOR_ITERATIONS) { - pthread_setspecific(thread_cleanup_key, (void *)(iter + 1)); - } else { - // This is the last iteration - unsafe_stack_free(); + CHECK_NE(unsafe_stack_start, nullptr); + pthread_setspecific(thread_cleanup_key, NULL); + + pthread_mutex_lock(&thread_stacks_mutex); + // Temporary list to hold the previous threads stacks so we don't hold the + // thread_stacks_mutex for long. + thread_stack_ll *temp_stacks = thread_stacks; + thread_stacks = nullptr; + pthread_mutex_unlock(&thread_stacks_mutex); + + pid_t pid = getpid(); + tid_t tid = GetTid(); + + // Free stacks for dead threads + thread_stack_ll **stackp = &temp_stacks; + while (*stackp) { + thread_stack_ll *stack = *stackp; + int error; + if (stack->pid != pid || + (internal_iserror(TgKill(stack->pid, stack->tid, 0), &error) && + error == ESRCH)) { + UnmapOrDie(stack->stack_base, stack->size); + *stackp = stack->next; + free(stack); + } else + stackp = &stack->next; } + + thread_stack_ll *cur_stack = + (thread_stack_ll *)malloc(sizeof(thread_stack_ll)); + cur_stack->stack_base = (char *)unsafe_stack_start - unsafe_stack_guard; + cur_stack->size = unsafe_stack_size + unsafe_stack_guard; + cur_stack->pid = pid; + cur_stack->tid = tid; + + pthread_mutex_lock(&thread_stacks_mutex); + // Merge thread_stacks with the current thread's stack and any remaining + // temp_stacks + *stackp = thread_stacks; + cur_stack->next = temp_stacks; + thread_stacks = cur_stack; + pthread_mutex_unlock(&thread_stacks_mutex); + + unsafe_stack_start = nullptr; } static void EnsureInterceptorsInitialized(); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h index 9655a2264f34..88017160acf9 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h @@ -14,13 +14,15 @@ #ifndef SANITIZER_ALLOCATOR_H #define SANITIZER_ALLOCATOR_H -#include "sanitizer_internal_defs.h" #include "sanitizer_common.h" +#include "sanitizer_internal_defs.h" +#include "sanitizer_lfstack.h" #include "sanitizer_libc.h" #include "sanitizer_list.h" +#include "sanitizer_local_address_space_view.h" #include "sanitizer_mutex.h" -#include "sanitizer_lfstack.h" #include "sanitizer_procmaps.h" +#include "sanitizer_type_traits.h" namespace __sanitizer { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_bytemap.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_bytemap.h index 7df3e4097bf2..ef26941fe609 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_bytemap.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_bytemap.h @@ -15,9 +15,10 @@ #endif // Maps integers in rage [0, kSize) to u8 values. -template<u64 kSize> +template <u64 kSize, typename AddressSpaceViewTy = LocalAddressSpaceView> class FlatByteMap { public: + using AddressSpaceView = AddressSpaceViewTy; void Init() { internal_memset(map_, 0, sizeof(map_)); } @@ -41,9 +42,12 @@ class FlatByteMap { // to kSize2-byte arrays. The secondary arrays are mmaped on demand. // Each value is initially zero and can be set to something else only once. // Setting and getting values from multiple threads is safe w/o extra locking. -template <u64 kSize1, u64 kSize2, class MapUnmapCallback = NoOpMapUnmapCallback> +template <u64 kSize1, u64 kSize2, + typename AddressSpaceViewTy = LocalAddressSpaceView, + class MapUnmapCallback = NoOpMapUnmapCallback> class TwoLevelByteMap { public: + using AddressSpaceView = AddressSpaceViewTy; void Init() { internal_memset(map1_, 0, sizeof(map1_)); mu_.Init(); @@ -73,7 +77,8 @@ class TwoLevelByteMap { CHECK_LT(idx, kSize1 * kSize2); u8 *map2 = Get(idx / kSize2); if (!map2) return 0; - return map2[idx % kSize2]; + auto value_ptr = AddressSpaceView::Load(&map2[idx % kSize2]); + return *value_ptr; } private: diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h index 1f874d60b92b..fcc4469c98cf 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h @@ -21,19 +21,28 @@ // PrimaryAllocator is used via a local AllocatorCache. // SecondaryAllocator can allocate anything, but is not efficient. template <class PrimaryAllocator, class AllocatorCache, - class SecondaryAllocator> // NOLINT + class SecondaryAllocator, + typename AddressSpaceViewTy = LocalAddressSpaceView> // NOLINT class CombinedAllocator { public: + using AddressSpaceView = AddressSpaceViewTy; + static_assert(is_same<AddressSpaceView, + typename PrimaryAllocator::AddressSpaceView>::value, + "PrimaryAllocator is using wrong AddressSpaceView"); + static_assert(is_same<AddressSpaceView, + typename SecondaryAllocator::AddressSpaceView>::value, + "SecondaryAllocator is using wrong AddressSpaceView"); + void InitLinkerInitialized(s32 release_to_os_interval_ms) { + stats_.InitLinkerInitialized(); primary_.Init(release_to_os_interval_ms); secondary_.InitLinkerInitialized(); - stats_.InitLinkerInitialized(); } void Init(s32 release_to_os_interval_ms) { + stats_.Init(); primary_.Init(release_to_os_interval_ms); secondary_.Init(); - stats_.Init(); } void *Allocate(AllocatorCache *cache, uptr size, uptr alignment) { @@ -194,4 +203,3 @@ class CombinedAllocator { SecondaryAllocator secondary_; AllocatorGlobalStats stats_; }; - diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h index c0c03d3f4345..30fc7042b6b5 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h @@ -37,7 +37,8 @@ struct AP32 { static const uptr kMetadataSize = 0; typedef InternalSizeClassMap SizeClassMap; static const uptr kRegionSizeLog = kInternalAllocatorRegionSizeLog; - typedef __sanitizer::ByteMap ByteMap; + using AddressSpaceView = LocalAddressSpaceView; + using ByteMap = __sanitizer::ByteMap; typedef NoOpMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h index 67970e95b31e..abaac3d1a70b 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h @@ -48,6 +48,7 @@ struct SizeClassAllocator32FlagMasks { // Bit masks. template <class Params> class SizeClassAllocator32 { public: + using AddressSpaceView = typename Params::AddressSpaceView; static const uptr kSpaceBeg = Params::kSpaceBeg; static const u64 kSpaceSize = Params::kSpaceSize; static const uptr kMetadataSize = Params::kMetadataSize; @@ -56,6 +57,10 @@ class SizeClassAllocator32 { typedef typename Params::ByteMap ByteMap; typedef typename Params::MapUnmapCallback MapUnmapCallback; + static_assert( + is_same<typename ByteMap::AddressSpaceView, AddressSpaceView>::value, + "AddressSpaceView type mismatch"); + static const bool kRandomShuffleChunks = Params::kFlags & SizeClassAllocator32FlagMasks::kRandomShuffleChunks; static const bool kUseSeparateSizeClassForBatch = Params::kFlags & diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h index 6acb4f8bc56d..b063bf0d302f 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h @@ -46,6 +46,7 @@ struct SizeClassAllocator64FlagMasks { // Bit masks. template <class Params> class SizeClassAllocator64 { public: + using AddressSpaceView = typename Params::AddressSpaceView; static const uptr kSpaceBeg = Params::kSpaceBeg; static const uptr kSpaceSize = Params::kSpaceSize; static const uptr kMetadataSize = Params::kMetadataSize; @@ -294,8 +295,10 @@ class SizeClassAllocator64 { RegionInfo *region = GetRegionInfo(class_id); uptr chunk_size = ClassIdToSize(class_id); uptr region_beg = SpaceBeg() + class_id * kRegionSize; + uptr region_allocated_user_size = + AddressSpaceView::Load(region)->allocated_user; for (uptr chunk = region_beg; - chunk < region_beg + region->allocated_user; + chunk < region_beg + region_allocated_user_size; chunk += chunk_size) { // Too slow: CHECK_EQ((void *)chunk, GetBlockBegin((void *)chunk)); callback(chunk, arg); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_secondary.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_secondary.h index ab680b5e2d11..0c8505c34c87 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_secondary.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_secondary.h @@ -68,9 +68,11 @@ typedef LargeMmapAllocatorPtrArrayDynamic DefaultLargeMmapAllocatorPtrArray; // The main purpose of this allocator is to cover large and rare allocation // sizes not covered by more efficient allocators (e.g. SizeClassAllocator64). template <class MapUnmapCallback = NoOpMapUnmapCallback, - class PtrArrayT = DefaultLargeMmapAllocatorPtrArray> + class PtrArrayT = DefaultLargeMmapAllocatorPtrArray, + class AddressSpaceViewTy = LocalAddressSpaceView> class LargeMmapAllocator { public: + using AddressSpaceView = AddressSpaceViewTy; void InitLinkerInitialized() { page_size_ = GetPageSizeCached(); chunks_ = reinterpret_cast<Header**>(ptr_array_.Init()); @@ -202,9 +204,10 @@ class LargeMmapAllocator { void EnsureSortedChunks() { if (chunks_sorted_) return; - Sort(reinterpret_cast<uptr *>(chunks_), n_chunks_); + Header **chunks = AddressSpaceView::LoadWritable(chunks_, n_chunks_); + Sort(reinterpret_cast<uptr *>(chunks), n_chunks_); for (uptr i = 0; i < n_chunks_; i++) - chunks_[i]->chunk_idx = i; + AddressSpaceView::LoadWritable(chunks[i])->chunk_idx = i; chunks_sorted_ = true; } @@ -272,12 +275,13 @@ class LargeMmapAllocator { // The allocator must be locked when calling this function. void ForEachChunk(ForEachChunkCallback callback, void *arg) { EnsureSortedChunks(); // Avoid doing the sort while iterating. + const Header *const *chunks = AddressSpaceView::Load(chunks_, n_chunks_); for (uptr i = 0; i < n_chunks_; i++) { - auto t = chunks_[i]; + const Header *t = chunks[i]; callback(reinterpret_cast<uptr>(GetUser(t)), arg); // Consistency check: verify that the array did not change. - CHECK_EQ(chunks_[i], t); - CHECK_EQ(chunks_[i]->chunk_idx, i); + CHECK_EQ(chunks[i], t); + CHECK_EQ(AddressSpaceView::Load(chunks[i])->chunk_idx, i); } } @@ -297,7 +301,7 @@ class LargeMmapAllocator { return GetHeader(reinterpret_cast<uptr>(p)); } - void *GetUser(Header *h) { + void *GetUser(const Header *h) { CHECK(IsAligned((uptr)h, page_size_)); return reinterpret_cast<void*>(reinterpret_cast<uptr>(h) + page_size_); } @@ -316,4 +320,3 @@ class LargeMmapAllocator { } stats; StaticSpinMutex mutex_; }; - diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_size_class_map.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_size_class_map.h index 77ab4fb544a2..1c05fb8ff06b 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_size_class_map.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_size_class_map.h @@ -232,3 +232,11 @@ class SizeClassMap { typedef SizeClassMap<3, 4, 8, 17, 128, 16> DefaultSizeClassMap; typedef SizeClassMap<3, 4, 8, 17, 64, 14> CompactSizeClassMap; typedef SizeClassMap<2, 5, 9, 16, 64, 14> VeryCompactSizeClassMap; + +// The following SizeClassMap only holds a way small number of cached entries, +// allowing for denser per-class arrays, smaller memory footprint and usually +// better performances in threaded environments. +typedef SizeClassMap<3, 4, 8, 17, 8, 10> DenseSizeClassMap; +// Similar to VeryCompact map above, this one has a small number of different +// size classes, and also reduced thread-local caches. +typedef SizeClassMap<2, 5, 9, 16, 8, 10> VeryDenseSizeClassMap; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_x86.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_x86.h index 38feb29287f1..195533ea2320 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_x86.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_x86.h @@ -61,8 +61,7 @@ INLINE typename T::Type atomic_load( "emms;" // Empty mmx state/Reset FP regs : "=m" (v) : "m" (a->val_dont_use) - : // mark the FP stack and mmx registers as clobbered - "st", "st(1)", "st(2)", "st(3)", "st(4)", "st(5)", "st(6)", "st(7)", + : // mark the mmx registers as clobbered #ifdef __MMX__ "mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7", #endif // #ifdef __MMX__ @@ -100,8 +99,7 @@ INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { "emms;" // Empty mmx state/Reset FP regs : "=m" (a->val_dont_use) : "m" (v) - : // mark the FP stack and mmx registers as clobbered - "st", "st(1)", "st(2)", "st(3)", "st(4)", "st(5)", "st(6)", "st(7)", + : // mark the mmx registers as clobbered #ifdef __MMX__ "mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7", #endif // #ifdef __MMX__ diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc index 7d72b0cfe4ce..6868961902ce 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc @@ -339,11 +339,6 @@ int __sanitizer_acquire_crash_state() { } SANITIZER_INTERFACE_ATTRIBUTE -void __sanitizer_set_death_callback(void (*callback)(void)) { - SetUserDieCallback(callback); -} - -SANITIZER_INTERFACE_ATTRIBUTE int __sanitizer_install_malloc_and_free_hooks(void (*malloc_hook)(const void *, uptr), void (*free_hook)(const void *)) { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h index 3b999edfbe58..d0aebd994129 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h @@ -61,6 +61,15 @@ INLINE int Verbosity() { return atomic_load(¤t_verbosity, memory_order_relaxed); } +#if SANITIZER_ANDROID +INLINE uptr GetPageSize() { +// Android post-M sysconf(_SC_PAGESIZE) crashes if called from .preinit_array. + return 4096; +} +INLINE uptr GetPageSizeCached() { + return 4096; +} +#else uptr GetPageSize(); extern uptr PageSizeCached; INLINE uptr GetPageSizeCached() { @@ -68,11 +77,13 @@ INLINE uptr GetPageSizeCached() { PageSizeCached = GetPageSize(); return PageSizeCached; } +#endif uptr GetMmapGranularity(); uptr GetMaxVirtualAddress(); uptr GetMaxUserVirtualAddress(); // Threads tid_t GetTid(); +int TgKill(pid_t pid, tid_t tid, int sig); uptr GetThreadSelf(); void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, uptr *stack_bottom); @@ -222,7 +233,9 @@ bool SetEnv(const char *name, const char *value); u32 GetUid(); void ReExec(); void CheckASLR(); +void CheckMPROTECT(); char **GetArgv(); +char **GetEnviron(); void PrintCmdline(); bool StackSizeIsUnlimited(); uptr GetStackSizeLimitInBytes(); @@ -896,6 +909,7 @@ struct SignalContext { bool IsMemoryAccess() const; }; +void InitializePlatformEarly(); void MaybeReexec(); template <typename Fn> diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc index 801d6a0bdeac..50f7837747e6 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc @@ -77,7 +77,15 @@ #define ctime __ctime50 #define ctime_r __ctime_r50 #define devname __devname50 +#define fgetpos __fgetpos50 +#define fsetpos __fsetpos50 +#define fts_children __fts_children60 +#define fts_close __fts_close60 +#define fts_open __fts_open60 +#define fts_read __fts_read60 +#define fts_set __fts_set60 #define getitimer __getitimer50 +#define getmntinfo __getmntinfo13 #define getpwent __getpwent50 #define getpwnam __getpwnam50 #define getpwnam_r __getpwnam_r50 @@ -87,6 +95,7 @@ #define getutxent __getutxent50 #define getutxid __getutxid50 #define getutxline __getutxline50 +#define pututxline __pututxline50 #define glob __glob30 #define gmtime __gmtime50 #define gmtime_r __gmtime_r50 @@ -109,6 +118,7 @@ #define stat __stat50 #define time __time50 #define times __times13 +#define unvis __unvis50 #define wait3 __wait350 #define wait4 __wait450 extern const unsigned short *_ctype_tab_; @@ -1810,7 +1820,10 @@ INTERCEPTOR(int, ioctl, int d, unsigned long request, ...) { #if SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS || \ SANITIZER_INTERCEPT_GETPWENT || SANITIZER_INTERCEPT_FGETPWENT || \ - SANITIZER_INTERCEPT_GETPWENT_R || SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS + SANITIZER_INTERCEPT_GETPWENT_R || \ + SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS || \ + SANITIZER_INTERCEPT_FGETPWENT_R || \ + SANITIZER_INTERCEPT_FGETGRENT_R static void unpoison_passwd(void *ctx, __sanitizer_passwd *pwd) { if (pwd) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd, sizeof(*pwd)); @@ -2034,36 +2047,51 @@ INTERCEPTOR(int, getpwent_r, __sanitizer_passwd *pwbuf, char *buf, if (pwbufp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwbufp, sizeof(*pwbufp)); return res; } -INTERCEPTOR(int, fgetpwent_r, void *fp, __sanitizer_passwd *pwbuf, char *buf, - SIZE_T buflen, __sanitizer_passwd **pwbufp) { +INTERCEPTOR(int, getgrent_r, __sanitizer_group *pwbuf, char *buf, SIZE_T buflen, + __sanitizer_group **pwbufp) { void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, fgetpwent_r, fp, pwbuf, buf, buflen, pwbufp); + COMMON_INTERCEPTOR_ENTER(ctx, getgrent_r, pwbuf, buf, buflen, pwbufp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See // https://github.com/google/sanitizers/issues/321. - int res = REAL(fgetpwent_r)(fp, pwbuf, buf, buflen, pwbufp); + int res = REAL(getgrent_r)(pwbuf, buf, buflen, pwbufp); if (!res) { - if (pwbufp && *pwbufp) unpoison_passwd(ctx, *pwbufp); + if (pwbufp && *pwbufp) unpoison_group(ctx, *pwbufp); COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, buflen); } if (pwbufp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwbufp, sizeof(*pwbufp)); return res; } -INTERCEPTOR(int, getgrent_r, __sanitizer_group *pwbuf, char *buf, SIZE_T buflen, - __sanitizer_group **pwbufp) { +#define INIT_GETPWENT_R \ + COMMON_INTERCEPT_FUNCTION(getpwent_r); \ + COMMON_INTERCEPT_FUNCTION(getgrent_r); +#else +#define INIT_GETPWENT_R +#endif + +#if SANITIZER_INTERCEPT_FGETPWENT_R +INTERCEPTOR(int, fgetpwent_r, void *fp, __sanitizer_passwd *pwbuf, char *buf, + SIZE_T buflen, __sanitizer_passwd **pwbufp) { void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, getgrent_r, pwbuf, buf, buflen, pwbufp); + COMMON_INTERCEPTOR_ENTER(ctx, fgetpwent_r, fp, pwbuf, buf, buflen, pwbufp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See // https://github.com/google/sanitizers/issues/321. - int res = REAL(getgrent_r)(pwbuf, buf, buflen, pwbufp); + int res = REAL(fgetpwent_r)(fp, pwbuf, buf, buflen, pwbufp); if (!res) { - if (pwbufp && *pwbufp) unpoison_group(ctx, *pwbufp); + if (pwbufp && *pwbufp) unpoison_passwd(ctx, *pwbufp); COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, buflen); } if (pwbufp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwbufp, sizeof(*pwbufp)); return res; } +#define INIT_FGETPWENT_R \ + COMMON_INTERCEPT_FUNCTION(fgetpwent_r); +#else +#define INIT_FGETPWENT_R +#endif + +#if SANITIZER_INTERCEPT_FGETGRENT_R INTERCEPTOR(int, fgetgrent_r, void *fp, __sanitizer_group *pwbuf, char *buf, SIZE_T buflen, __sanitizer_group **pwbufp) { void *ctx; @@ -2079,13 +2107,10 @@ INTERCEPTOR(int, fgetgrent_r, void *fp, __sanitizer_group *pwbuf, char *buf, if (pwbufp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwbufp, sizeof(*pwbufp)); return res; } -#define INIT_GETPWENT_R \ - COMMON_INTERCEPT_FUNCTION(getpwent_r); \ - COMMON_INTERCEPT_FUNCTION(fgetpwent_r); \ - COMMON_INTERCEPT_FUNCTION(getgrent_r); \ +#define INIT_FGETGRENT_R \ COMMON_INTERCEPT_FUNCTION(fgetgrent_r); #else -#define INIT_GETPWENT_R +#define INIT_FGETGRENT_R #endif #if SANITIZER_INTERCEPT_SETPWENT @@ -2149,6 +2174,8 @@ INTERCEPTOR(int, clock_gettime, u32 clk_id, void *tp) { namespace __sanitizer { extern "C" { int real_clock_gettime(u32 clk_id, void *tp) { + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) + return internal_clock_gettime(clk_id, tp); return REAL(clock_gettime)(clk_id, tp); } } // extern "C" @@ -3345,14 +3372,14 @@ INTERCEPTOR(INTMAX_T, strtoimax, const char *nptr, char **endptr, int base) { return res; } -INTERCEPTOR(INTMAX_T, strtoumax, const char *nptr, char **endptr, int base) { +INTERCEPTOR(UINTMAX_T, strtoumax, const char *nptr, char **endptr, int base) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, strtoumax, nptr, endptr, base); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See // https://github.com/google/sanitizers/issues/321. char *real_endptr; - INTMAX_T res = REAL(strtoumax)(nptr, &real_endptr, base); + UINTMAX_T res = REAL(strtoumax)(nptr, &real_endptr, base); StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base); return res; } @@ -4254,11 +4281,16 @@ INTERCEPTOR(int, statvfs, char *path, void *buf) { INTERCEPTOR(int, fstatvfs, int fd, void *buf) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, fstatvfs, fd, buf); + COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See // https://github.com/google/sanitizers/issues/321. int res = REAL(fstatvfs)(fd, buf); - if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs_sz); + if (!res) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs_sz); + if (fd >= 0) + COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd); + } return res; } #define INIT_STATVFS \ @@ -4819,6 +4851,14 @@ INTERCEPTOR(float, remquof, float x, float y, int *quo) { if (quo) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, quo, sizeof(*quo)); return res; } +#define INIT_REMQUO \ + COMMON_INTERCEPT_FUNCTION(remquo); \ + COMMON_INTERCEPT_FUNCTION(remquof); +#else +#define INIT_REMQUO +#endif + +#if SANITIZER_INTERCEPT_REMQUOL INTERCEPTOR(long double, remquol, long double x, long double y, int *quo) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, remquol, x, y, quo); @@ -4829,12 +4869,10 @@ INTERCEPTOR(long double, remquol, long double x, long double y, int *quo) { if (quo) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, quo, sizeof(*quo)); return res; } -#define INIT_REMQUO \ - COMMON_INTERCEPT_FUNCTION(remquo); \ - COMMON_INTERCEPT_FUNCTION(remquof); \ +#define INIT_REMQUOL \ COMMON_INTERCEPT_FUNCTION_LDBL(remquol); #else -#define INIT_REMQUO +#define INIT_REMQUOL #endif #if SANITIZER_INTERCEPT_LGAMMA @@ -4853,6 +4891,14 @@ INTERCEPTOR(float, lgammaf, float x) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, &signgam, sizeof(signgam)); return res; } +#define INIT_LGAMMA \ + COMMON_INTERCEPT_FUNCTION(lgamma); \ + COMMON_INTERCEPT_FUNCTION(lgammaf); +#else +#define INIT_LGAMMA +#endif + +#if SANITIZER_INTERCEPT_LGAMMAL INTERCEPTOR(long double, lgammal, long double x) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, lgammal, x); @@ -4860,12 +4906,10 @@ INTERCEPTOR(long double, lgammal, long double x) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, &signgam, sizeof(signgam)); return res; } -#define INIT_LGAMMA \ - COMMON_INTERCEPT_FUNCTION(lgamma); \ - COMMON_INTERCEPT_FUNCTION(lgammaf); \ +#define INIT_LGAMMAL \ COMMON_INTERCEPT_FUNCTION_LDBL(lgammal); #else -#define INIT_LGAMMA +#define INIT_LGAMMAL #endif #if SANITIZER_INTERCEPT_LGAMMA_R @@ -5651,9 +5695,15 @@ INTERCEPTOR(void *, tsearch, void *key, void **rootp, void unpoison_file(__sanitizer_FILE *fp) { #if SANITIZER_HAS_STRUCT_FILE COMMON_INTERCEPTOR_INITIALIZE_RANGE(fp, sizeof(*fp)); +#if SANITIZER_NETBSD + if (fp->_bf._base && fp->_bf._size > 0) + COMMON_INTERCEPTOR_INITIALIZE_RANGE(fp->_bf._base, + fp->_bf._size); +#else if (fp->_IO_read_base && fp->_IO_read_base < fp->_IO_read_end) COMMON_INTERCEPTOR_INITIALIZE_RANGE(fp->_IO_read_base, fp->_IO_read_end - fp->_IO_read_base); +#endif #endif // SANITIZER_HAS_STRUCT_FILE } #endif @@ -6523,10 +6573,21 @@ INTERCEPTOR(void *, getutxline, void *ut) { COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, __sanitizer::struct_utmpx_sz); return res; } +INTERCEPTOR(void *, pututxline, const void *ut) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, pututxline, ut); + if (ut) + COMMON_INTERCEPTOR_READ_RANGE(ctx, ut, __sanitizer::struct_utmpx_sz); + void *res = REAL(pututxline)(ut); + if (res) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, __sanitizer::struct_utmpx_sz); + return res; +} #define INIT_UTMPX \ COMMON_INTERCEPT_FUNCTION(getutxent); \ COMMON_INTERCEPT_FUNCTION(getutxid); \ - COMMON_INTERCEPT_FUNCTION(getutxline); + COMMON_INTERCEPT_FUNCTION(getutxline); \ + COMMON_INTERCEPT_FUNCTION(pututxline); #else #define INIT_UTMPX #endif @@ -7018,12 +7079,19 @@ INTERCEPTOR(char *, devname, u64 dev, u32 type) { #endif #if SANITIZER_INTERCEPT_DEVNAME_R -INTERCEPTOR(int, devname_r, u64 dev, u32 type, char *path, uptr len) { +#if SANITIZER_NETBSD +#define DEVNAME_R_RETTYPE int +#define DEVNAME_R_SUCCESS(x) (!(x)) +#else +#define DEVNAME_R_RETTYPE char* +#define DEVNAME_R_SUCCESS(x) (x) +#endif +INTERCEPTOR(DEVNAME_R_RETTYPE, devname_r, u64 dev, u32 type, char *path, + uptr len) { void *ctx; - int res; COMMON_INTERCEPTOR_ENTER(ctx, devname_r, dev, type, path, len); - res = REAL(devname_r)(dev, type, path, len); - if (!res) + DEVNAME_R_RETTYPE res = REAL(devname_r)(dev, type, path, len); + if (DEVNAME_R_SUCCESS(res)) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, path, REAL(strlen)(path) + 1); return res; } @@ -7235,9 +7303,2174 @@ INTERCEPTOR(struct __sanitizer_netent *, getnetbyaddr, u32 net, int type) { #define INIT_NETENT #endif +#if SANITIZER_INTERCEPT_GETMNTINFO +INTERCEPTOR(int, getmntinfo, void **mntbufp, int flags) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, getmntinfo, mntbufp, flags); + int cnt = REAL(getmntinfo)(mntbufp, flags); + if (cnt > 0 && mntbufp) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mntbufp, sizeof(void *)); + if (*mntbufp) +#if SANITIZER_NETBSD + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *mntbufp, cnt * struct_statvfs_sz); +#else + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *mntbufp, cnt * struct_statfs_sz); +#endif + } + return cnt; +} +#define INIT_GETMNTINFO COMMON_INTERCEPT_FUNCTION(getmntinfo) +#else +#define INIT_GETMNTINFO +#endif + +#if SANITIZER_INTERCEPT_MI_VECTOR_HASH +INTERCEPTOR(void, mi_vector_hash, const void *key, SIZE_T len, u32 seed, + u32 hashes[3]) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, mi_vector_hash, key, len, seed, hashes); + if (key) + COMMON_INTERCEPTOR_READ_RANGE(ctx, key, len); + REAL(mi_vector_hash)(key, len, seed, hashes); + if (hashes) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, hashes, sizeof(hashes[0]) * 3); +} +#define INIT_MI_VECTOR_HASH COMMON_INTERCEPT_FUNCTION(mi_vector_hash) +#else +#define INIT_MI_VECTOR_HASH +#endif + +#if SANITIZER_INTERCEPT_SETVBUF +INTERCEPTOR(int, setvbuf, __sanitizer_FILE *stream, char *buf, int mode, + SIZE_T size) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, setvbuf, stream, buf, mode, size); + int ret = REAL(setvbuf)(stream, buf, mode, size); + if (buf) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, size); + if (stream) + unpoison_file(stream); + return ret; +} + +INTERCEPTOR(void, setbuf, __sanitizer_FILE *stream, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, setbuf, stream, buf); + REAL(setbuf)(stream, buf); + if (buf) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer_bufsiz); + } + if (stream) + unpoison_file(stream); +} + +INTERCEPTOR(void, setbuffer, __sanitizer_FILE *stream, char *buf, int mode) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, setbuffer, stream, buf, mode); + REAL(setbuffer)(stream, buf, mode); + if (buf) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer_bufsiz); + } + if (stream) + unpoison_file(stream); +} + +INTERCEPTOR(void, setlinebuf, __sanitizer_FILE *stream) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, setlinebuf, stream); + REAL(setlinebuf)(stream); + if (stream) + unpoison_file(stream); +} +#define INIT_SETVBUF COMMON_INTERCEPT_FUNCTION(setvbuf); \ + COMMON_INTERCEPT_FUNCTION(setbuf); \ + COMMON_INTERCEPT_FUNCTION(setbuffer); \ + COMMON_INTERCEPT_FUNCTION(setlinebuf) +#else +#define INIT_SETVBUF +#endif + +#if SANITIZER_INTERCEPT_GETVFSSTAT +INTERCEPTOR(int, getvfsstat, void *buf, SIZE_T bufsize, int flags) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, getvfsstat, buf, bufsize, flags); + int ret = REAL(getvfsstat)(buf, bufsize, flags); + if (buf && ret > 0) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, ret * struct_statvfs_sz); + return ret; +} +#define INIT_GETVFSSTAT COMMON_INTERCEPT_FUNCTION(getvfsstat) +#else +#define INIT_GETVFSSTAT +#endif + +#if SANITIZER_INTERCEPT_REGEX +INTERCEPTOR(int, regcomp, void *preg, const char *pattern, int cflags) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, regcomp, preg, pattern, cflags); + if (pattern) + COMMON_INTERCEPTOR_READ_RANGE(ctx, pattern, REAL(strlen)(pattern) + 1); + int res = REAL(regcomp)(preg, pattern, cflags); + if (!res) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, preg, struct_regex_sz); + return res; +} +INTERCEPTOR(int, regexec, const void *preg, const char *string, SIZE_T nmatch, + struct __sanitizer_regmatch *pmatch[], int eflags) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, regexec, preg, string, nmatch, pmatch, eflags); + if (preg) + COMMON_INTERCEPTOR_READ_RANGE(ctx, preg, struct_regex_sz); + if (string) + COMMON_INTERCEPTOR_READ_RANGE(ctx, string, REAL(strlen)(string) + 1); + int res = REAL(regexec)(preg, string, nmatch, pmatch, eflags); + if (!res && pmatch) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pmatch, nmatch * struct_regmatch_sz); + return res; +} +INTERCEPTOR(SIZE_T, regerror, int errcode, const void *preg, char *errbuf, + SIZE_T errbuf_size) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, regerror, errcode, preg, errbuf, errbuf_size); + if (preg) + COMMON_INTERCEPTOR_READ_RANGE(ctx, preg, struct_regex_sz); + SIZE_T res = REAL(regerror)(errcode, preg, errbuf, errbuf_size); + if (errbuf) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, errbuf, REAL(strlen)(errbuf) + 1); + return res; +} +INTERCEPTOR(void, regfree, const void *preg) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, regfree, preg); + if (preg) + COMMON_INTERCEPTOR_READ_RANGE(ctx, preg, struct_regex_sz); + REAL(regfree)(preg); +} +#define INIT_REGEX \ + COMMON_INTERCEPT_FUNCTION(regcomp); \ + COMMON_INTERCEPT_FUNCTION(regexec); \ + COMMON_INTERCEPT_FUNCTION(regerror); \ + COMMON_INTERCEPT_FUNCTION(regfree); +#else +#define INIT_REGEX +#endif + +#if SANITIZER_INTERCEPT_REGEXSUB +INTERCEPTOR(SSIZE_T, regnsub, char *buf, SIZE_T bufsiz, const char *sub, + const struct __sanitizer_regmatch *rm, const char *str) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, regnsub, buf, bufsiz, sub, rm, str); + if (sub) + COMMON_INTERCEPTOR_READ_RANGE(ctx, sub, REAL(strlen)(sub) + 1); + // The implementation demands and hardcodes 10 elements + if (rm) + COMMON_INTERCEPTOR_READ_RANGE(ctx, rm, 10 * struct_regmatch_sz); + if (str) + COMMON_INTERCEPTOR_READ_RANGE(ctx, str, REAL(strlen)(str) + 1); + SSIZE_T res = REAL(regnsub)(buf, bufsiz, sub, rm, str); + if (res > 0 && buf) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, REAL(strlen)(buf) + 1); + return res; +} +INTERCEPTOR(SSIZE_T, regasub, char **buf, const char *sub, + const struct __sanitizer_regmatch *rm, const char *sstr) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, regasub, buf, sub, rm, sstr); + if (sub) + COMMON_INTERCEPTOR_READ_RANGE(ctx, sub, REAL(strlen)(sub) + 1); + // Hardcode 10 elements as this is hardcoded size + if (rm) + COMMON_INTERCEPTOR_READ_RANGE(ctx, rm, 10 * struct_regmatch_sz); + if (sstr) + COMMON_INTERCEPTOR_READ_RANGE(ctx, sstr, REAL(strlen)(sstr) + 1); + SSIZE_T res = REAL(regasub)(buf, sub, rm, sstr); + if (res > 0 && buf) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, sizeof(char *)); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *buf, REAL(strlen)(*buf) + 1); + } + return res; +} + +#define INIT_REGEXSUB \ + COMMON_INTERCEPT_FUNCTION(regnsub); \ + COMMON_INTERCEPT_FUNCTION(regasub); +#else +#define INIT_REGEXSUB +#endif + +#if SANITIZER_INTERCEPT_FTS +INTERCEPTOR(void *, fts_open, char *const *path_argv, int options, + int (*compar)(void **, void **)) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fts_open, path_argv, options, compar); + if (path_argv) { + for (char *const *pa = path_argv; ; ++pa) { + COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, sizeof(char **)); + if (!*pa) + break; + COMMON_INTERCEPTOR_READ_RANGE(ctx, *pa, REAL(strlen)(*pa) + 1); + } + } + // TODO(kamil): handle compar callback + void *fts = REAL(fts_open)(path_argv, options, compar); + if (fts) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, fts, struct_FTS_sz); + return fts; +} + +INTERCEPTOR(void *, fts_read, void *ftsp) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fts_read, ftsp); + if (ftsp) + COMMON_INTERCEPTOR_READ_RANGE(ctx, ftsp, struct_FTS_sz); + void *ftsent = REAL(fts_read)(ftsp); + if (ftsent) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ftsent, struct_FTSENT_sz); + return ftsent; +} + +INTERCEPTOR(void *, fts_children, void *ftsp, int options) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fts_children, ftsp, options); + if (ftsp) + COMMON_INTERCEPTOR_READ_RANGE(ctx, ftsp, struct_FTS_sz); + void *ftsent = REAL(fts_children)(ftsp, options); + if (ftsent) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ftsent, struct_FTSENT_sz); + return ftsent; +} + +INTERCEPTOR(int, fts_set, void *ftsp, void *f, int options) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fts_set, ftsp, f, options); + if (ftsp) + COMMON_INTERCEPTOR_READ_RANGE(ctx, ftsp, struct_FTS_sz); + if (f) + COMMON_INTERCEPTOR_READ_RANGE(ctx, f, struct_FTSENT_sz); + return REAL(fts_set)(ftsp, f, options); +} + +INTERCEPTOR(int, fts_close, void *ftsp) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fts_close, ftsp); + if (ftsp) + COMMON_INTERCEPTOR_READ_RANGE(ctx, ftsp, struct_FTS_sz); + return REAL(fts_close)(ftsp); +} +#define INIT_FTS \ + COMMON_INTERCEPT_FUNCTION(fts_open); \ + COMMON_INTERCEPT_FUNCTION(fts_read); \ + COMMON_INTERCEPT_FUNCTION(fts_children); \ + COMMON_INTERCEPT_FUNCTION(fts_set); \ + COMMON_INTERCEPT_FUNCTION(fts_close); +#else +#define INIT_FTS +#endif + +#if SANITIZER_INTERCEPT_SYSCTL +INTERCEPTOR(int, sysctl, int *name, unsigned int namelen, void *oldp, + SIZE_T *oldlenp, void *newp, SIZE_T newlen) { + void *ctx; + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) + return internal_sysctl(name, namelen, oldp, oldlenp, newp, newlen); + COMMON_INTERCEPTOR_ENTER(ctx, sysctl, name, namelen, oldp, oldlenp, newp, + newlen); + if (name) + COMMON_INTERCEPTOR_READ_RANGE(ctx, name, namelen * sizeof(*name)); + if (oldlenp) + COMMON_INTERCEPTOR_READ_RANGE(ctx, oldlenp, sizeof(*oldlenp)); + if (newp && newlen) + COMMON_INTERCEPTOR_READ_RANGE(ctx, newp, newlen); + int res = REAL(sysctl)(name, namelen, oldp, oldlenp, newp, newlen); + if (!res) { + if (oldlenp) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldlenp, sizeof(*oldlenp)); + if (oldp) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldp, *oldlenp); + } + } + return res; +} + +INTERCEPTOR(int, sysctlbyname, char *sname, void *oldp, SIZE_T *oldlenp, + void *newp, SIZE_T newlen) { + void *ctx; + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) + return internal_sysctlbyname(sname, oldp, oldlenp, newp, newlen); + COMMON_INTERCEPTOR_ENTER(ctx, sysctlbyname, sname, oldp, oldlenp, newp, + newlen); + if (sname) + COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1); + if (oldlenp) + COMMON_INTERCEPTOR_READ_RANGE(ctx, oldlenp, sizeof(*oldlenp)); + if (newp && newlen) + COMMON_INTERCEPTOR_READ_RANGE(ctx, newp, newlen); + int res = REAL(sysctlbyname)(sname, oldp, oldlenp, newp, newlen); + if (!res) { + if (oldlenp) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldlenp, sizeof(*oldlenp)); + if (oldp) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldp, *oldlenp); + } + } + return res; +} + +INTERCEPTOR(int, sysctlnametomib, const char *sname, int *name, + SIZE_T *namelenp) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, sysctlnametomib, sname, name, namelenp); + if (sname) + COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1); + if (namelenp) + COMMON_INTERCEPTOR_READ_RANGE(ctx, namelenp, sizeof(*namelenp)); + int res = REAL(sysctlnametomib)(sname, name, namelenp); + if (!res) { + if (namelenp) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, namelenp, sizeof(*namelenp)); + if (name) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, *namelenp * sizeof(*name)); + } + } + return res; +} + +#define INIT_SYSCTL \ + COMMON_INTERCEPT_FUNCTION(sysctl); \ + COMMON_INTERCEPT_FUNCTION(sysctlbyname); \ + COMMON_INTERCEPT_FUNCTION(sysctlnametomib); +#else +#define INIT_SYSCTL +#endif + +#if SANITIZER_INTERCEPT_ASYSCTL +INTERCEPTOR(void *, asysctl, const int *name, SIZE_T namelen, SIZE_T *len) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, asysctl, name, namelen, len); + if (name) + COMMON_INTERCEPTOR_READ_RANGE(ctx, name, sizeof(*name) * namelen); + void *res = REAL(asysctl)(name, namelen, len); + if (res && len) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, len, sizeof(*len)); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, *len); + } + return res; +} + +INTERCEPTOR(void *, asysctlbyname, const char *sname, SIZE_T *len) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, asysctlbyname, sname, len); + if (sname) + COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1); + void *res = REAL(asysctlbyname)(sname, len); + if (res && len) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, len, sizeof(*len)); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, *len); + } + return res; +} +#define INIT_ASYSCTL \ + COMMON_INTERCEPT_FUNCTION(asysctl); \ + COMMON_INTERCEPT_FUNCTION(asysctlbyname); +#else +#define INIT_ASYSCTL +#endif + +#if SANITIZER_INTERCEPT_SYSCTLGETMIBINFO +INTERCEPTOR(int, sysctlgetmibinfo, char *sname, int *name, + unsigned int *namelenp, char *cname, SIZE_T *csz, void **rnode, + int v) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, sysctlgetmibinfo, sname, name, namelenp, cname, + csz, rnode, v); + if (sname) + COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1); + if (namelenp) + COMMON_INTERCEPTOR_READ_RANGE(ctx, namelenp, sizeof(*namelenp)); + if (csz) + COMMON_INTERCEPTOR_READ_RANGE(ctx, csz, sizeof(*csz)); + // Skip rnode, it's rarely used and not trivial to sanitize + // It's also used mostly internally + int res = REAL(sysctlgetmibinfo)(sname, name, namelenp, cname, csz, rnode, v); + if (!res) { + if (namelenp) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, namelenp, sizeof(*namelenp)); + if (name) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, *namelenp * sizeof(*name)); + } + if (csz) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, csz, sizeof(*csz)); + if (cname) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cname, *csz); + } + } + return res; +} +#define INIT_SYSCTLGETMIBINFO \ + COMMON_INTERCEPT_FUNCTION(sysctlgetmibinfo); +#else +#define INIT_SYSCTLGETMIBINFO +#endif + +#if SANITIZER_INTERCEPT_NL_LANGINFO +INTERCEPTOR(char *, nl_langinfo, long item) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, nl_langinfo, item); + char *ret = REAL(nl_langinfo)(item); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, REAL(strlen)(ret) + 1); + return ret; +} +#define INIT_NL_LANGINFO COMMON_INTERCEPT_FUNCTION(nl_langinfo) +#else +#define INIT_NL_LANGINFO +#endif + +#if SANITIZER_INTERCEPT_MODCTL +INTERCEPTOR(int, modctl, int operation, void *argp) { + void *ctx; + int ret; + COMMON_INTERCEPTOR_ENTER(ctx, modctl, operation, argp); + + if (operation == modctl_load) { + if (argp) { + __sanitizer_modctl_load_t *ml = (__sanitizer_modctl_load_t *)argp; + COMMON_INTERCEPTOR_READ_RANGE(ctx, ml, sizeof(*ml)); + if (ml->ml_filename) + COMMON_INTERCEPTOR_READ_RANGE(ctx, ml->ml_filename, + REAL(strlen)(ml->ml_filename) + 1); + if (ml->ml_props) + COMMON_INTERCEPTOR_READ_RANGE(ctx, ml->ml_props, ml->ml_propslen); + } + ret = REAL(modctl)(operation, argp); + } else if (operation == modctl_unload) { + if (argp) { + const char *name = (const char *)argp; + COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1); + } + ret = REAL(modctl)(operation, argp); + } else if (operation == modctl_stat) { + uptr iov_len; + struct __sanitizer_iovec *iov = (struct __sanitizer_iovec *)argp; + if (iov) { + COMMON_INTERCEPTOR_READ_RANGE(ctx, iov, sizeof(*iov)); + iov_len = iov->iov_len; + } + ret = REAL(modctl)(operation, argp); + if (iov) + COMMON_INTERCEPTOR_WRITE_RANGE( + ctx, iov->iov_base, Min(iov_len, iov->iov_len)); + } else if (operation == modctl_exists) + ret = REAL(modctl)(operation, argp); + else + ret = REAL(modctl)(operation, argp); + + return ret; +} +#define INIT_MODCTL COMMON_INTERCEPT_FUNCTION(modctl) +#else +#define INIT_MODCTL +#endif + +#if SANITIZER_INTERCEPT_STRTONUM +INTERCEPTOR(long long, strtonum, const char *nptr, long long minval, + long long maxval, const char **errstr) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strtonum, nptr, minval, maxval, errstr); + + // TODO(kamil): Implement strtoll as a common inteceptor + char *real_endptr; + long long ret = (long long)REAL(strtoimax)(nptr, &real_endptr, 10); + StrtolFixAndCheck(ctx, nptr, nullptr, real_endptr, 10); + + ret = REAL(strtonum)(nptr, minval, maxval, errstr); + if (errstr) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, errstr, sizeof(const char *)); + if (*errstr) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *errstr, REAL(strlen)(*errstr) + 1); + } + return ret; +} +#define INIT_STRTONUM COMMON_INTERCEPT_FUNCTION(strtonum) +#else +#define INIT_STRTONUM +#endif + +#if SANITIZER_INTERCEPT_FPARSELN +INTERCEPTOR(char *, fparseln, __sanitizer_FILE *stream, SIZE_T *len, + SIZE_T *lineno, const char delim[3], int flags) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fparseln, stream, len, lineno, delim, flags); + if (lineno) + COMMON_INTERCEPTOR_READ_RANGE(ctx, lineno, sizeof(*lineno)); + if (delim) + COMMON_INTERCEPTOR_READ_RANGE(ctx, delim, sizeof(delim[0]) * 3); + char *ret = REAL(fparseln)(stream, len, lineno, delim, flags); + if (ret) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, REAL(strlen)(ret) + 1); + if (len) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, len, sizeof(*len)); + if (lineno) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, lineno, sizeof(*lineno)); + } + return ret; +} +#define INIT_FPARSELN COMMON_INTERCEPT_FUNCTION(fparseln) +#else +#define INIT_FPARSELN +#endif + +#if SANITIZER_INTERCEPT_STATVFS1 +INTERCEPTOR(int, statvfs1, const char *path, void *buf, int flags) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, statvfs1, path, buf, flags); + if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); + int res = REAL(statvfs1)(path, buf, flags); + if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs_sz); + return res; +} +INTERCEPTOR(int, fstatvfs1, int fd, void *buf, int flags) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fstatvfs1, fd, buf, flags); + COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); + int res = REAL(fstatvfs1)(fd, buf, flags); + if (!res) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs_sz); + if (fd >= 0) + COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd); + } + return res; +} +#define INIT_STATVFS1 \ + COMMON_INTERCEPT_FUNCTION(statvfs1); \ + COMMON_INTERCEPT_FUNCTION(fstatvfs1); +#else +#define INIT_STATVFS1 +#endif + +#if SANITIZER_INTERCEPT_STRTOI +INTERCEPTOR(INTMAX_T, strtoi, const char *nptr, char **endptr, int base, + INTMAX_T low, INTMAX_T high, int *rstatus) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strtoi, nptr, endptr, base, low, high, rstatus); + char *real_endptr; + INTMAX_T ret = REAL(strtoi)(nptr, &real_endptr, base, low, high, rstatus); + StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base); + if (rstatus) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rstatus, sizeof(*rstatus)); + return ret; +} + +INTERCEPTOR(UINTMAX_T, strtou, const char *nptr, char **endptr, int base, + UINTMAX_T low, UINTMAX_T high, int *rstatus) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strtou, nptr, endptr, base, low, high, rstatus); + char *real_endptr; + UINTMAX_T ret = REAL(strtou)(nptr, &real_endptr, base, low, high, rstatus); + StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base); + if (rstatus) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rstatus, sizeof(*rstatus)); + return ret; +} +#define INIT_STRTOI \ + COMMON_INTERCEPT_FUNCTION(strtoi); \ + COMMON_INTERCEPT_FUNCTION(strtou) +#else +#define INIT_STRTOI +#endif + +#if SANITIZER_INTERCEPT_CAPSICUM +#define CAP_RIGHTS_INIT_INTERCEPTOR(cap_rights_init, rights, ...) \ + { \ + void *ctx; \ + COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_init, rights, ##__VA_ARGS__); \ + if (rights) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, rights, sizeof(*rights)); \ + __sanitizer_cap_rights_t *ret = \ + REAL(cap_rights_init)(rights, ##__VA_ARGS__); \ + if (ret) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, sizeof(*ret)); \ + return ret; \ + } + +#define CAP_RIGHTS_SET_INTERCEPTOR(cap_rights_set, rights, ...) \ + { \ + void *ctx; \ + COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_set, rights, ##__VA_ARGS__); \ + if (rights) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, rights, sizeof(*rights)); \ + __sanitizer_cap_rights_t *ret = \ + REAL(cap_rights_set)(rights, ##__VA_ARGS__); \ + if (ret) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, sizeof(*ret)); \ + return ret; \ + } + +#define CAP_RIGHTS_CLEAR_INTERCEPTOR(cap_rights_clear, rights, ...) \ + { \ + void *ctx; \ + COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_clear, rights, ##__VA_ARGS__); \ + if (rights) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, rights, sizeof(*rights)); \ + __sanitizer_cap_rights_t *ret = \ + REAL(cap_rights_clear)(rights, ##__VA_ARGS__); \ + if (ret) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, sizeof(*ret)); \ + return ret; \ + } + +#define CAP_RIGHTS_IS_SET_INTERCEPTOR(cap_rights_is_set, rights, ...) \ + { \ + void *ctx; \ + COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_is_set, rights, ##__VA_ARGS__); \ + if (rights) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, rights, sizeof(*rights)); \ + return REAL(cap_rights_is_set)(rights, ##__VA_ARGS__); \ + } + +INTERCEPTOR(__sanitizer_cap_rights_t *, cap_rights_init, + __sanitizer_cap_rights_t *rights) { + CAP_RIGHTS_INIT_INTERCEPTOR(cap_rights_init, rights); +} + +INTERCEPTOR(__sanitizer_cap_rights_t *, cap_rights_set, + __sanitizer_cap_rights_t *rights) { + CAP_RIGHTS_SET_INTERCEPTOR(cap_rights_set, rights); +} + +INTERCEPTOR(__sanitizer_cap_rights_t *, cap_rights_clear, + __sanitizer_cap_rights_t *rights) { + CAP_RIGHTS_CLEAR_INTERCEPTOR(cap_rights_clear, rights); +} + +INTERCEPTOR(bool, cap_rights_is_set, + __sanitizer_cap_rights_t *rights) { + CAP_RIGHTS_IS_SET_INTERCEPTOR(cap_rights_is_set, rights); +} + +INTERCEPTOR(int, cap_rights_limit, int fd, + const __sanitizer_cap_rights_t *rights) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_limit, fd, rights); + if (rights) + COMMON_INTERCEPTOR_READ_RANGE(ctx, rights, sizeof(*rights)); + + return REAL(cap_rights_limit)(fd, rights); +} + +INTERCEPTOR(int, cap_rights_get, int fd, __sanitizer_cap_rights_t *rights) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_get, fd, rights); + int ret = REAL(cap_rights_get)(fd, rights); + if (!ret && rights) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rights, sizeof(*rights)); + + return ret; +} + +INTERCEPTOR(bool, cap_rights_is_valid, const __sanitizer_cap_rights_t *rights) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_is_valid, rights); + if (rights) + COMMON_INTERCEPTOR_READ_RANGE(ctx, rights, sizeof(*rights)); + + return REAL(cap_rights_is_valid(rights)); +} + +INTERCEPTOR(__sanitizer_cap_rights *, cap_rights_merge, + __sanitizer_cap_rights *dst, const __sanitizer_cap_rights *src) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_merge, dst, src); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, sizeof(*src)); + + __sanitizer_cap_rights *ret = REAL(cap_rights_merge)(dst, src); + if (dst) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, sizeof(*dst)); + + return ret; +} + +INTERCEPTOR(__sanitizer_cap_rights *, cap_rights_remove, + __sanitizer_cap_rights *dst, const __sanitizer_cap_rights *src) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_remove, dst, src); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, sizeof(*src)); + + __sanitizer_cap_rights *ret = REAL(cap_rights_remove)(dst, src); + if (dst) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, sizeof(*dst)); + + return ret; +} + +INTERCEPTOR(bool, cap_rights_contains, const __sanitizer_cap_rights *big, + const __sanitizer_cap_rights *little) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_contains, big, little); + if (little) + COMMON_INTERCEPTOR_READ_RANGE(ctx, little, sizeof(*little)); + if (big) + COMMON_INTERCEPTOR_READ_RANGE(ctx, big, sizeof(*big)); + + return REAL(cap_rights_contains)(big, little); +} + +INTERCEPTOR(int, cap_ioctls_limit, int fd, const uptr *cmds, SIZE_T ncmds) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cap_ioctls_limit, fd, cmds, ncmds); + if (cmds) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cmds, sizeof(*cmds) * ncmds); + + return REAL(cap_ioctls_limit)(fd, cmds, ncmds); +} + +INTERCEPTOR(int, cap_ioctls_get, int fd, uptr *cmds, SIZE_T maxcmds) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cap_ioctls_get, fd, cmds, maxcmds); + int ret = REAL(cap_ioctls_get)(fd, cmds, maxcmds); + if (!ret && cmds) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cmds, sizeof(*cmds) * maxcmds); + + return ret; +} +#define INIT_CAPSICUM \ + COMMON_INTERCEPT_FUNCTION(cap_rights_init); \ + COMMON_INTERCEPT_FUNCTION(cap_rights_set); \ + COMMON_INTERCEPT_FUNCTION(cap_rights_clear); \ + COMMON_INTERCEPT_FUNCTION(cap_rights_is_set); \ + COMMON_INTERCEPT_FUNCTION(cap_rights_get); \ + COMMON_INTERCEPT_FUNCTION(cap_rights_limit); \ + COMMON_INTERCEPT_FUNCTION(cap_rights_contains); \ + COMMON_INTERCEPT_FUNCTION(cap_rights_remove); \ + COMMON_INTERCEPT_FUNCTION(cap_rights_merge); \ + COMMON_INTERCEPT_FUNCTION(cap_rights_is_valid); \ + COMMON_INTERCEPT_FUNCTION(cap_ioctls_get); \ + COMMON_INTERCEPT_FUNCTION(cap_ioctls_limit) +#else +#define INIT_CAPSICUM +#endif + +#if SANITIZER_INTERCEPT_SHA1 +INTERCEPTOR(void, SHA1Init, void *context) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, SHA1Init, context); + REAL(SHA1Init)(context); + if (context) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, SHA1_CTX_sz); +} +INTERCEPTOR(void, SHA1Update, void *context, const u8 *data, unsigned len) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, SHA1Update, context, data, len); + if (data && len > 0) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA1_CTX_sz); + REAL(SHA1Update)(context, data, len); + if (context) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, SHA1_CTX_sz); +} +INTERCEPTOR(void, SHA1Final, u8 digest[20], void *context) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, SHA1Final, digest, context); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA1_CTX_sz); + REAL(SHA1Final)(digest, context); + if (digest) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, sizeof(u8) * 20); +} +INTERCEPTOR(void, SHA1Transform, u32 state[5], u8 buffer[64]) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, SHA1Transform, state, buffer); + if (state) + COMMON_INTERCEPTOR_READ_RANGE(ctx, state, sizeof(u32) * 5); + if (buffer) + COMMON_INTERCEPTOR_READ_RANGE(ctx, buffer, sizeof(u8) * 64); + REAL(SHA1Transform)(state, buffer); + if (state) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, state, sizeof(u32) * 5); +} +INTERCEPTOR(char *, SHA1End, void *context, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, SHA1End, context, buf); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA1_CTX_sz); + char *ret = REAL(SHA1End)(context, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA1_return_length); + return ret; +} +INTERCEPTOR(char *, SHA1File, char *filename, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, SHA1File, filename, buf); + if (filename) + COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1); + char *ret = REAL(SHA1File)(filename, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA1_return_length); + return ret; +} +INTERCEPTOR(char *, SHA1FileChunk, char *filename, char *buf, OFF_T offset, + OFF_T length) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, SHA1FileChunk, filename, buf, offset, length); + if (filename) + COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1); + char *ret = REAL(SHA1FileChunk)(filename, buf, offset, length); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA1_return_length); + return ret; +} +INTERCEPTOR(char *, SHA1Data, u8 *data, SIZE_T len, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, SHA1Data, data, len, buf); + if (data) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); + char *ret = REAL(SHA1Data)(data, len, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA1_return_length); + return ret; +} +#define INIT_SHA1 \ + COMMON_INTERCEPT_FUNCTION(SHA1Init); \ + COMMON_INTERCEPT_FUNCTION(SHA1Update); \ + COMMON_INTERCEPT_FUNCTION(SHA1Final); \ + COMMON_INTERCEPT_FUNCTION(SHA1Transform); \ + COMMON_INTERCEPT_FUNCTION(SHA1End); \ + COMMON_INTERCEPT_FUNCTION(SHA1File); \ + COMMON_INTERCEPT_FUNCTION(SHA1FileChunk); \ + COMMON_INTERCEPT_FUNCTION(SHA1Data) +#else +#define INIT_SHA1 +#endif + +#if SANITIZER_INTERCEPT_MD4 +INTERCEPTOR(void, MD4Init, void *context) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD4Init, context); + REAL(MD4Init)(context); + if (context) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD4_CTX_sz); +} + +INTERCEPTOR(void, MD4Update, void *context, const unsigned char *data, + unsigned int len) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD4Update, context, data, len); + if (data && len > 0) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD4_CTX_sz); + REAL(MD4Update)(context, data, len); + if (context) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD4_CTX_sz); +} + +INTERCEPTOR(void, MD4Final, unsigned char digest[16], void *context) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD4Final, digest, context); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD4_CTX_sz); + REAL(MD4Final)(digest, context); + if (digest) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, sizeof(unsigned char) * 16); +} + +INTERCEPTOR(char *, MD4End, void *context, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD4End, context, buf); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD4_CTX_sz); + char *ret = REAL(MD4End)(context, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD4_return_length); + return ret; +} + +INTERCEPTOR(char *, MD4File, const char *filename, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD4File, filename, buf); + if (filename) + COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1); + char *ret = REAL(MD4File)(filename, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD4_return_length); + return ret; +} + +INTERCEPTOR(char *, MD4Data, const unsigned char *data, unsigned int len, + char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD4Data, data, len, buf); + if (data && len > 0) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); + char *ret = REAL(MD4Data)(data, len, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD4_return_length); + return ret; +} + +#define INIT_MD4 \ + COMMON_INTERCEPT_FUNCTION(MD4Init); \ + COMMON_INTERCEPT_FUNCTION(MD4Update); \ + COMMON_INTERCEPT_FUNCTION(MD4Final); \ + COMMON_INTERCEPT_FUNCTION(MD4End); \ + COMMON_INTERCEPT_FUNCTION(MD4File); \ + COMMON_INTERCEPT_FUNCTION(MD4Data) +#else +#define INIT_MD4 +#endif + +#if SANITIZER_INTERCEPT_RMD160 +INTERCEPTOR(void, RMD160Init, void *context) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, RMD160Init, context); + REAL(RMD160Init)(context); + if (context) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, RMD160_CTX_sz); +} +INTERCEPTOR(void, RMD160Update, void *context, const u8 *data, unsigned len) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, RMD160Update, context, data, len); + if (data && len > 0) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, RMD160_CTX_sz); + REAL(RMD160Update)(context, data, len); + if (context) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, RMD160_CTX_sz); +} +INTERCEPTOR(void, RMD160Final, u8 digest[20], void *context) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, RMD160Final, digest, context); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, RMD160_CTX_sz); + REAL(RMD160Final)(digest, context); + if (digest) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, sizeof(u8) * 20); +} +INTERCEPTOR(void, RMD160Transform, u32 state[5], u16 buffer[16]) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, RMD160Transform, state, buffer); + if (state) + COMMON_INTERCEPTOR_READ_RANGE(ctx, state, sizeof(u32) * 5); + if (buffer) + COMMON_INTERCEPTOR_READ_RANGE(ctx, buffer, sizeof(u32) * 16); + REAL(RMD160Transform)(state, buffer); + if (state) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, state, sizeof(u32) * 5); +} +INTERCEPTOR(char *, RMD160End, void *context, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, RMD160End, context, buf); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, RMD160_CTX_sz); + char *ret = REAL(RMD160End)(context, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, RMD160_return_length); + return ret; +} +INTERCEPTOR(char *, RMD160File, char *filename, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, RMD160File, filename, buf); + if (filename) + COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1); + char *ret = REAL(RMD160File)(filename, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, RMD160_return_length); + return ret; +} +INTERCEPTOR(char *, RMD160FileChunk, char *filename, char *buf, OFF_T offset, + OFF_T length) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, RMD160FileChunk, filename, buf, offset, length); + if (filename) + COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1); + char *ret = REAL(RMD160FileChunk)(filename, buf, offset, length); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, RMD160_return_length); + return ret; +} +INTERCEPTOR(char *, RMD160Data, u8 *data, SIZE_T len, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, RMD160Data, data, len, buf); + if (data && len > 0) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); + char *ret = REAL(RMD160Data)(data, len, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, RMD160_return_length); + return ret; +} +#define INIT_RMD160 \ + COMMON_INTERCEPT_FUNCTION(RMD160Init); \ + COMMON_INTERCEPT_FUNCTION(RMD160Update); \ + COMMON_INTERCEPT_FUNCTION(RMD160Final); \ + COMMON_INTERCEPT_FUNCTION(RMD160Transform); \ + COMMON_INTERCEPT_FUNCTION(RMD160End); \ + COMMON_INTERCEPT_FUNCTION(RMD160File); \ + COMMON_INTERCEPT_FUNCTION(RMD160FileChunk); \ + COMMON_INTERCEPT_FUNCTION(RMD160Data) +#else +#define INIT_RMD160 +#endif + +#if SANITIZER_INTERCEPT_MD5 +INTERCEPTOR(void, MD5Init, void *context) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD5Init, context); + REAL(MD5Init)(context); + if (context) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD5_CTX_sz); +} + +INTERCEPTOR(void, MD5Update, void *context, const unsigned char *data, + unsigned int len) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD5Update, context, data, len); + if (data && len > 0) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD5_CTX_sz); + REAL(MD5Update)(context, data, len); + if (context) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD5_CTX_sz); +} + +INTERCEPTOR(void, MD5Final, unsigned char digest[16], void *context) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD5Final, digest, context); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD5_CTX_sz); + REAL(MD5Final)(digest, context); + if (digest) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, sizeof(unsigned char) * 16); +} + +INTERCEPTOR(char *, MD5End, void *context, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD5End, context, buf); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD5_CTX_sz); + char *ret = REAL(MD5End)(context, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD5_return_length); + return ret; +} + +INTERCEPTOR(char *, MD5File, const char *filename, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD5File, filename, buf); + if (filename) + COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1); + char *ret = REAL(MD5File)(filename, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD5_return_length); + return ret; +} + +INTERCEPTOR(char *, MD5Data, const unsigned char *data, unsigned int len, + char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD5Data, data, len, buf); + if (data && len > 0) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); + char *ret = REAL(MD5Data)(data, len, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD5_return_length); + return ret; +} + +#define INIT_MD5 \ + COMMON_INTERCEPT_FUNCTION(MD5Init); \ + COMMON_INTERCEPT_FUNCTION(MD5Update); \ + COMMON_INTERCEPT_FUNCTION(MD5Final); \ + COMMON_INTERCEPT_FUNCTION(MD5End); \ + COMMON_INTERCEPT_FUNCTION(MD5File); \ + COMMON_INTERCEPT_FUNCTION(MD5Data) +#else +#define INIT_MD5 +#endif + +#if SANITIZER_INTERCEPT_FSEEK +INTERCEPTOR(int, fseek, __sanitizer_FILE *stream, long int offset, int whence) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fseek, stream, offset, whence); + return REAL(fseek)(stream, offset, whence); +} +INTERCEPTOR(int, fseeko, __sanitizer_FILE *stream, OFF_T offset, int whence) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fseeko, stream, offset, whence); + return REAL(fseeko)(stream, offset, whence); +} +INTERCEPTOR(long int, ftell, __sanitizer_FILE *stream) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, ftell, stream); + return REAL(ftell)(stream); +} +INTERCEPTOR(OFF_T, ftello, __sanitizer_FILE *stream) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, ftello, stream); + return REAL(ftello)(stream); +} +INTERCEPTOR(void, rewind, __sanitizer_FILE *stream) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, rewind, stream); + return REAL(rewind)(stream); +} +INTERCEPTOR(int, fgetpos, __sanitizer_FILE *stream, void *pos) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fgetpos, stream, pos); + int ret = REAL(fgetpos)(stream, pos); + if (pos && !ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pos, fpos_t_sz); + return ret; +} +INTERCEPTOR(int, fsetpos, __sanitizer_FILE *stream, const void *pos) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fsetpos, stream, pos); + if (pos) + COMMON_INTERCEPTOR_READ_RANGE(ctx, pos, fpos_t_sz); + return REAL(fsetpos)(stream, pos); +} +#define INIT_FSEEK \ + COMMON_INTERCEPT_FUNCTION(fseek); \ + COMMON_INTERCEPT_FUNCTION(fseeko); \ + COMMON_INTERCEPT_FUNCTION(ftell); \ + COMMON_INTERCEPT_FUNCTION(ftello); \ + COMMON_INTERCEPT_FUNCTION(rewind); \ + COMMON_INTERCEPT_FUNCTION(fgetpos); \ + COMMON_INTERCEPT_FUNCTION(fsetpos) +#else +#define INIT_FSEEK +#endif + +#if SANITIZER_INTERCEPT_MD2 +INTERCEPTOR(void, MD2Init, void *context) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD2Init, context); + REAL(MD2Init)(context); + if (context) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD2_CTX_sz); +} + +INTERCEPTOR(void, MD2Update, void *context, const unsigned char *data, + unsigned int len) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD2Update, context, data, len); + if (data && len > 0) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD2_CTX_sz); + REAL(MD2Update)(context, data, len); + if (context) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD2_CTX_sz); +} + +INTERCEPTOR(void, MD2Final, unsigned char digest[16], void *context) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD2Final, digest, context); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD2_CTX_sz); + REAL(MD2Final)(digest, context); + if (digest) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, sizeof(unsigned char) * 16); +} + +INTERCEPTOR(char *, MD2End, void *context, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD2End, context, buf); + if (context) + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD2_CTX_sz); + char *ret = REAL(MD2End)(context, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD2_return_length); + return ret; +} + +INTERCEPTOR(char *, MD2File, const char *filename, char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD2File, filename, buf); + if (filename) + COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1); + char *ret = REAL(MD2File)(filename, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD2_return_length); + return ret; +} + +INTERCEPTOR(char *, MD2Data, const unsigned char *data, unsigned int len, + char *buf) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, MD2Data, data, len, buf); + if (data && len > 0) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); + char *ret = REAL(MD2Data)(data, len, buf); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD2_return_length); + return ret; +} + +#define INIT_MD2 \ + COMMON_INTERCEPT_FUNCTION(MD2Init); \ + COMMON_INTERCEPT_FUNCTION(MD2Update); \ + COMMON_INTERCEPT_FUNCTION(MD2Final); \ + COMMON_INTERCEPT_FUNCTION(MD2End); \ + COMMON_INTERCEPT_FUNCTION(MD2File); \ + COMMON_INTERCEPT_FUNCTION(MD2Data) +#else +#define INIT_MD2 +#endif + +#if SANITIZER_INTERCEPT_SHA2 +#define SHA2_INTERCEPTORS(LEN, SHA2_STATE_T) \ + INTERCEPTOR(void, SHA##LEN##_Init, void *context) { \ + void *ctx; \ + COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Init, context); \ + REAL(SHA##LEN##_Init)(context); \ + if (context) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, SHA##LEN##_CTX_sz); \ + } \ + INTERCEPTOR(void, SHA##LEN##_Update, void *context, \ + const u8 *data, SIZE_T len) { \ + void *ctx; \ + COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Update, context, data, len); \ + if (data && len > 0) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); \ + if (context) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA##LEN##_CTX_sz); \ + REAL(SHA##LEN##_Update)(context, data, len); \ + if (context) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, SHA##LEN##_CTX_sz); \ + } \ + INTERCEPTOR(void, SHA##LEN##_Final, u8 digest[LEN/8], \ + void *context) { \ + void *ctx; \ + CHECK_EQ(SHA##LEN##_digest_length, LEN/8); \ + COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Final, digest, context); \ + if (context) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA##LEN##_CTX_sz); \ + REAL(SHA##LEN##_Final)(digest, context); \ + if (digest) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, \ + sizeof(digest[0]) * \ + SHA##LEN##_digest_length); \ + } \ + INTERCEPTOR(char *, SHA##LEN##_End, void *context, char *buf) { \ + void *ctx; \ + COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_End, context, buf); \ + if (context) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA##LEN##_CTX_sz); \ + char *ret = REAL(SHA##LEN##_End)(context, buf); \ + if (ret) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \ + return ret; \ + } \ + INTERCEPTOR(char *, SHA##LEN##_File, const char *filename, char *buf) { \ + void *ctx; \ + COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_File, filename, buf); \ + if (filename) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);\ + char *ret = REAL(SHA##LEN##_File)(filename, buf); \ + if (ret) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \ + return ret; \ + } \ + INTERCEPTOR(char *, SHA##LEN##_FileChunk, const char *filename, char *buf, \ + OFF_T offset, OFF_T length) { \ + void *ctx; \ + COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_FileChunk, filename, buf, offset, \ + length); \ + if (filename) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);\ + char *ret = REAL(SHA##LEN##_FileChunk)(filename, buf, offset, length); \ + if (ret) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \ + return ret; \ + } \ + INTERCEPTOR(char *, SHA##LEN##_Data, u8 *data, SIZE_T len, char *buf) { \ + void *ctx; \ + COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Data, data, len, buf); \ + if (data && len > 0) \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); \ + char *ret = REAL(SHA##LEN##_Data)(data, len, buf); \ + if (ret) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \ + return ret; \ + } + +SHA2_INTERCEPTORS(224, u32); +SHA2_INTERCEPTORS(256, u32); +SHA2_INTERCEPTORS(384, u64); +SHA2_INTERCEPTORS(512, u64); + +#define INIT_SHA2_INTECEPTORS(LEN) \ + COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Init); \ + COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Update); \ + COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Final); \ + COMMON_INTERCEPT_FUNCTION(SHA##LEN##_End); \ + COMMON_INTERCEPT_FUNCTION(SHA##LEN##_File); \ + COMMON_INTERCEPT_FUNCTION(SHA##LEN##_FileChunk); \ + COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Data) + +#define INIT_SHA2 \ + INIT_SHA2_INTECEPTORS(224); \ + INIT_SHA2_INTECEPTORS(256); \ + INIT_SHA2_INTECEPTORS(384); \ + INIT_SHA2_INTECEPTORS(512) +#undef SHA2_INTERCEPTORS +#else +#define INIT_SHA2 +#endif + +#if SANITIZER_INTERCEPT_VIS +INTERCEPTOR(char *, vis, char *dst, int c, int flag, int nextc) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, vis, dst, c, flag, nextc); + char *end = REAL(vis)(dst, c, flag, nextc); + // dst is NULL terminated and end points to the NULL char + if (dst && end) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, end - dst + 1); + return end; +} +INTERCEPTOR(char *, nvis, char *dst, SIZE_T dlen, int c, int flag, int nextc) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, nvis, dst, dlen, c, flag, nextc); + char *end = REAL(nvis)(dst, dlen, c, flag, nextc); + // nvis cannot make sure the dst is NULL terminated + if (dst && end) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, end - dst + 1); + return end; +} +INTERCEPTOR(int, strvis, char *dst, const char *src, int flag) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strvis, dst, src, flag); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1); + int len = REAL(strvis)(dst, src, flag); + if (dst) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, len + 1); + return len; +} +INTERCEPTOR(int, stravis, char **dst, const char *src, int flag) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, stravis, dst, src, flag); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1); + int len = REAL(stravis)(dst, src, flag); + if (dst) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, sizeof(char *)); + if (*dst) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *dst, len + 1); + } + return len; +} +INTERCEPTOR(int, strnvis, char *dst, SIZE_T dlen, const char *src, int flag) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strnvis, dst, dlen, src, flag); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1); + int len = REAL(strnvis)(dst, dlen, src, flag); + // The interface will be valid even if there is no space for NULL char + if (dst && len > 0) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, len + 1); + return len; +} +INTERCEPTOR(int, strvisx, char *dst, const char *src, SIZE_T len, int flag) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strvisx, dst, src, len, flag); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len); + int ret = REAL(strvisx)(dst, src, len, flag); + if (dst) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1); + return ret; +} +INTERCEPTOR(int, strnvisx, char *dst, SIZE_T dlen, const char *src, SIZE_T len, + int flag) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strnvisx, dst, dlen, src, len, flag); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len); + int ret = REAL(strnvisx)(dst, dlen, src, len, flag); + if (dst && ret >= 0) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1); + return ret; +} +INTERCEPTOR(int, strenvisx, char *dst, SIZE_T dlen, const char *src, SIZE_T len, + int flag, int *cerr_ptr) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strenvisx, dst, dlen, src, len, flag, cerr_ptr); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len); + // FIXME: only need to be checked when "flag | VIS_NOLOCALE" doesn't hold + // according to the implementation + if (cerr_ptr) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cerr_ptr, sizeof(int)); + int ret = REAL(strenvisx)(dst, dlen, src, len, flag, cerr_ptr); + if (dst && ret >= 0) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1); + if (cerr_ptr) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cerr_ptr, sizeof(int)); + return ret; +} +INTERCEPTOR(char *, svis, char *dst, int c, int flag, int nextc, + const char *extra) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, svis, dst, c, flag, nextc, extra); + if (extra) + COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1); + char *end = REAL(svis)(dst, c, flag, nextc, extra); + if (dst && end) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, end - dst + 1); + return end; +} +INTERCEPTOR(char *, snvis, char *dst, SIZE_T dlen, int c, int flag, int nextc, + const char *extra) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, snvis, dst, dlen, c, flag, nextc, extra); + if (extra) + COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1); + char *end = REAL(snvis)(dst, dlen, c, flag, nextc, extra); + if (dst && end) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, + Min((SIZE_T)(end - dst + 1), dlen)); + return end; +} +INTERCEPTOR(int, strsvis, char *dst, const char *src, int flag, + const char *extra) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strsvis, dst, src, flag, extra); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1); + if (extra) + COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1); + int len = REAL(strsvis)(dst, src, flag, extra); + if (dst) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, len + 1); + return len; +} +INTERCEPTOR(int, strsnvis, char *dst, SIZE_T dlen, const char *src, int flag, + const char *extra) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strsnvis, dst, dlen, src, flag, extra); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1); + if (extra) + COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1); + int len = REAL(strsnvis)(dst, dlen, src, flag, extra); + // The interface will be valid even if there is no space for NULL char + if (dst && len >= 0) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, len + 1); + return len; +} +INTERCEPTOR(int, strsvisx, char *dst, const char *src, SIZE_T len, int flag, + const char *extra) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strsvisx, dst, src, len, flag, extra); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len); + if (extra) + COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1); + int ret = REAL(strsvisx)(dst, src, len, flag, extra); + if (dst) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1); + return ret; +} +INTERCEPTOR(int, strsnvisx, char *dst, SIZE_T dlen, const char *src, SIZE_T len, + int flag, const char *extra) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strsnvisx, dst, dlen, src, len, flag, extra); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len); + if (extra) + COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1); + int ret = REAL(strsnvisx)(dst, dlen, src, len, flag, extra); + if (dst && ret >= 0) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1); + return ret; +} +INTERCEPTOR(int, strsenvisx, char *dst, SIZE_T dlen, const char *src, + SIZE_T len, int flag, const char *extra, int *cerr_ptr) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strsenvisx, dst, dlen, src, len, flag, extra, + cerr_ptr); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len); + if (extra) + COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1); + // FIXME: only need to be checked when "flag | VIS_NOLOCALE" doesn't hold + // according to the implementation + if (cerr_ptr) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cerr_ptr, sizeof(int)); + int ret = REAL(strsenvisx)(dst, dlen, src, len, flag, extra, cerr_ptr); + if (dst && ret >= 0) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1); + if (cerr_ptr) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cerr_ptr, sizeof(int)); + return ret; +} +INTERCEPTOR(int, unvis, char *cp, int c, int *astate, int flag) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, unvis, cp, c, astate, flag); + if (astate) + COMMON_INTERCEPTOR_READ_RANGE(ctx, astate, sizeof(*astate)); + int ret = REAL(unvis)(cp, c, astate, flag); + if (ret == unvis_valid || ret == unvis_validpush) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cp, sizeof(*cp)); + } + return ret; +} +INTERCEPTOR(int, strunvis, char *dst, const char *src) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strunvis, dst, src); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1); + int ret = REAL(strunvis)(dst, src); + if (ret != -1) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1); + return ret; +} +INTERCEPTOR(int, strnunvis, char *dst, SIZE_T dlen, const char *src) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strnunvis, dst, dlen, src); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1); + int ret = REAL(strnunvis)(dst, dlen, src); + if (ret != -1) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1); + return ret; +} +INTERCEPTOR(int, strunvisx, char *dst, const char *src, int flag) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strunvisx, dst, src, flag); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1); + int ret = REAL(strunvisx)(dst, src, flag); + if (ret != -1) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1); + return ret; +} +INTERCEPTOR(int, strnunvisx, char *dst, SIZE_T dlen, const char *src, + int flag) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strnunvisx, dst, dlen, src, flag); + if (src) + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1); + int ret = REAL(strnunvisx)(dst, dlen, src, flag); + if (ret != -1) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1); + return ret; +} +#define INIT_VIS \ + COMMON_INTERCEPT_FUNCTION(vis); \ + COMMON_INTERCEPT_FUNCTION(nvis); \ + COMMON_INTERCEPT_FUNCTION(strvis); \ + COMMON_INTERCEPT_FUNCTION(stravis); \ + COMMON_INTERCEPT_FUNCTION(strnvis); \ + COMMON_INTERCEPT_FUNCTION(strvisx); \ + COMMON_INTERCEPT_FUNCTION(strnvisx); \ + COMMON_INTERCEPT_FUNCTION(strenvisx); \ + COMMON_INTERCEPT_FUNCTION(svis); \ + COMMON_INTERCEPT_FUNCTION(snvis); \ + COMMON_INTERCEPT_FUNCTION(strsvis); \ + COMMON_INTERCEPT_FUNCTION(strsnvis); \ + COMMON_INTERCEPT_FUNCTION(strsvisx); \ + COMMON_INTERCEPT_FUNCTION(strsnvisx); \ + COMMON_INTERCEPT_FUNCTION(strsenvisx); \ + COMMON_INTERCEPT_FUNCTION(unvis); \ + COMMON_INTERCEPT_FUNCTION(strunvis); \ + COMMON_INTERCEPT_FUNCTION(strnunvis); \ + COMMON_INTERCEPT_FUNCTION(strunvisx); \ + COMMON_INTERCEPT_FUNCTION(strnunvisx) +#else +#define INIT_VIS +#endif + +#if SANITIZER_INTERCEPT_CDB +INTERCEPTOR(struct __sanitizer_cdbr *, cdbr_open, const char *path, int flags) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbr_open, path, flags); + if (path) + COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); + struct __sanitizer_cdbr *cdbr = REAL(cdbr_open)(path, flags); + if (cdbr) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cdbr, sizeof(*cdbr)); + return cdbr; +} + +INTERCEPTOR(struct __sanitizer_cdbr *, cdbr_open_mem, void *base, SIZE_T size, + int flags, void (*unmap)(void *, void *, SIZE_T), void *cookie) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbr_open_mem, base, size, flags, unmap, + cookie); + if (base && size) + COMMON_INTERCEPTOR_READ_RANGE(ctx, base, size); + struct __sanitizer_cdbr *cdbr = + REAL(cdbr_open_mem)(base, size, flags, unmap, cookie); + if (cdbr) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cdbr, sizeof(*cdbr)); + return cdbr; +} + +INTERCEPTOR(u32, cdbr_entries, struct __sanitizer_cdbr *cdbr) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbr_entries, cdbr); + if (cdbr) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbr, sizeof(*cdbr)); + return REAL(cdbr_entries)(cdbr); +} + +INTERCEPTOR(int, cdbr_get, struct __sanitizer_cdbr *cdbr, u32 index, + const void **data, SIZE_T *datalen) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbr_get, cdbr, index, data, datalen); + if (cdbr) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbr, sizeof(*cdbr)); + int ret = REAL(cdbr_get)(cdbr, index, data, datalen); + if (!ret) { + if (data) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, sizeof(*data)); + if (datalen) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, datalen, sizeof(*datalen)); + if (data && datalen) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *data, *datalen); + } + return ret; +} + +INTERCEPTOR(int, cdbr_find, struct __sanitizer_cdbr *cdbr, const void *key, + SIZE_T keylen, const void **data, SIZE_T *datalen) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbr_find, cdbr, key, keylen, data, datalen); + if (cdbr) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbr, sizeof(*cdbr)); + if (key) + COMMON_INTERCEPTOR_READ_RANGE(ctx, key, keylen); + int ret = REAL(cdbr_find)(cdbr, key, keylen, data, datalen); + if (!ret) { + if (data) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, sizeof(*data)); + if (datalen) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, datalen, sizeof(*datalen)); + if (data && datalen) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *data, *datalen); + } + return ret; +} + +INTERCEPTOR(void, cdbr_close, struct __sanitizer_cdbr *cdbr) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbr_close, cdbr); + if (cdbr) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbr, sizeof(*cdbr)); + REAL(cdbr_close)(cdbr); +} + +INTERCEPTOR(struct __sanitizer_cdbw *, cdbw_open) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbw_open); + struct __sanitizer_cdbw *ret = REAL(cdbw_open)(); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, sizeof(*ret)); + return ret; +} + +INTERCEPTOR(int, cdbw_put, struct __sanitizer_cdbw *cdbw, const void *key, + SIZE_T keylen, const void *data, SIZE_T datalen) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbw_put, cdbw, key, keylen, data, datalen); + if (cdbw) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbw, sizeof(*cdbw)); + if (data && datalen) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, datalen); + if (key && keylen) + COMMON_INTERCEPTOR_READ_RANGE(ctx, key, keylen); + int ret = REAL(cdbw_put)(cdbw, key, keylen, data, datalen); + if (!ret && cdbw) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cdbw, sizeof(*cdbw)); + return ret; +} + +INTERCEPTOR(int, cdbw_put_data, struct __sanitizer_cdbw *cdbw, const void *data, + SIZE_T datalen, u32 *index) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbw_put_data, cdbw, data, datalen, index); + if (cdbw) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbw, sizeof(*cdbw)); + if (data && datalen) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, datalen); + int ret = REAL(cdbw_put_data)(cdbw, data, datalen, index); + if (!ret) { + if (index) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, index, sizeof(*index)); + if (cdbw) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cdbw, sizeof(*cdbw)); + } + return ret; +} + +INTERCEPTOR(int, cdbw_put_key, struct __sanitizer_cdbw *cdbw, const void *key, + SIZE_T keylen, u32 index) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbw_put_key, cdbw, key, keylen, index); + if (cdbw) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbw, sizeof(*cdbw)); + if (key && keylen) + COMMON_INTERCEPTOR_READ_RANGE(ctx, key, keylen); + int ret = REAL(cdbw_put_key)(cdbw, key, keylen, index); + if (!ret && cdbw) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cdbw, sizeof(*cdbw)); + return ret; +} + +INTERCEPTOR(int, cdbw_output, struct __sanitizer_cdbw *cdbw, int output, + const char descr[16], u32 (*seedgen)(void)) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbw_output, cdbw, output, descr, seedgen); + COMMON_INTERCEPTOR_FD_ACCESS(ctx, output); + if (cdbw) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbw, sizeof(*cdbw)); + if (descr) + COMMON_INTERCEPTOR_READ_RANGE(ctx, descr, internal_strnlen(descr, 16)); + if (seedgen) + COMMON_INTERCEPTOR_READ_RANGE(ctx, (void *)seedgen, sizeof(seedgen)); + int ret = REAL(cdbw_output)(cdbw, output, descr, seedgen); + if (!ret) { + if (cdbw) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cdbw, sizeof(*cdbw)); + if (output >= 0) + COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, output); + } + return ret; +} + +INTERCEPTOR(void, cdbw_close, struct __sanitizer_cdbw *cdbw) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cdbw_close, cdbw); + if (cdbw) + COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbw, sizeof(*cdbw)); + REAL(cdbw_close)(cdbw); +} + +#define INIT_CDB \ + COMMON_INTERCEPT_FUNCTION(cdbr_open); \ + COMMON_INTERCEPT_FUNCTION(cdbr_open_mem); \ + COMMON_INTERCEPT_FUNCTION(cdbr_entries); \ + COMMON_INTERCEPT_FUNCTION(cdbr_get); \ + COMMON_INTERCEPT_FUNCTION(cdbr_find); \ + COMMON_INTERCEPT_FUNCTION(cdbr_close); \ + COMMON_INTERCEPT_FUNCTION(cdbw_open); \ + COMMON_INTERCEPT_FUNCTION(cdbw_put); \ + COMMON_INTERCEPT_FUNCTION(cdbw_put_data); \ + COMMON_INTERCEPT_FUNCTION(cdbw_put_key); \ + COMMON_INTERCEPT_FUNCTION(cdbw_output); \ + COMMON_INTERCEPT_FUNCTION(cdbw_close) +#else +#define INIT_CDB +#endif + +#if SANITIZER_INTERCEPT_GETFSENT +INTERCEPTOR(void *, getfsent) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, getfsent); + void *ret = REAL(getfsent)(); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, struct_fstab_sz); + return ret; +} + +INTERCEPTOR(void *, getfsspec, const char *spec) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, getfsspec, spec); + if (spec) + COMMON_INTERCEPTOR_READ_RANGE(ctx, spec, REAL(strlen)(spec) + 1); + void *ret = REAL(getfsspec)(spec); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, struct_fstab_sz); + return ret; +} + +INTERCEPTOR(void *, getfsfile, const char *file) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, getfsfile, file); + if (file) + COMMON_INTERCEPTOR_READ_RANGE(ctx, file, REAL(strlen)(file) + 1); + void *ret = REAL(getfsfile)(file); + if (ret) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, struct_fstab_sz); + return ret; +} + +#define INIT_GETFSENT \ + COMMON_INTERCEPT_FUNCTION(getfsent); \ + COMMON_INTERCEPT_FUNCTION(getfsspec); \ + COMMON_INTERCEPT_FUNCTION(getfsfile); +#else +#define INIT_GETFSENT +#endif + +#if SANITIZER_INTERCEPT_ARC4RANDOM +INTERCEPTOR(void, arc4random_buf, void *buf, SIZE_T len) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, arc4random_buf, buf, len); + REAL(arc4random_buf)(buf, len); + if (buf && len) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, len); +} + +INTERCEPTOR(void, arc4random_addrandom, u8 *dat, int datlen) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, arc4random_addrandom, dat, datlen); + if (dat && datlen) + COMMON_INTERCEPTOR_READ_RANGE(ctx, dat, datlen); + REAL(arc4random_addrandom)(dat, datlen); +} + +#define INIT_ARC4RANDOM \ + COMMON_INTERCEPT_FUNCTION(arc4random_buf); \ + COMMON_INTERCEPT_FUNCTION(arc4random_addrandom); +#else +#define INIT_ARC4RANDOM +#endif + +#if SANITIZER_INTERCEPT_POPEN +INTERCEPTOR(__sanitizer_FILE *, popen, const char *command, const char *type) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, popen, command, type); + if (command) + COMMON_INTERCEPTOR_READ_RANGE(ctx, command, REAL(strlen)(command) + 1); + if (type) + COMMON_INTERCEPTOR_READ_RANGE(ctx, type, REAL(strlen)(type) + 1); + __sanitizer_FILE *res = REAL(popen)(command, type); + COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, nullptr); + if (res) unpoison_file(res); + return res; +} +#define INIT_POPEN COMMON_INTERCEPT_FUNCTION(popen) +#else +#define INIT_POPEN +#endif + +#if SANITIZER_INTERCEPT_POPENVE +INTERCEPTOR(__sanitizer_FILE *, popenve, const char *path, + char *const *argv, char *const *envp, const char *type) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, popenve, path, argv, envp, type); + if (path) + COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); + if (argv) { + for (char *const *pa = argv; ; ++pa) { + COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, sizeof(char **)); + if (!*pa) + break; + COMMON_INTERCEPTOR_READ_RANGE(ctx, *pa, REAL(strlen)(*pa) + 1); + } + } + if (envp) { + for (char *const *pa = envp; ; ++pa) { + COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, sizeof(char **)); + if (!*pa) + break; + COMMON_INTERCEPTOR_READ_RANGE(ctx, *pa, REAL(strlen)(*pa) + 1); + } + } + if (type) + COMMON_INTERCEPTOR_READ_RANGE(ctx, type, REAL(strlen)(type) + 1); + __sanitizer_FILE *res = REAL(popenve)(path, argv, envp, type); + COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, nullptr); + if (res) unpoison_file(res); + return res; +} +#define INIT_POPENVE COMMON_INTERCEPT_FUNCTION(popenve) +#else +#define INIT_POPENVE +#endif + +#if SANITIZER_INTERCEPT_PCLOSE +INTERCEPTOR(int, pclose, __sanitizer_FILE *fp) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, pclose, fp); + COMMON_INTERCEPTOR_FILE_CLOSE(ctx, fp); + const FileMetadata *m = GetInterceptorMetadata(fp); + int res = REAL(pclose)(fp); + if (m) { + COMMON_INTERCEPTOR_INITIALIZE_RANGE(*m->addr, *m->size); + DeleteInterceptorMetadata(fp); + } + return res; +} +#define INIT_PCLOSE COMMON_INTERCEPT_FUNCTION(pclose); +#else +#define INIT_PCLOSE +#endif + +#if SANITIZER_INTERCEPT_FUNOPEN +typedef int (*funopen_readfn)(void *cookie, char *buf, int len); +typedef int (*funopen_writefn)(void *cookie, const char *buf, int len); +typedef OFF_T (*funopen_seekfn)(void *cookie, OFF_T offset, int whence); +typedef int (*funopen_closefn)(void *cookie); + +struct WrappedFunopenCookie { + void *real_cookie; + funopen_readfn real_read; + funopen_writefn real_write; + funopen_seekfn real_seek; + funopen_closefn real_close; +}; + +static int wrapped_funopen_read(void *cookie, char *buf, int len) { + COMMON_INTERCEPTOR_UNPOISON_PARAM(3); + WrappedFunopenCookie *wrapped_cookie = (WrappedFunopenCookie *)cookie; + funopen_readfn real_read = wrapped_cookie->real_read; + return real_read(wrapped_cookie->real_cookie, buf, len); +} + +static int wrapped_funopen_write(void *cookie, const char *buf, int len) { + COMMON_INTERCEPTOR_UNPOISON_PARAM(3); + WrappedFunopenCookie *wrapped_cookie = (WrappedFunopenCookie *)cookie; + funopen_writefn real_write = wrapped_cookie->real_write; + return real_write(wrapped_cookie->real_cookie, buf, len); +} + +static OFF_T wrapped_funopen_seek(void *cookie, OFF_T offset, int whence) { + COMMON_INTERCEPTOR_UNPOISON_PARAM(3); + WrappedFunopenCookie *wrapped_cookie = (WrappedFunopenCookie *)cookie; + funopen_seekfn real_seek = wrapped_cookie->real_seek; + return real_seek(wrapped_cookie->real_cookie, offset, whence); +} + +static int wrapped_funopen_close(void *cookie) { + COMMON_INTERCEPTOR_UNPOISON_PARAM(1); + WrappedFunopenCookie *wrapped_cookie = (WrappedFunopenCookie *)cookie; + funopen_closefn real_close = wrapped_cookie->real_close; + int res = real_close(wrapped_cookie->real_cookie); + InternalFree(wrapped_cookie); + return res; +} + +INTERCEPTOR(__sanitizer_FILE *, funopen, void *cookie, funopen_readfn readfn, + funopen_writefn writefn, funopen_seekfn seekfn, + funopen_closefn closefn) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, funopen, cookie, readfn, writefn, seekfn, + closefn); + + WrappedFunopenCookie *wrapped_cookie = + (WrappedFunopenCookie *)InternalAlloc(sizeof(WrappedFunopenCookie)); + wrapped_cookie->real_cookie = cookie; + wrapped_cookie->real_read = readfn; + wrapped_cookie->real_write = writefn; + wrapped_cookie->real_seek = seekfn; + wrapped_cookie->real_close = closefn; + + __sanitizer_FILE *res = + REAL(funopen)(wrapped_cookie, + readfn ? wrapped_funopen_read : nullptr, + writefn ? wrapped_funopen_write : nullptr, + seekfn ? wrapped_funopen_seek : nullptr, + closefn ? wrapped_funopen_close : nullptr); + if (res) + unpoison_file(res); + return res; +} +#define INIT_FUNOPEN COMMON_INTERCEPT_FUNCTION(funopen) +#else +#define INIT_FUNOPEN +#endif + +#if SANITIZER_INTERCEPT_FUNOPEN2 +typedef SSIZE_T (*funopen2_readfn)(void *cookie, void *buf, SIZE_T len); +typedef SSIZE_T (*funopen2_writefn)(void *cookie, const void *buf, SIZE_T len); +typedef OFF_T (*funopen2_seekfn)(void *cookie, OFF_T offset, int whence); +typedef int (*funopen2_flushfn)(void *cookie); +typedef int (*funopen2_closefn)(void *cookie); + +struct WrappedFunopen2Cookie { + void *real_cookie; + funopen2_readfn real_read; + funopen2_writefn real_write; + funopen2_seekfn real_seek; + funopen2_flushfn real_flush; + funopen2_closefn real_close; +}; + +static SSIZE_T wrapped_funopen2_read(void *cookie, void *buf, SIZE_T len) { + COMMON_INTERCEPTOR_UNPOISON_PARAM(3); + WrappedFunopen2Cookie *wrapped_cookie = (WrappedFunopen2Cookie *)cookie; + funopen2_readfn real_read = wrapped_cookie->real_read; + return real_read(wrapped_cookie->real_cookie, buf, len); +} + +static SSIZE_T wrapped_funopen2_write(void *cookie, const void *buf, + SIZE_T len) { + COMMON_INTERCEPTOR_UNPOISON_PARAM(3); + WrappedFunopen2Cookie *wrapped_cookie = (WrappedFunopen2Cookie *)cookie; + funopen2_writefn real_write = wrapped_cookie->real_write; + return real_write(wrapped_cookie->real_cookie, buf, len); +} + +static OFF_T wrapped_funopen2_seek(void *cookie, OFF_T offset, int whence) { + COMMON_INTERCEPTOR_UNPOISON_PARAM(3); + WrappedFunopen2Cookie *wrapped_cookie = (WrappedFunopen2Cookie *)cookie; + funopen2_seekfn real_seek = wrapped_cookie->real_seek; + return real_seek(wrapped_cookie->real_cookie, offset, whence); +} + +static int wrapped_funopen2_flush(void *cookie) { + COMMON_INTERCEPTOR_UNPOISON_PARAM(1); + WrappedFunopen2Cookie *wrapped_cookie = (WrappedFunopen2Cookie *)cookie; + funopen2_flushfn real_flush = wrapped_cookie->real_flush; + return real_flush(wrapped_cookie->real_cookie); +} + +static int wrapped_funopen2_close(void *cookie) { + COMMON_INTERCEPTOR_UNPOISON_PARAM(1); + WrappedFunopen2Cookie *wrapped_cookie = (WrappedFunopen2Cookie *)cookie; + funopen2_closefn real_close = wrapped_cookie->real_close; + int res = real_close(wrapped_cookie->real_cookie); + InternalFree(wrapped_cookie); + return res; +} + +INTERCEPTOR(__sanitizer_FILE *, funopen2, void *cookie, funopen2_readfn readfn, + funopen2_writefn writefn, funopen2_seekfn seekfn, + funopen2_flushfn flushfn, funopen2_closefn closefn) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, funopen2, cookie, readfn, writefn, seekfn, + flushfn, closefn); + + WrappedFunopen2Cookie *wrapped_cookie = + (WrappedFunopen2Cookie *)InternalAlloc(sizeof(WrappedFunopen2Cookie)); + wrapped_cookie->real_cookie = cookie; + wrapped_cookie->real_read = readfn; + wrapped_cookie->real_write = writefn; + wrapped_cookie->real_seek = seekfn; + wrapped_cookie->real_flush = flushfn; + wrapped_cookie->real_close = closefn; + + __sanitizer_FILE *res = + REAL(funopen2)(wrapped_cookie, + readfn ? wrapped_funopen2_read : nullptr, + writefn ? wrapped_funopen2_write : nullptr, + seekfn ? wrapped_funopen2_seek : nullptr, + flushfn ? wrapped_funopen2_flush : nullptr, + closefn ? wrapped_funopen2_close : nullptr); + if (res) + unpoison_file(res); + return res; +} +#define INIT_FUNOPEN2 COMMON_INTERCEPT_FUNCTION(funopen2) +#else +#define INIT_FUNOPEN2 +#endif + +#if SANITIZER_INTERCEPT_FDEVNAME +INTERCEPTOR(char *, fdevname, int fd) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fdevname, fd); + COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); + char *name = REAL(fdevname)(fd); + if (name) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, REAL(strlen)(name) + 1); + if (fd > 0) + COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd); + } + return name; +} + +INTERCEPTOR(char *, fdevname_r, int fd, char *buf, SIZE_T len) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, fdevname_r, fd, buf, len); + COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); + char *name = REAL(fdevname_r)(fd, buf, len); + if (name && buf && len > 0) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, REAL(strlen)(buf) + 1); + if (fd > 0) + COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd); + } + return name; +} + +#define INIT_FDEVNAME \ + COMMON_INTERCEPT_FUNCTION(fdevname); \ + COMMON_INTERCEPT_FUNCTION(fdevname_r); +#else +#define INIT_FDEVNAME +#endif + +#if SANITIZER_INTERCEPT_GETUSERSHELL +INTERCEPTOR(char *, getusershell) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, getusershell); + char *res = REAL(getusershell)(); + if (res) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1); + return res; +} + +#define INIT_GETUSERSHELL COMMON_INTERCEPT_FUNCTION(getusershell); +#else +#define INIT_GETUSERSHELL +#endif + +#if SANITIZER_INTERCEPT_SL_INIT +INTERCEPTOR(void *, sl_init) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, sl_init); + void *res = REAL(sl_init)(); + if (res) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, __sanitizer::struct_StringList_sz); + return res; +} + +INTERCEPTOR(int, sl_add, void *sl, char *item) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, sl_add, sl, item); + if (sl) + COMMON_INTERCEPTOR_READ_RANGE(ctx, sl, __sanitizer::struct_StringList_sz); + if (item) + COMMON_INTERCEPTOR_READ_RANGE(ctx, item, REAL(strlen)(item) + 1); + int res = REAL(sl_add)(sl, item); + if (!res) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sl, __sanitizer::struct_StringList_sz); + return res; +} + +INTERCEPTOR(char *, sl_find, void *sl, const char *item) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, sl_find, sl, item); + if (sl) + COMMON_INTERCEPTOR_READ_RANGE(ctx, sl, __sanitizer::struct_StringList_sz); + if (item) + COMMON_INTERCEPTOR_READ_RANGE(ctx, item, REAL(strlen)(item) + 1); + char *res = REAL(sl_find)(sl, item); + if (res) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1); + return res; +} + +INTERCEPTOR(void, sl_free, void *sl, int freeall) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, sl_free, sl, freeall); + if (sl) + COMMON_INTERCEPTOR_READ_RANGE(ctx, sl, __sanitizer::struct_StringList_sz); + REAL(sl_free)(sl, freeall); +} + +#define INIT_SL_INIT \ + COMMON_INTERCEPT_FUNCTION(sl_init); \ + COMMON_INTERCEPT_FUNCTION(sl_add); \ + COMMON_INTERCEPT_FUNCTION(sl_find); \ + COMMON_INTERCEPT_FUNCTION(sl_free); +#else +#define INIT_SL_INIT +#endif + static void InitializeCommonInterceptors() { static u64 metadata_mem[sizeof(MetadataHashMap) / sizeof(u64) + 1]; - interceptor_metadata_map = new((void *)&metadata_mem) MetadataHashMap(); + interceptor_metadata_map = + new ((void *)&metadata_mem) MetadataHashMap(); // NOLINT INIT_MMAP; INIT_MMAP64; @@ -7299,6 +9532,8 @@ static void InitializeCommonInterceptors() { INIT_GETPWENT; INIT_FGETPWENT; INIT_GETPWENT_R; + INIT_FGETPWENT_R; + INIT_FGETGRENT_R; INIT_SETPWENT; INIT_CLOCK_GETTIME; INIT_GETITIMER; @@ -7408,7 +9643,9 @@ static void InitializeCommonInterceptors() { INIT_PTHREAD_GETNAME_NP; INIT_SINCOS; INIT_REMQUO; + INIT_REMQUOL; INIT_LGAMMA; + INIT_LGAMMAL; INIT_LGAMMA_R; INIT_LGAMMAL_R; INIT_DRAND48_R; @@ -7484,6 +9721,42 @@ static void InitializeCommonInterceptors() { INIT_TTYENT; INIT_PROTOENT; INIT_NETENT; + INIT_GETMNTINFO; + INIT_MI_VECTOR_HASH; + INIT_SETVBUF; + INIT_GETVFSSTAT; + INIT_REGEX; + INIT_REGEXSUB; + INIT_FTS; + INIT_SYSCTL; + INIT_ASYSCTL; + INIT_SYSCTLGETMIBINFO; + INIT_NL_LANGINFO; + INIT_MODCTL; + INIT_STRTONUM; + INIT_FPARSELN; + INIT_STATVFS1; + INIT_STRTOI; + INIT_CAPSICUM; + INIT_SHA1; + INIT_MD4; + INIT_RMD160; + INIT_MD5; + INIT_FSEEK; + INIT_MD2; + INIT_SHA2; + INIT_VIS; + INIT_CDB; + INIT_GETFSENT; + INIT_ARC4RANDOM; + INIT_POPEN; + INIT_POPENVE; + INIT_PCLOSE; + INIT_FUNOPEN; + INIT_FUNOPEN2; + INIT_FDEVNAME; + INIT_GETUSERSHELL; + INIT_SL_INIT; INIT___PRINTF_CHK; } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc index 796a054858bc..1c0995b2d27b 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc @@ -25,7 +25,7 @@ void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded)) { SoftRssLimitExceededCallback = Callback; } -#if SANITIZER_LINUX && !SANITIZER_GO +#if (SANITIZER_LINUX || SANITIZER_NETBSD) && !SANITIZER_GO // Weak default implementation for when sanitizer_stackdepot is not linked in. SANITIZER_WEAK_ATTRIBUTE StackDepotStats *StackDepotGetStats() { return nullptr; @@ -100,18 +100,21 @@ void WriteToSyslog(const char *msg) { // Print one line at a time. // syslog, at least on Android, has an implicit message length limit. - do { - q = internal_strchr(p, '\n'); - if (q) - *q = '\0'; + while ((q = internal_strchr(p, '\n'))) { + *q = '\0'; + WriteOneLineToSyslog(p); + p = q + 1; + } + // Print remaining characters, if there are any. + // Note that this will add an extra newline at the end. + // FIXME: buffer extra output. This would need a thread-local buffer, which + // on Android requires plugging into the tools (ex. ASan's) Thread class. + if (*p) WriteOneLineToSyslog(p); - if (q) - p = q + 1; - } while (q); } void MaybeStartBackgroudThread() { -#if SANITIZER_LINUX && \ +#if (SANITIZER_LINUX || SANITIZER_NETBSD) && \ !SANITIZER_GO // Need to implement/test on other platforms. // Start the background thread if one of the rss limits is given. if (!common_flags()->hard_rss_limit_mb && diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc index b4ffcca5c30b..9ff8798a5731 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc @@ -31,6 +31,7 @@ #include "sanitizer_atomic.h" #include "sanitizer_common.h" #include "sanitizer_internal_defs.h" +#include "sanitizer_symbolizer_fuchsia.h" #include <zircon/process.h> #include <zircon/sanitizer.h> @@ -101,7 +102,7 @@ class TracePcGuardController final { // uses the `dumpfile` symbolizer markup element to highlight the // dump. See the explanation for this in: // https://fuchsia.googlesource.com/zircon/+/master/docs/symbolizer_markup.md - Printf("SanitizerCoverage: {{{dumpfile:%s:%s}}} with up to %u PCs\n", + Printf("SanitizerCoverage: " FORMAT_DUMPFILE " with up to %u PCs\n", kSancovSinkName, vmo_name_, next_index_ - 1); } } @@ -146,9 +147,9 @@ class TracePcGuardController final { // indices, but we'll never move the mapping address so we don't have // any multi-thread synchronization issues with that. uintptr_t mapping; - status = _zx_vmar_map_old(_zx_vmar_root_self(), 0, vmo_, 0, MappingSize, - ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE, - &mapping); + status = + _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, + 0, vmo_, 0, MappingSize, &mapping); CHECK_EQ(status, ZX_OK); // Hereafter other threads are free to start storing into diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_win_sections.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_win_sections.cc index 4b0bbf1ed1e5..108f76effeeb 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_win_sections.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_win_sections.cc @@ -7,16 +7,57 @@ // //===----------------------------------------------------------------------===// // -// This file defines delimiters for Sanitizer Coverage's section. +// This file defines delimiters for Sanitizer Coverage's section. It contains +// Windows specific tricks to coax the linker into giving us the start and stop +// addresses of a section, as ELF linkers can do, to get the size of certain +// arrays. According to https://msdn.microsoft.com/en-us/library/7977wcck.aspx +// sections with the same name before "$" are sorted alphabetically by the +// string that comes after "$" and merged into one section. We take advantage +// of this by putting data we want the size of into the middle (M) of a section, +// by using the letter "M" after "$". We get the start of this data (ie: +// __start_section_name) by making the start variable come at the start of the +// section (using the letter A after "$"). We do the same to get the end of the +// data by using the letter "Z" after "$" to make the end variable come after +// the data. Note that because of our technique the address of the start +// variable is actually the address of data that comes before our middle +// section. We also need to prevent the linker from adding any padding. Each +// technique we use for this is explained in the comments below. //===----------------------------------------------------------------------===// #include "sanitizer_platform.h" #if SANITIZER_WINDOWS #include <stdint.h> -#pragma section(".SCOV$A", read, write) // NOLINT -#pragma section(".SCOV$Z", read, write) // NOLINT extern "C" { -__declspec(allocate(".SCOV$A")) uint32_t __start___sancov_guards = 0; -__declspec(allocate(".SCOV$Z")) uint32_t __stop___sancov_guards = 0; +// The Guard array and counter array should both be merged into the .data +// section to reduce the number of PE sections However, because PCTable is +// constant it should be merged with the .rdata section. +#pragma section(".SCOV$GA", read, write) // NOLINT +// Use align(1) to avoid adding any padding that will mess up clients trying to +// determine the start and end of the array. +__declspec(allocate(".SCOV$GA")) __declspec(align(1)) uint64_t + __start___sancov_guards = 0; +#pragma section(".SCOV$GZ", read, write) // NOLINT +__declspec(allocate(".SCOV$GZ")) __declspec(align(1)) uint64_t + __stop___sancov_guards = 0; + +#pragma section(".SCOV$CA", read, write) // NOLINT +__declspec(allocate(".SCOV$CA")) __declspec(align(1)) uint64_t + __start___sancov_cntrs = 0; +#pragma section(".SCOV$CZ", read, write) // NOLINT +__declspec(allocate(".SCOV$CZ")) __declspec(align(1)) uint64_t + __stop___sancov_cntrs = 0; + +#pragma comment(linker, "/MERGE:.SCOV=.data") + +// Use uint64_t so there won't be any issues if the linker tries to word align +// the pc array. +#pragma section(".SCOVP$A", read) // NOLINT +__declspec(allocate(".SCOVP$A")) __declspec(align(1)) uint64_t + __start___sancov_pcs = 0; +#pragma section(".SCOVP$Z", read) // NOLINT +__declspec(allocate(".SCOVP$Z")) __declspec(align(1)) uint64_t + __stop___sancov_pcs = 0; + +#pragma comment(linker, "/MERGE:.SCOVP=.rdata") } -#endif // SANITIZER_WINDOWS +#endif // SANITIZER_WINDOWS diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.h index 9a12ab7342c7..52e6ef8fb1ac 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.h @@ -66,9 +66,6 @@ bool ReadFromFile(fd_t fd, void *buff, uptr buff_size, bool WriteToFile(fd_t fd, const void *buff, uptr buff_size, uptr *bytes_written = nullptr, error_t *error_p = nullptr); -bool RenameFile(const char *oldpath, const char *newpath, - error_t *error_p = nullptr); - // Scoped file handle closer. struct FileCloser { explicit FileCloser(fd_t fd) : fd(fd) {} diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc index cfe8af893423..b047741b9083 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc @@ -103,7 +103,7 @@ COMMON_FLAG(bool, allow_user_segv_handler, true, "handle_*=1 will be upgraded to handle_*=2.") COMMON_FLAG(bool, use_sigaltstack, true, "If set, uses alternate stack for signal handling.") -COMMON_FLAG(bool, detect_deadlocks, false, +COMMON_FLAG(bool, detect_deadlocks, true, "If set, deadlock detection is enabled.") COMMON_FLAG( uptr, clear_shadow_mmap_threshold, 64 * 1024, @@ -243,3 +243,6 @@ COMMON_FLAG(bool, dump_registers, true, COMMON_FLAG(bool, detect_write_exec, false, "If true, triggers warning when writable-executable pages requests " "are being made") +COMMON_FLAG(bool, test_only_emulate_no_memorymap, false, + "TEST ONLY fail to read memory mappings to emulate sanitized " + "\"init\"") diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc index 391620690f30..2c259b6cd7da 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc @@ -29,9 +29,6 @@ namespace __sanitizer { -// TODO(phosek): remove this and replace it with ZX_TIME_INFINITE -#define ZX_TIME_INFINITE_OLD INT64_MAX - void NORETURN internal__exit(int exitcode) { _zx_process_exit(exitcode); } uptr internal_sched_yield() { @@ -89,8 +86,10 @@ void GetThreadStackTopAndBottom(bool, uptr *stack_top, uptr *stack_bottom) { *stack_top = *stack_bottom + size; } +void InitializePlatformEarly() {} void MaybeReexec() {} void CheckASLR() {} +void CheckMPROTECT() {} void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {} void DisableCoreDumperIfNecessary() {} void InstallDeadlySignalHandlers(SignalHandlerType handler) {} @@ -122,8 +121,9 @@ void BlockingMutex::Lock() { if (atomic_exchange(m, MtxLocked, memory_order_acquire) == MtxUnlocked) return; while (atomic_exchange(m, MtxSleeping, memory_order_acquire) != MtxUnlocked) { - zx_status_t status = _zx_futex_wait(reinterpret_cast<zx_futex_t *>(m), - MtxSleeping, ZX_TIME_INFINITE_OLD); + zx_status_t status = + _zx_futex_wait(reinterpret_cast<zx_futex_t *>(m), MtxSleeping, + ZX_HANDLE_INVALID, ZX_TIME_INFINITE); if (status != ZX_ERR_BAD_STATE) // Normal race. CHECK_EQ(status, ZX_OK); } @@ -175,8 +175,8 @@ static void *DoAnonymousMmapOrDie(uptr size, const char *mem_type, // TODO(mcgrathr): Maybe allocate a VMAR for all sanitizer heap and use that? uintptr_t addr; status = - _zx_vmar_map_old(_zx_vmar_root_self(), 0, vmo, 0, size, - ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE, &addr); + _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0, + vmo, 0, size, &addr); _zx_handle_close(vmo); if (status != ZX_OK) { @@ -210,10 +210,10 @@ uptr ReservedAddressRange::Init(uptr init_size, const char *name, uintptr_t base; zx_handle_t vmar; zx_status_t status = - _zx_vmar_allocate_old(_zx_vmar_root_self(), 0, init_size, - ZX_VM_FLAG_CAN_MAP_READ | ZX_VM_FLAG_CAN_MAP_WRITE | - ZX_VM_FLAG_CAN_MAP_SPECIFIC, - &vmar, &base); + _zx_vmar_allocate( + _zx_vmar_root_self(), + ZX_VM_CAN_MAP_READ | ZX_VM_CAN_MAP_WRITE | ZX_VM_CAN_MAP_SPECIFIC, + 0, init_size, &vmar, &base); if (status != ZX_OK) ReportMmapFailureAndDie(init_size, name, "zx_vmar_allocate", status); base_ = reinterpret_cast<void *>(base); @@ -239,10 +239,9 @@ static uptr DoMmapFixedOrDie(zx_handle_t vmar, uptr fixed_addr, uptr map_size, DCHECK_GE(base + size_, map_size + offset); uintptr_t addr; - status = _zx_vmar_map_old( - vmar, offset, vmo, 0, map_size, - ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE | ZX_VM_FLAG_SPECIFIC, - &addr); + status = + _zx_vmar_map(vmar, ZX_VM_PERM_READ | ZX_VM_PERM_WRITE | ZX_VM_SPECIFIC, + offset, vmo, 0, map_size, &addr); _zx_handle_close(vmo); if (status != ZX_OK) { if (status != ZX_ERR_NO_MEMORY || die_for_nomem) { @@ -281,14 +280,22 @@ void UnmapOrDieVmar(void *addr, uptr size, zx_handle_t target_vmar) { void ReservedAddressRange::Unmap(uptr addr, uptr size) { CHECK_LE(size, size_); - if (addr == reinterpret_cast<uptr>(base_)) - // If we unmap the whole range, just null out the base. - base_ = (size == size_) ? nullptr : reinterpret_cast<void*>(addr + size); - else + const zx_handle_t vmar = static_cast<zx_handle_t>(os_handle_); + if (addr == reinterpret_cast<uptr>(base_)) { + if (size == size_) { + // Destroying the vmar effectively unmaps the whole mapping. + _zx_vmar_destroy(vmar); + _zx_handle_close(vmar); + os_handle_ = static_cast<uptr>(ZX_HANDLE_INVALID); + DecreaseTotalMmap(size); + return; + } + } else { CHECK_EQ(addr + size, reinterpret_cast<uptr>(base_) + size_); - size_ -= size; - UnmapOrDieVmar(reinterpret_cast<void *>(addr), size, - static_cast<zx_handle_t>(os_handle_)); + } + // Partial unmapping does not affect the fact that the initial range is still + // reserved, and the resulting unmapped memory can't be reused. + UnmapOrDieVmar(reinterpret_cast<void *>(addr), size, vmar); } // This should never be called. @@ -321,8 +328,8 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment, size_t map_size = size + alignment; uintptr_t addr; status = - _zx_vmar_map_old(_zx_vmar_root_self(), 0, vmo, 0, map_size, - ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE, &addr); + _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0, + vmo, 0, map_size, &addr); if (status == ZX_OK) { uintptr_t map_addr = addr; uintptr_t map_end = map_addr + map_size; @@ -334,11 +341,10 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment, sizeof(info), NULL, NULL); if (status == ZX_OK) { uintptr_t new_addr; - status = _zx_vmar_map_old(_zx_vmar_root_self(), addr - info.base, vmo, - 0, size, - ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE | - ZX_VM_FLAG_SPECIFIC_OVERWRITE, - &new_addr); + status = _zx_vmar_map( + _zx_vmar_root_self(), + ZX_VM_PERM_READ | ZX_VM_PERM_WRITE | ZX_VM_SPECIFIC_OVERWRITE, + addr - info.base, vmo, 0, size, &new_addr); if (status == ZX_OK) CHECK_EQ(new_addr, addr); } } @@ -398,8 +404,8 @@ bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size, if (vmo_size < max_len) max_len = vmo_size; size_t map_size = RoundUpTo(max_len, PAGE_SIZE); uintptr_t addr; - status = _zx_vmar_map_old(_zx_vmar_root_self(), 0, vmo, 0, map_size, - ZX_VM_FLAG_PERM_READ, &addr); + status = _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ, 0, vmo, 0, + map_size, &addr); if (status == ZX_OK) { *buff = reinterpret_cast<char *>(addr); *buff_size = map_size; @@ -448,6 +454,7 @@ char **StoredArgv; char **StoredEnviron; char **GetArgv() { return StoredArgv; } +char **GetEnviron() { return StoredEnviron; } const char *GetEnv(const char *name) { if (StoredEnviron) { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc index 9a2a9f120684..86cb44047606 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc @@ -25,7 +25,7 @@ struct ioctl_desc { const char *name; }; -const unsigned ioctl_table_max = 1198; +const unsigned ioctl_table_max = 1202; static ioctl_desc ioctl_table[ioctl_table_max]; static unsigned ioctl_table_size = 0; @@ -288,6 +288,8 @@ static void ioctl_table_fill() { _(MLX_GET_CINFO, WRITE, struct_mlx_cinfo_sz); /* Entries from file: dev/ic/nvmeio.h */ _(NVME_PASSTHROUGH_CMD, READWRITE, struct_nvme_pt_command_sz); + /* Entries from file: dev/ic/qemufwcfgio.h */ + _(FWCFGIO_SET_INDEX, READ, sizeof(u16)); /* Entries from file: dev/ir/irdaio.h */ _(IRDA_RESET_PARAMS, NONE, 0); _(IRDA_SET_PARAMS, READ, struct_irda_params_sz); @@ -647,6 +649,26 @@ static void ioctl_table_fill() { _(SPKRTUNE, NONE, 0); _(SPKRGETVOL, WRITE, sizeof(unsigned int)); _(SPKRSETVOL, READ, sizeof(unsigned int)); + /* Entries from file: dev/nvmm/nvmm_ioctl.h */ +#if 0 /* WIP */ + _(NVMM_IOC_CAPABILITY, WRITE, struct_nvmm_ioc_capability_sz); + _(NVMM_IOC_MACHINE_CREATE, READWRITE, struct_nvmm_ioc_machine_create_sz); + _(NVMM_IOC_MACHINE_DESTROY, READ, struct_nvmm_ioc_machine_destroy_sz); + _(NVMM_IOC_MACHINE_CONFIGURE, READ, struct_nvmm_ioc_machine_configure_sz); + _(NVMM_IOC_VCPU_CREATE, READ, struct_nvmm_ioc_vcpu_create_sz); + _(NVMM_IOC_VCPU_DESTROY, READ, struct_nvmm_ioc_vcpu_destroy_sz); + _(NVMM_IOC_VCPU_SETSTATE, READ, struct_nvmm_ioc_vcpu_setstate_sz); + _(NVMM_IOC_VCPU_GETSTATE, READ, struct_nvmm_ioc_vcpu_getstate_sz); + _(NVMM_IOC_VCPU_INJECT, READWRITE, struct_nvmm_ioc_vcpu_inject_sz); + _(NVMM_IOC_VCPU_RUN, READWRITE, struct_nvmm_ioc_vcpu_run_sz); + _(NVMM_IOC_GPA_MAP, READ, struct_nvmm_ioc_gpa_map_sz); + _(NVMM_IOC_GPA_UNMAP, READ, struct_nvmm_ioc_gpa_unmap_sz); + _(NVMM_IOC_HVA_MAP, READ, struct_nvmm_ioc_hva_map_sz); + _(NVMM_IOC_HVA_UNMAP, READ, struct_nvmm_ioc_hva_unmap_sz); +#endif + /* Entries from file: fs/autofs/autofs_ioctl.h */ + _(AUTOFSREQUEST, WRITE, struct_autofs_daemon_request_sz); + _(AUTOFSDONE, READ, struct_autofs_daemon_done_sz); /* Entries from file: net/bpf.h */ _(BIOCGBLEN, WRITE, sizeof(unsigned int)); _(BIOCSBLEN, READWRITE, sizeof(unsigned int)); @@ -666,20 +688,12 @@ static void ioctl_table_fill() { _(BIOCSHDRCMPLT, READ, sizeof(unsigned int)); _(BIOCSDLT, READ, sizeof(unsigned int)); _(BIOCGDLTLIST, READWRITE, struct_bpf_dltlist_sz); - _(BIOCGSEESENT, WRITE, sizeof(unsigned int)); - _(BIOCSSEESENT, READ, sizeof(unsigned int)); + _(BIOCGDIRECTION, WRITE, sizeof(unsigned int)); + _(BIOCSDIRECTION, READ, sizeof(unsigned int)); _(BIOCSRTIMEOUT, READ, struct_timeval_sz); _(BIOCGRTIMEOUT, WRITE, struct_timeval_sz); _(BIOCGFEEDBACK, WRITE, sizeof(unsigned int)); _(BIOCSFEEDBACK, READ, sizeof(unsigned int)); - /* Entries from file: net/if_atm.h */ - _(SIOCRAWATM, READWRITE, sizeof(int)); - _(SIOCATMENA, READWRITE, struct_atm_pseudoioctl_sz); - _(SIOCATMDIS, READWRITE, struct_atm_pseudoioctl_sz); - _(SIOCSPVCTX, READWRITE, struct_pvctxreq_sz); - _(SIOCGPVCTX, READWRITE, struct_pvctxreq_sz); - _(SIOCSPVCSIF, READWRITE, struct_ifreq_sz); - _(SIOCGPVCSIF, READWRITE, struct_ifreq_sz); /* Entries from file: net/if_gre.h */ _(GRESADDRS, READ, struct_ifreq_sz); _(GRESADDRD, READ, struct_ifreq_sz); @@ -715,12 +729,12 @@ static void ioctl_table_fill() { /* Entries from file: net/npf.h */ _(IOC_NPF_VERSION, WRITE, sizeof(int)); _(IOC_NPF_SWITCH, READ, sizeof(int)); - _(IOC_NPF_LOAD, READWRITE, struct_plistref_sz); + _(IOC_NPF_LOAD, READWRITE, struct_nvlist_ref_sz); _(IOC_NPF_TABLE, READ, struct_npf_ioctl_table_sz); _(IOC_NPF_STATS, READ, sizeof(uptr)); - _(IOC_NPF_SAVE, WRITE, struct_plistref_sz); - _(IOC_NPF_RULE, READWRITE, struct_plistref_sz); - _(IOC_NPF_CONN_LOOKUP, READWRITE, struct_plistref_sz); + _(IOC_NPF_SAVE, WRITE, struct_nvlist_ref_sz); + _(IOC_NPF_RULE, READWRITE, struct_nvlist_ref_sz); + _(IOC_NPF_CONN_LOOKUP, READWRITE, struct_nvlist_ref_sz); /* Entries from file: net/if_pppoe.h */ _(PPPOESETPARMS, READ, struct_pppoediscparms_sz); _(PPPOEGETPARMS, READWRITE, struct_pppoediscparms_sz); @@ -843,6 +857,9 @@ static void ioctl_table_fill() { _(SIOCGNATS, READWRITE, struct_ipfobj_sz); _(SIOCGNATL, READWRITE, struct_ipfobj_sz); _(SIOCPURGENAT, READWRITE, struct_ipfobj_sz); + /* Entries from file: netinet/sctp_uio.h */ + _(SIOCCONNECTX, READWRITE, struct_sctp_connectx_addrs_sz); + _(SIOCCONNECTXDEL, READWRITE, struct_sctp_connectx_addrs_sz); /* Entries from file: netinet6/in6_var.h */ _(SIOCSIFINFO_FLAGS, READWRITE, struct_in6_ndireq_sz); _(SIOCAADDRCTL_POLICY, READ, struct_in6_addrpolicy_sz); @@ -1002,6 +1019,8 @@ static void ioctl_table_fill() { /* Entries from file: sys/filio.h */ _(FIOCLEX, NONE, 0); _(FIONCLEX, NONE, 0); + _(FIOSEEKDATA, READWRITE, sizeof(uptr)); + _(FIOSEEKHOLE, READWRITE, sizeof(uptr)); _(FIONREAD, WRITE, sizeof(int)); _(FIONBIO, READ, sizeof(int)); _(FIOASYNC, READ, sizeof(int)); @@ -1095,7 +1114,6 @@ static void ioctl_table_fill() { /* Entries from file: sys/power.h */ _(POWER_EVENT_RECVDICT, READWRITE, struct_plistref_sz); _(POWER_IOC_GET_TYPE, WRITE, struct_power_type_sz); - _(POWER_IOC_GET_TYPE_WITH_LOSSAGE, WRITE, sizeof(uptr)); /* Entries from file: sys/radioio.h */ _(RIOCGINFO, WRITE, struct_radio_info_sz); _(RIOCSINFO, READWRITE, struct_radio_info_sz); @@ -1133,6 +1151,7 @@ static void ioctl_table_fill() { _(SIOCATMARK, WRITE, sizeof(int)); _(SIOCSPGRP, READ, sizeof(int)); _(SIOCGPGRP, WRITE, sizeof(int)); + _(SIOCPEELOFF, READWRITE, sizeof(int)); _(SIOCADDRT, READ, struct_ortentry_sz); _(SIOCDELRT, READ, struct_ortentry_sz); _(SIOCSIFADDR, READ, struct_ifreq_sz); @@ -1190,6 +1209,10 @@ static void ioctl_table_fill() { _(SIOCSLINKSTR, READ, struct_ifdrv_sz); _(SIOCGETHERCAP, READWRITE, struct_eccapreq_sz); _(SIOCGIFINDEX, READWRITE, struct_ifreq_sz); + _(SIOCSETHERCAP, READ, struct_eccapreq_sz); + _(SIOCGUMBINFO, READWRITE, struct_ifreq_sz); + _(SIOCSUMBPARAM, READ, struct_ifreq_sz); + _(SIOCGUMBPARAM, READWRITE, struct_ifreq_sz); _(SIOCSETPFSYNC, READ, struct_ifreq_sz); _(SIOCGETPFSYNC, READWRITE, struct_ifreq_sz); /* Entries from file: sys/timepps.h */ diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h index f8a405ba6e4d..14258d617e62 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h @@ -105,8 +105,8 @@ // // FIXME: do we have anything like this on Mac? #ifndef SANITIZER_CAN_USE_PREINIT_ARRAY -#if ((SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_OPENBSD) && \ - !defined(PIC) +#if ((SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_OPENBSD || \ + SANITIZER_FUCHSIA) && !defined(PIC) #define SANITIZER_CAN_USE_PREINIT_ARRAY 1 // Before Solaris 11.4, .preinit_array is fully supported only with GNU ld. // FIXME: Check for those conditions. @@ -172,6 +172,7 @@ typedef int pid_t; #if SANITIZER_FREEBSD || SANITIZER_NETBSD || \ SANITIZER_OPENBSD || SANITIZER_MAC || \ + (SANITIZER_SOLARIS && (defined(_LP64) || _FILE_OFFSET_BITS == 64)) || \ (SANITIZER_LINUX && defined(__x86_64__)) typedef u64 OFF_T; #else @@ -196,7 +197,9 @@ typedef u64 tid_t; // This header should NOT include any other headers to avoid portability issues. // Common defs. +#ifndef INLINE #define INLINE inline +#endif #define INTERFACE_ATTRIBUTE SANITIZER_INTERFACE_ATTRIBUTE #define SANITIZER_WEAK_DEFAULT_IMPL \ extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE NOINLINE @@ -275,8 +278,6 @@ typedef thread_return_t (THREAD_CALLING_CONV *thread_callback_t)(void* arg); // NOTE: Functions below must be defined in each run-time. void NORETURN Die(); -// FIXME: No, this shouldn't be in the sanitizer interface. -SANITIZER_INTERFACE_ATTRIBUTE void NORETURN CheckFailed(const char *file, int line, const char *cond, u64 v1, u64 v2); @@ -431,6 +432,7 @@ namespace __scudo { using namespace __sanitizer; } // NOLINT namespace __ubsan { using namespace __sanitizer; } // NOLINT namespace __xray { using namespace __sanitizer; } // NOLINT namespace __interception { using namespace __sanitizer; } // NOLINT +namespace __hwasan { using namespace __sanitizer; } // NOLINT #endif // SANITIZER_DEFS_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libc.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libc.cc index 4b462bfe9728..4032cb10461f 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libc.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libc.cc @@ -73,6 +73,18 @@ void *internal_memmove(void *dest, const void *src, uptr n) { } void *internal_memset(void* s, int c, uptr n) { + // Optimize for the most performance-critical case: + if ((reinterpret_cast<uptr>(s) % 16) == 0 && (n % 16) == 0) { + u64 *p = reinterpret_cast<u64*>(s); + u64 *e = p + n / 8; + u64 v = c; + v |= v << 8; + v |= v << 16; + v |= v << 32; + for (; p < e; p += 2) + p[0] = p[1] = v; + return s; + } // The next line prevents Clang from making a call to memset() instead of the // loop below. // FIXME: building the runtime with -ffreestanding is a better idea. However diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc index 96d6c1eff42c..48795674c54c 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc @@ -31,10 +31,6 @@ #include <asm/param.h> #endif -#if SANITIZER_NETBSD -#include <lwp.h> -#endif - // For mips64, syscall(__NR_stat) fills the buffer in the 'struct kernel_stat' // format. Struct kernel_stat is defined as 'struct stat' in asm/stat.h. To // access stat from asm/stat.h, without conflicting with definition in @@ -55,6 +51,7 @@ #include <sched.h> #include <signal.h> #include <sys/mman.h> +#include <sys/param.h> #if !SANITIZER_SOLARIS #include <sys/ptrace.h> #endif @@ -68,6 +65,7 @@ #endif #if SANITIZER_OPENBSD #include <sys/futex.h> +#include <sys/sysctl.h> #endif #include <unistd.h> @@ -116,6 +114,9 @@ struct kernel_timeval { // <linux/futex.h> is broken on some linux distributions. const int FUTEX_WAIT = 0; const int FUTEX_WAKE = 1; +const int FUTEX_PRIVATE_FLAG = 128; +const int FUTEX_WAIT_PRIVATE = FUTEX_WAIT | FUTEX_PRIVATE_FLAG; +const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE | FUTEX_PRIVATE_FLAG; #endif // SANITIZER_LINUX // Are we using 32-bit or 64-bit Linux syscalls? @@ -149,7 +150,11 @@ extern void internal_sigreturn(); #if SANITIZER_OPENBSD # define SANITIZER_USE_GETENTROPY 1 #else -# define SANITIZER_USE_GETENTROPY 0 +# if SANITIZER_FREEBSD && __FreeBSD_version >= 1200000 +# define SANITIZER_USE_GETENTROPY 1 +# else +# define SANITIZER_USE_GETENTROPY 0 +# endif #endif // SANITIZER_USE_GETENTROPY namespace __sanitizer { @@ -165,14 +170,11 @@ namespace __sanitizer { #endif // --------------- sanitizer_libc.h -#if !SANITIZER_SOLARIS +#if !SANITIZER_SOLARIS && !SANITIZER_NETBSD #if !SANITIZER_S390 && !SANITIZER_OPENBSD uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd, OFF_T offset) { -#if SANITIZER_NETBSD - return internal_syscall64(SYSCALL(mmap), addr, length, prot, flags, fd, - (long)0, offset); -#elif SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS +#if SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS return internal_syscall(SYSCALL(mmap), (uptr)addr, length, prot, flags, fd, offset); #else @@ -186,11 +188,11 @@ uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd, #if !SANITIZER_OPENBSD uptr internal_munmap(void *addr, uptr length) { - return internal_syscall_ptr(SYSCALL(munmap), (uptr)addr, length); + return internal_syscall(SYSCALL(munmap), (uptr)addr, length); } int internal_mprotect(void *addr, uptr length, int prot) { - return internal_syscall_ptr(SYSCALL(mprotect), (uptr)addr, length, prot); + return internal_syscall(SYSCALL(mprotect), (uptr)addr, length, prot); } #endif @@ -202,7 +204,7 @@ uptr internal_open(const char *filename, int flags) { #if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags); #else - return internal_syscall_ptr(SYSCALL(open), (uptr)filename, flags); + return internal_syscall(SYSCALL(open), (uptr)filename, flags); #endif } @@ -211,32 +213,28 @@ uptr internal_open(const char *filename, int flags, u32 mode) { return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags, mode); #else - return internal_syscall_ptr(SYSCALL(open), (uptr)filename, flags, mode); + return internal_syscall(SYSCALL(open), (uptr)filename, flags, mode); #endif } uptr internal_read(fd_t fd, void *buf, uptr count) { sptr res; - HANDLE_EINTR(res, (sptr)internal_syscall_ptr(SYSCALL(read), fd, (uptr)buf, - count)); + HANDLE_EINTR(res, + (sptr)internal_syscall(SYSCALL(read), fd, (uptr)buf, count)); return res; } uptr internal_write(fd_t fd, const void *buf, uptr count) { sptr res; - HANDLE_EINTR(res, (sptr)internal_syscall_ptr(SYSCALL(write), fd, (uptr)buf, - count)); + HANDLE_EINTR(res, + (sptr)internal_syscall(SYSCALL(write), fd, (uptr)buf, count)); return res; } uptr internal_ftruncate(fd_t fd, uptr size) { sptr res; -#if SANITIZER_NETBSD - HANDLE_EINTR(res, internal_syscall64(SYSCALL(ftruncate), fd, 0, (s64)size)); -#else HANDLE_EINTR(res, (sptr)internal_syscall(SYSCALL(ftruncate), fd, (OFF_T)size)); -#endif return res; } @@ -308,9 +306,8 @@ static void kernel_stat_to_stat(struct kernel_stat *in, struct stat *out) { #endif uptr internal_stat(const char *path, void *buf) { -#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD - return internal_syscall_ptr(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, - 0); +#if SANITIZER_FREEBSD || SANITIZER_OPENBSD + return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0); #elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0); @@ -333,9 +330,7 @@ uptr internal_stat(const char *path, void *buf) { } uptr internal_lstat(const char *path, void *buf) { -#if SANITIZER_NETBSD - return internal_syscall_ptr(SYSCALL(lstat), path, buf); -#elif SANITIZER_FREEBSD || SANITIZER_OPENBSD +#if SANITIZER_FREEBSD || SANITIZER_OPENBSD return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, AT_SYMLINK_NOFOLLOW); #elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS @@ -360,16 +355,16 @@ uptr internal_lstat(const char *path, void *buf) { } uptr internal_fstat(fd_t fd, void *buf) { -#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD || \ +#if SANITIZER_FREEBSD || SANITIZER_OPENBSD || \ SANITIZER_LINUX_USES_64BIT_SYSCALLS -#if SANITIZER_MIPS64 && !SANITIZER_NETBSD && !SANITIZER_OPENBSD +#if SANITIZER_MIPS64 && !SANITIZER_OPENBSD // For mips64, fstat syscall fills buffer in the format of kernel_stat struct kernel_stat kbuf; int res = internal_syscall(SYSCALL(fstat), fd, &kbuf); kernel_stat_to_stat(&kbuf, (struct stat *)buf); return res; # else - return internal_syscall_ptr(SYSCALL(fstat), fd, (uptr)buf); + return internal_syscall(SYSCALL(fstat), fd, (uptr)buf); # endif #else struct stat64 buf64; @@ -386,6 +381,10 @@ uptr internal_filesize(fd_t fd) { return (uptr)st.st_size; } +uptr internal_dup(int oldfd) { + return internal_syscall(SYSCALL(dup), oldfd); +} + uptr internal_dup2(int oldfd, int newfd) { #if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS return internal_syscall(SYSCALL(dup3), oldfd, newfd, 0); @@ -399,10 +398,10 @@ uptr internal_readlink(const char *path, char *buf, uptr bufsize) { return internal_syscall(SYSCALL(readlinkat), AT_FDCWD, (uptr)path, (uptr)buf, bufsize); #elif SANITIZER_OPENBSD - return internal_syscall_ptr(SYSCALL(readlinkat), AT_FDCWD, (uptr)path, - (uptr)buf, bufsize); + return internal_syscall(SYSCALL(readlinkat), AT_FDCWD, (uptr)path, (uptr)buf, + bufsize); #else - return internal_syscall_ptr(SYSCALL(readlink), path, buf, bufsize); + return internal_syscall(SYSCALL(readlink), path, buf, bufsize); #endif } @@ -410,7 +409,7 @@ uptr internal_unlink(const char *path) { #if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS || SANITIZER_OPENBSD return internal_syscall(SYSCALL(unlinkat), AT_FDCWD, (uptr)path, 0); #else - return internal_syscall_ptr(SYSCALL(unlink), (uptr)path); + return internal_syscall(SYSCALL(unlink), (uptr)path); #endif } @@ -419,7 +418,7 @@ uptr internal_rename(const char *oldpath, const char *newpath) { return internal_syscall(SYSCALL(renameat), AT_FDCWD, (uptr)oldpath, AT_FDCWD, (uptr)newpath); #else - return internal_syscall_ptr(SYSCALL(rename), (uptr)oldpath, (uptr)newpath); + return internal_syscall(SYSCALL(rename), (uptr)oldpath, (uptr)newpath); #endif } @@ -428,7 +427,7 @@ uptr internal_sched_yield() { } void internal__exit(int exitcode) { -#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD +#if SANITIZER_FREEBSD || SANITIZER_OPENBSD internal_syscall(SYSCALL(exit), exitcode); #else internal_syscall(SYSCALL(exit_group), exitcode); @@ -438,22 +437,24 @@ void internal__exit(int exitcode) { unsigned int internal_sleep(unsigned int seconds) { struct timespec ts; - ts.tv_sec = 1; + ts.tv_sec = seconds; ts.tv_nsec = 0; - int res = internal_syscall_ptr(SYSCALL(nanosleep), &ts, &ts); + int res = internal_syscall(SYSCALL(nanosleep), &ts, &ts); if (res) return ts.tv_sec; return 0; } uptr internal_execve(const char *filename, char *const argv[], char *const envp[]) { - return internal_syscall_ptr(SYSCALL(execve), (uptr)filename, (uptr)argv, - (uptr)envp); + return internal_syscall(SYSCALL(execve), (uptr)filename, (uptr)argv, + (uptr)envp); } -#endif // !SANITIZER_SOLARIS +#endif // !SANITIZER_SOLARIS && !SANITIZER_NETBSD // ----------------- sanitizer_common.h bool FileExists(const char *filename) { + if (ShouldMockFailureToOpen(filename)) + return false; struct stat st; #if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS if (internal_syscall(SYSCALL(newfstatat), AT_FDCWD, filename, &st, 0)) @@ -465,6 +466,7 @@ bool FileExists(const char *filename) { return S_ISREG(st.st_mode); } +#if !SANITIZER_NETBSD tid_t GetTid() { #if SANITIZER_FREEBSD long Tid; @@ -472,8 +474,6 @@ tid_t GetTid() { return Tid; #elif SANITIZER_OPENBSD return internal_syscall(SYSCALL(getthrid)); -#elif SANITIZER_NETBSD - return _lwp_self(); #elif SANITIZER_SOLARIS return thr_self(); #else @@ -481,28 +481,43 @@ tid_t GetTid() { #endif } -#if !SANITIZER_SOLARIS +int TgKill(pid_t pid, tid_t tid, int sig) { +#if SANITIZER_LINUX + return internal_syscall(SYSCALL(tgkill), pid, tid, sig); +#elif SANITIZER_FREEBSD + return internal_syscall(SYSCALL(thr_kill2), pid, tid, sig); +#elif SANITIZER_OPENBSD + (void)pid; + return internal_syscall(SYSCALL(thrkill), tid, sig, nullptr); +#elif SANITIZER_SOLARIS + (void)pid; + return thr_kill(tid, sig); +#endif +} +#endif + +#if !SANITIZER_SOLARIS && !SANITIZER_NETBSD u64 NanoTime() { -#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD +#if SANITIZER_FREEBSD || SANITIZER_OPENBSD timeval tv; #else kernel_timeval tv; #endif internal_memset(&tv, 0, sizeof(tv)); - internal_syscall_ptr(SYSCALL(gettimeofday), &tv, 0); + internal_syscall(SYSCALL(gettimeofday), &tv, 0); return (u64)tv.tv_sec * 1000*1000*1000 + tv.tv_usec * 1000; } uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) { - return internal_syscall_ptr(SYSCALL(clock_gettime), clk_id, tp); + return internal_syscall(SYSCALL(clock_gettime), clk_id, tp); } -#endif // !SANITIZER_SOLARIS +#endif // !SANITIZER_SOLARIS && !SANITIZER_NETBSD // Like getenv, but reads env directly from /proc (on Linux) or parses the // 'environ' array (on some others) and does not use libc. This function // should be called first inside __asan_init. const char *GetEnv(const char *name) { -#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD || \ +#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD || \ SANITIZER_SOLARIS if (::environ != 0) { uptr NameLen = internal_strlen(name); @@ -580,8 +595,8 @@ static void GetArgsAndEnv(char ***argv, char ***envp) { // kern.ps_strings sysctl, which returns a pointer to a structure containing // this information. See also <sys/exec.h>. ps_strings *pss; - size_t sz = sizeof(pss); - if (sysctlbyname("kern.ps_strings", &pss, &sz, NULL, 0) == -1) { + uptr sz = sizeof(pss); + if (internal_sysctlbyname("kern.ps_strings", &pss, &sz, NULL, 0) == -1) { Printf("sysctl kern.ps_strings failed\n"); Die(); } @@ -614,33 +629,13 @@ char **GetArgv() { return argv; } -void ReExec() { +char **GetEnviron() { char **argv, **envp; - const char *pathname = "/proc/self/exe"; - -#if SANITIZER_NETBSD - static const int name[] = { - CTL_KERN, KERN_PROC_ARGS, -1, KERN_PROC_PATHNAME, - }; - char path[400]; - size_t len; - - len = sizeof(path); - if (sysctl(name, ARRAY_SIZE(name), path, &len, NULL, 0) != -1) - pathname = path; -#elif SANITIZER_SOLARIS - pathname = getexecname(); - CHECK_NE(pathname, NULL); -#endif - GetArgsAndEnv(&argv, &envp); - uptr rv = internal_execve(pathname, argv, envp); - int rverrno; - CHECK_EQ(internal_iserror(rv, &rverrno), true); - Printf("execve failed, errno %d\n", rverrno); - Die(); + return envp; } -#endif + +#endif // !SANITIZER_OPENBSD #if !SANITIZER_SOLARIS enum MutexState { @@ -664,7 +659,8 @@ void BlockingMutex::Lock() { #elif SANITIZER_NETBSD sched_yield(); /* No userspace futex-like synchronization */ #else - internal_syscall(SYSCALL(futex), (uptr)m, FUTEX_WAIT, MtxSleeping, 0, 0, 0); + internal_syscall(SYSCALL(futex), (uptr)m, FUTEX_WAIT_PRIVATE, MtxSleeping, + 0, 0, 0); #endif } } @@ -679,7 +675,7 @@ void BlockingMutex::Unlock() { #elif SANITIZER_NETBSD /* No userspace futex-like synchronization */ #else - internal_syscall(SYSCALL(futex), (uptr)m, FUTEX_WAKE, 1, 0, 0, 0); + internal_syscall(SYSCALL(futex), (uptr)m, FUTEX_WAKE_PRIVATE, 1, 0, 0, 0); #endif } } @@ -694,7 +690,9 @@ void BlockingMutex::CheckLocked() { // The actual size of this structure is specified by d_reclen. // Note that getdents64 uses a different structure format. We only provide the // 32-bit syscall here. -#if SANITIZER_NETBSD || SANITIZER_OPENBSD +#if SANITIZER_NETBSD +// Not used +#elif SANITIZER_OPENBSD // struct dirent is different for Linux and us. At this moment, we use only // d_fileno (Linux call this d_ino), d_reclen, and d_name. struct linux_dirent { @@ -721,27 +719,15 @@ struct linux_dirent { }; #endif -#if !SANITIZER_SOLARIS +#if !SANITIZER_SOLARIS && !SANITIZER_NETBSD // Syscall wrappers. uptr internal_ptrace(int request, int pid, void *addr, void *data) { -#if SANITIZER_NETBSD - // XXX We need additional work for ptrace: - // - for request, we use PT_FOO whereas Linux uses PTRACE_FOO - // - data is int for us, but void * for Linux - // - Linux sometimes uses data in the case where we use addr instead - // At this moment, this function is used only within - // "#if SANITIZER_LINUX && defined(__x86_64__)" block in - // sanitizer_stoptheworld_linux_libcdep.cc. - return internal_syscall_ptr(SYSCALL(ptrace), request, pid, (uptr)addr, - (uptr)data); -#else return internal_syscall(SYSCALL(ptrace), request, pid, (uptr)addr, (uptr)data); -#endif } uptr internal_waitpid(int pid, int *status, int options) { - return internal_syscall_ptr(SYSCALL(wait4), pid, (uptr)status, options, + return internal_syscall(SYSCALL(wait4), pid, (uptr)status, options, 0 /* rusage */); } @@ -759,16 +745,12 @@ uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count) { #elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS return internal_syscall(SYSCALL(getdents64), fd, (uptr)dirp, count); #else - return internal_syscall_ptr(SYSCALL(getdents), fd, (uptr)dirp, count); + return internal_syscall(SYSCALL(getdents), fd, (uptr)dirp, count); #endif } uptr internal_lseek(fd_t fd, OFF_T offset, int whence) { -#if SANITIZER_NETBSD - return internal_syscall64(SYSCALL(lseek), fd, 0, offset, whence); -#else return internal_syscall(SYSCALL(lseek), fd, offset, whence); -#endif } #if SANITIZER_LINUX @@ -778,7 +760,7 @@ uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5) { #endif uptr internal_sigaltstack(const void *ss, void *oss) { - return internal_syscall_ptr(SYSCALL(sigaltstack), (uptr)ss, (uptr)oss); + return internal_syscall(SYSCALL(sigaltstack), (uptr)ss, (uptr)oss); } int internal_fork() { @@ -789,6 +771,26 @@ int internal_fork() { #endif } +#if SANITIZER_FREEBSD || SANITIZER_OPENBSD +int internal_sysctl(const int *name, unsigned int namelen, void *oldp, + uptr *oldlenp, const void *newp, uptr newlen) { +#if SANITIZER_OPENBSD + return sysctl(name, namelen, oldp, (size_t *)oldlenp, (void *)newp, + (size_t)newlen); +#else + return internal_syscall(SYSCALL(__sysctl), name, namelen, oldp, + (size_t *)oldlenp, newp, (size_t)newlen); +#endif +} + +#if SANITIZER_FREEBSD +int internal_sysctlbyname(const char *sname, void *oldp, uptr *oldlenp, + const void *newp, uptr newlen) { + return sysctlbyname(sname, oldp, (size_t *)oldlenp, newp, (size_t)newlen); +} +#endif +#endif + #if SANITIZER_LINUX #define SA_RESTORER 0x04000000 // Doesn't set sa_restorer if the caller did not set it, so use with caution @@ -858,8 +860,8 @@ int internal_sigaction_syscall(int signum, const void *act, void *oldact) { uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset) { -#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD - return internal_syscall_ptr(SYSCALL(sigprocmask), how, set, oldset); +#if SANITIZER_FREEBSD || SANITIZER_OPENBSD + return internal_syscall(SYSCALL(sigprocmask), how, set, oldset); #else __sanitizer_kernel_sigset_t *k_set = (__sanitizer_kernel_sigset_t *)set; __sanitizer_kernel_sigset_t *k_oldset = (__sanitizer_kernel_sigset_t *)oldset; @@ -897,9 +899,20 @@ bool internal_sigismember(__sanitizer_sigset_t *set, int signum) { const uptr bit = signum % (sizeof(k_set->sig[0]) * 8); return k_set->sig[idx] & (1 << bit); } -#endif // SANITIZER_LINUX +#elif SANITIZER_FREEBSD +void internal_sigdelset(__sanitizer_sigset_t *set, int signum) { + sigset_t *rset = reinterpret_cast<sigset_t *>(set); + sigdelset(rset, signum); +} + +bool internal_sigismember(__sanitizer_sigset_t *set, int signum) { + sigset_t *rset = reinterpret_cast<sigset_t *>(set); + return sigismember(rset, signum); +} +#endif #endif // !SANITIZER_SOLARIS +#if !SANITIZER_NETBSD // ThreadLister implementation. ThreadLister::ThreadLister(pid_t pid) : pid_(pid), buffer_(4096) { char task_directory_path[80]; @@ -986,6 +999,7 @@ ThreadLister::~ThreadLister() { if (!internal_iserror(descriptor_)) internal_close(descriptor_); } +#endif #if SANITIZER_WORDSIZE == 32 // Take care of unusable kernel area in top gigabyte. @@ -996,6 +1010,8 @@ static uptr GetKernelAreaSize() { // Firstly check if there are writable segments // mapped to top gigabyte (e.g. stack). MemoryMappingLayout proc_maps(/*cache_enabled*/true); + if (proc_maps.Error()) + return 0; MemoryMappedSegment segment; while (proc_maps.Next(&segment)) { if ((segment.end >= 3 * gbyte) && segment.IsWritable()) return 0; @@ -1061,18 +1077,25 @@ uptr GetMaxUserVirtualAddress() { return addr; } +#if !SANITIZER_ANDROID uptr GetPageSize() { -// Android post-M sysconf(_SC_PAGESIZE) crashes if called from .preinit_array. -#if SANITIZER_ANDROID - return 4096; -#elif SANITIZER_LINUX && (defined(__x86_64__) || defined(__i386__)) +#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__i386__)) return EXEC_PAGESIZE; #elif SANITIZER_USE_GETAUXVAL return getauxval(AT_PAGESZ); +#elif SANITIZER_FREEBSD || SANITIZER_NETBSD +// Use sysctl as sysconf can trigger interceptors internally. + int pz = 0; + uptr pzl = sizeof(pz); + int mib[2] = {CTL_HW, HW_PAGESIZE}; + int rv = internal_sysctl(mib, 2, &pz, &pzl, nullptr, 0); + CHECK_EQ(rv, 0); + return (uptr)pz; #else return sysconf(_SC_PAGESIZE); // EXEC_PAGESIZE may not be trustworthy. #endif } +#endif // !SANITIZER_ANDROID #if !SANITIZER_OPENBSD uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) { @@ -1088,8 +1111,9 @@ uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) { const int Mib[4] = {CTL_KERN, KERN_PROC_ARGS, -1, KERN_PROC_PATHNAME}; #endif const char *default_module_name = "kern.proc.pathname"; - size_t Size = buf_len; - bool IsErr = (sysctl(Mib, ARRAY_SIZE(Mib), buf, &Size, NULL, 0) != 0); + uptr Size = buf_len; + bool IsErr = + (internal_sysctl(Mib, ARRAY_SIZE(Mib), buf, &Size, NULL, 0) != 0); int readlink_error = IsErr ? errno : 0; uptr module_name_len = Size; #else @@ -1239,7 +1263,7 @@ uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, "d"(parent_tidptr), "r"(r8), "r"(r10) - : "rsp", "memory", "r11", "rcx"); + : "memory", "r11", "rcx"); return res; } #elif defined(__mips__) @@ -1631,6 +1655,16 @@ static int dl_iterate_phdr_test_cb(struct dl_phdr_info *info, size_t size, static atomic_uint32_t android_api_level; +static AndroidApiLevel AndroidDetectApiLevelStatic() { +#if __ANDROID_API__ <= 19 + return ANDROID_KITKAT; +#elif __ANDROID_API__ <= 22 + return ANDROID_LOLLIPOP_MR1; +#else + return ANDROID_POST_LOLLIPOP; +#endif +} + static AndroidApiLevel AndroidDetectApiLevel() { if (!&dl_iterate_phdr) return ANDROID_KITKAT; // K or lower @@ -1643,11 +1677,14 @@ static AndroidApiLevel AndroidDetectApiLevel() { // interesting to detect. } +extern "C" __attribute__((weak)) void* _DYNAMIC; + AndroidApiLevel AndroidGetApiLevel() { AndroidApiLevel level = (AndroidApiLevel)atomic_load(&android_api_level, memory_order_relaxed); if (level) return level; - level = AndroidDetectApiLevel(); + level = &_DYNAMIC == nullptr ? AndroidDetectApiLevelStatic() + : AndroidDetectApiLevel(); atomic_store(&android_api_level, level, memory_order_relaxed); return level; } @@ -1905,14 +1942,14 @@ static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { #elif defined(__sparc__) ucontext_t *ucontext = (ucontext_t*)context; uptr *stk_ptr; -# if defined (__sparcv9) +# if defined(__sparcv9) || defined (__arch64__) # ifndef MC_PC # define MC_PC REG_PC # endif # ifndef MC_O6 # define MC_O6 REG_O6 # endif -# ifdef SANITIZER_SOLARIS +# if SANITIZER_SOLARIS # define mc_gregs gregs # endif *pc = ucontext->uc_mcontext.mc_gregs[MC_PC]; @@ -1946,6 +1983,10 @@ static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); } +void InitializePlatformEarly() { + // Do nothing. +} + void MaybeReexec() { // No need to re-exec on Linux. } @@ -1954,13 +1995,13 @@ void CheckASLR() { #if SANITIZER_NETBSD int mib[3]; int paxflags; - size_t len = sizeof(paxflags); + uptr len = sizeof(paxflags); mib[0] = CTL_PROC; mib[1] = internal_getpid(); mib[2] = PROC_PID_PAXFLAGS; - if (UNLIKELY(sysctl(mib, 3, &paxflags, &len, NULL, 0) == -1)) { + if (UNLIKELY(internal_sysctl(mib, 3, &paxflags, &len, NULL, 0) == -1)) { Printf("sysctl failed\n"); Die(); } @@ -1969,6 +2010,41 @@ void CheckASLR() { Printf("This sanitizer is not compatible with enabled ASLR\n"); Die(); } +#elif SANITIZER_PPC64V2 + // Disable ASLR for Linux PPC64LE. + int old_personality = personality(0xffffffff); + if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) { + VReport(1, "WARNING: Program is being run with address space layout " + "randomization (ASLR) enabled which prevents the thread and " + "memory sanitizers from working on powerpc64le.\n" + "ASLR will be disabled and the program re-executed.\n"); + CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1); + ReExec(); + } +#else + // Do nothing +#endif +} + +void CheckMPROTECT() { +#if SANITIZER_NETBSD + int mib[3]; + int paxflags; + uptr len = sizeof(paxflags); + + mib[0] = CTL_PROC; + mib[1] = internal_getpid(); + mib[2] = PROC_PID_PAXFLAGS; + + if (UNLIKELY(internal_sysctl(mib, 3, &paxflags, &len, NULL, 0) == -1)) { + Printf("sysctl failed\n"); + Die(); + } + + if (UNLIKELY(paxflags & CTL_PROC_PAXFLAGS_MPROTECT)) { + Printf("This sanitizer is not compatible with enabled MPROTECT\n"); + Die(); + } #else // Do nothing #endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h index 975d6541d88a..c309e33f81b1 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h @@ -18,6 +18,7 @@ SANITIZER_OPENBSD || SANITIZER_SOLARIS #include "sanitizer_common.h" #include "sanitizer_internal_defs.h" +#include "sanitizer_platform_limits_freebsd.h" #include "sanitizer_platform_limits_netbsd.h" #include "sanitizer_platform_limits_openbsd.h" #include "sanitizer_platform_limits_posix.h" @@ -69,6 +70,8 @@ void internal_sigdelset(__sanitizer_sigset_t *set, int signum); uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, int *parent_tidptr, void *newtls, int *child_tidptr); #endif +#elif SANITIZER_FREEBSD +void internal_sigdelset(__sanitizer_sigset_t *set, int signum); #endif // SANITIZER_LINUX // This class reads thread IDs from /proc/<pid>/task using only syscalls. @@ -131,13 +134,13 @@ void ForEachMappedRegion(link_map *map, void (*cb)(const void *, uptr)); #error "Unsupported architecture." #endif -// The Android Bionic team has allocated a TLS slot for TSan starting with N, -// given that Android currently doesn't support ELF TLS. It is used to store -// Sanitizers thread specific data. -static const int TLS_SLOT_TSAN = 8; +// The Android Bionic team has allocated a TLS slot for sanitizers starting +// with Q, given that Android currently doesn't support ELF TLS. It is used to +// store sanitizer thread specific data. +static const int TLS_SLOT_SANITIZER = 6; ALWAYS_INLINE uptr *get_android_tls_ptr() { - return reinterpret_cast<uptr *>(&__get_tls()[TLS_SLOT_TSAN]); + return reinterpret_cast<uptr *>(&__get_tls()[TLS_SLOT_SANITIZER]); } #endif // SANITIZER_ANDROID diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc index 4962ff8322d6..6ce47ec620dc 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc @@ -23,6 +23,7 @@ #include "sanitizer_file.h" #include "sanitizer_flags.h" #include "sanitizer_freebsd.h" +#include "sanitizer_getauxval.h" #include "sanitizer_linux.h" #include "sanitizer_placement_new.h" #include "sanitizer_procmaps.h" @@ -99,6 +100,10 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, // Find the mapping that contains a stack variable. MemoryMappingLayout proc_maps(/*cache_enabled*/true); + if (proc_maps.Error()) { + *stack_top = *stack_bottom = 0; + return; + } MemoryMappedSegment segment; uptr prev_end = 0; while (proc_maps.Next(&segment)) { @@ -652,10 +657,10 @@ u32 GetNumberOfCPUs() { #if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD u32 ncpu; int req[2]; - size_t len = sizeof(ncpu); + uptr len = sizeof(ncpu); req[0] = CTL_HW; req[1] = HW_NCPU; - CHECK_EQ(sysctl(req, 2, &ncpu, &len, NULL, 0), 0); + CHECK_EQ(internal_sysctl(req, 2, &ncpu, &len, NULL, 0), 0); return ncpu; #elif SANITIZER_ANDROID && !defined(CPU_COUNT) && !defined(__aarch64__) // Fall back to /sys/devices/system/cpu on Android when cpu_set_t doesn't @@ -806,6 +811,40 @@ u64 MonotonicNanoTime() { } #endif // SANITIZER_LINUX && !SANITIZER_GO +#if !SANITIZER_OPENBSD +void ReExec() { + const char *pathname = "/proc/self/exe"; + +#if SANITIZER_NETBSD + static const int name[] = { + CTL_KERN, + KERN_PROC_ARGS, + -1, + KERN_PROC_PATHNAME, + }; + char path[400]; + uptr len; + + len = sizeof(path); + if (internal_sysctl(name, ARRAY_SIZE(name), path, &len, NULL, 0) != -1) + pathname = path; +#elif SANITIZER_SOLARIS + pathname = getexecname(); + CHECK_NE(pathname, NULL); +#elif SANITIZER_USE_GETAUXVAL + // Calling execve with /proc/self/exe sets that as $EXEC_ORIGIN. Binaries that + // rely on that will fail to load shared libraries. Query AT_EXECFN instead. + pathname = reinterpret_cast<const char *>(getauxval(AT_EXECFN)); +#endif + + uptr rv = internal_execve(pathname, GetArgv(), GetEnviron()); + int rverrno; + CHECK_EQ(internal_iserror(rv, &rverrno), true); + Printf("execve failed, errno %d\n", rverrno); + Die(); +} +#endif // !SANITIZER_OPENBSD + } // namespace __sanitizer #endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_local_address_space_view.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_local_address_space_view.h new file mode 100644 index 000000000000..ec1847abc53d --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_local_address_space_view.h @@ -0,0 +1,77 @@ +//===-- sanitizer_local_address_space_view.h --------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// `LocalAddressSpaceView` provides the local (i.e. target and current address +// space are the same) implementation of the `AddressSpaveView` interface which +// provides a simple interface to load memory from another process (i.e. +// out-of-process) +// +// The `AddressSpaceView` interface requires that the type can be used as a +// template parameter to objects that wish to be able to operate in an +// out-of-process manner. In normal usage, objects are in-process and are thus +// instantiated with the `LocalAddressSpaceView` type. This type is used to +// load any pointers in instance methods. This implementation is effectively +// a no-op. When an object is to be used in an out-of-process manner it is +// instansiated with the `RemoteAddressSpaceView` type. +// +// By making `AddressSpaceView` a template parameter of an object, it can +// change its implementation at compile time which has no run time overhead. +// This also allows unifying in-process and out-of-process code which avoids +// code duplication. +// +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_LOCAL_ADDRES_SPACE_VIEW_H +#define SANITIZER_LOCAL_ADDRES_SPACE_VIEW_H + +namespace __sanitizer { +struct LocalAddressSpaceView { + // Load memory `sizeof(T) * num_elements` bytes of memory from the target + // process (always local for this implementation) starting at address + // `target_address`. The local copy of this memory is returned as a pointer. + // The caller should not write to this memory. The behaviour when doing so is + // undefined. Callers should use `LoadWritable()` to get access to memory + // that is writable. + // + // The lifetime of loaded memory is implementation defined. + template <typename T> + static const T *Load(const T *target_address, uptr num_elements = 1) { + // The target address space is the local address space so + // nothing needs to be copied. Just return the pointer. + return target_address; + } + + // Load memory `sizeof(T) * num_elements` bytes of memory from the target + // process (always local for this implementation) starting at address + // `target_address`. The local copy of this memory is returned as a pointer. + // The memory returned may be written to. + // + // Writes made to the returned memory will be visible in the memory returned + // by subsequent `Load()` or `LoadWritable()` calls provided the + // `target_address` parameter is the same. It is not guaranteed that the + // memory returned by previous calls to `Load()` will contain any performed + // writes. If two or more overlapping regions of memory are loaded via + // separate calls to `LoadWritable()`, it is implementation defined whether + // writes made to the region returned by one call are visible in the regions + // returned by other calls. + // + // Given the above it is recommended to load the largest possible object + // that requires modification (e.g. a class) rather than individual fields + // from a class to avoid issues with overlapping writable regions. + // + // The lifetime of loaded memory is implementation defined. + template <typename T> + static T *LoadWritable(T *target_address, uptr num_elements = 1) { + // The target address space is the local address space so + // nothing needs to be copied. Just return the pointer. + return target_address; + } +}; +} // namespace __sanitizer + +#endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc index 180d7c199aef..8954a7a88073 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc @@ -108,9 +108,20 @@ extern "C" int __munmap(void *, size_t) SANITIZER_WEAK_ATTRIBUTE; #define VM_MEMORY_SANITIZER 99 #endif +// XNU on Darwin provides a mmap flag that optimizes allocation/deallocation of +// giant memory regions (i.e. shadow memory regions). +#define kXnuFastMmapFd 0x4 +static size_t kXnuFastMmapThreshold = 2 << 30; // 2 GB +static bool use_xnu_fast_mmap = false; + uptr internal_mmap(void *addr, size_t length, int prot, int flags, int fd, u64 offset) { - if (fd == -1) fd = VM_MAKE_TAG(VM_MEMORY_SANITIZER); + if (fd == -1) { + fd = VM_MAKE_TAG(VM_MEMORY_SANITIZER); + if (length >= kXnuFastMmapThreshold) { + if (use_xnu_fast_mmap) fd |= kXnuFastMmapFd; + } + } if (&__mmap) return (uptr)__mmap(addr, length, prot, flags, fd, offset); return (uptr)mmap(addr, length, prot, flags, fd, offset); } @@ -163,6 +174,10 @@ uptr internal_filesize(fd_t fd) { return (uptr)st.st_size; } +uptr internal_dup(int oldfd) { + return dup(oldfd); +} + uptr internal_dup2(int oldfd, int newfd) { return dup2(oldfd, newfd); } @@ -213,6 +228,18 @@ int internal_fork() { return fork(); } +int internal_sysctl(const int *name, unsigned int namelen, void *oldp, + uptr *oldlenp, const void *newp, uptr newlen) { + return sysctl(const_cast<int *>(name), namelen, oldp, (size_t *)oldlenp, + const_cast<void *>(newp), (size_t)newlen); +} + +int internal_sysctlbyname(const char *sname, void *oldp, uptr *oldlenp, + const void *newp, uptr newlen) { + return sysctlbyname(sname, oldp, (size_t *)oldlenp, const_cast<void *>(newp), + (size_t)newlen); +} + int internal_forkpty(int *amaster) { int master, slave; if (openpty(&master, &slave, nullptr, nullptr, nullptr) == -1) return -1; @@ -255,6 +282,8 @@ uptr internal_waitpid(int pid, int *status, int options) { // ----------------- sanitizer_common.h bool FileExists(const char *filename) { + if (ShouldMockFailureToOpen(filename)) + return false; struct stat st; if (stat(filename, &st)) return false; @@ -350,6 +379,10 @@ void CheckASLR() { // Do nothing } +void CheckMPROTECT() { + // Do nothing +} + uptr GetPageSize() { return sysconf(_SC_PAGESIZE); } @@ -499,26 +532,38 @@ MacosVersion GetMacosVersionInternal() { uptr len = 0, maxlen = sizeof(version) / sizeof(version[0]); for (uptr i = 0; i < maxlen; i++) version[i] = '\0'; // Get the version length. - CHECK_NE(sysctl(mib, 2, 0, &len, 0, 0), -1); + CHECK_NE(internal_sysctl(mib, 2, 0, &len, 0, 0), -1); CHECK_LT(len, maxlen); - CHECK_NE(sysctl(mib, 2, version, &len, 0, 0), -1); - switch (version[0]) { - case '9': return MACOS_VERSION_LEOPARD; - case '1': { - switch (version[1]) { - case '0': return MACOS_VERSION_SNOW_LEOPARD; - case '1': return MACOS_VERSION_LION; - case '2': return MACOS_VERSION_MOUNTAIN_LION; - case '3': return MACOS_VERSION_MAVERICKS; - case '4': return MACOS_VERSION_YOSEMITE; - default: - if (IsDigit(version[1])) - return MACOS_VERSION_UNKNOWN_NEWER; - else - return MACOS_VERSION_UNKNOWN; - } - } - default: return MACOS_VERSION_UNKNOWN; + CHECK_NE(internal_sysctl(mib, 2, version, &len, 0, 0), -1); + + // Expect <major>.<minor>(.<patch>) + CHECK_GE(len, 3); + const char *p = version; + int major = internal_simple_strtoll(p, &p, /*base=*/10); + if (*p != '.') return MACOS_VERSION_UNKNOWN; + p += 1; + int minor = internal_simple_strtoll(p, &p, /*base=*/10); + if (*p != '.') return MACOS_VERSION_UNKNOWN; + + switch (major) { + case 9: return MACOS_VERSION_LEOPARD; + case 10: return MACOS_VERSION_SNOW_LEOPARD; + case 11: return MACOS_VERSION_LION; + case 12: return MACOS_VERSION_MOUNTAIN_LION; + case 13: return MACOS_VERSION_MAVERICKS; + case 14: return MACOS_VERSION_YOSEMITE; + case 15: return MACOS_VERSION_EL_CAPITAN; + case 16: return MACOS_VERSION_SIERRA; + case 17: + // Not a typo, 17.5 Darwin Kernel Version maps to High Sierra 10.13.4. + if (minor >= 5) + return MACOS_VERSION_HIGH_SIERRA_DOT_RELEASE_4; + return MACOS_VERSION_HIGH_SIERRA; + case 18: + return MACOS_VERSION_MOJAVE; + default: + if (major < 9) return MACOS_VERSION_UNKNOWN; + return MACOS_VERSION_UNKNOWN_NEWER; } } @@ -661,6 +706,16 @@ static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); } +void InitializePlatformEarly() { + // Only use xnu_fast_mmap when on x86_64 and the OS supports it. + use_xnu_fast_mmap = +#if defined(__x86_64__) + GetMacosVersion() >= MACOS_VERSION_HIGH_SIERRA_DOT_RELEASE_4; +#else + false; +#endif +} + #if !SANITIZER_GO static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES"; LowLevelAllocator allocator_for_env; @@ -890,10 +945,10 @@ struct __sanitizer_task_vm_info { (sizeof(__sanitizer_task_vm_info) / sizeof(natural_t))) uptr GetTaskInfoMaxAddress() { - __sanitizer_task_vm_info vm_info = {}; + __sanitizer_task_vm_info vm_info = {} /* zero initialize */; mach_msg_type_number_t count = __SANITIZER_TASK_VM_INFO_COUNT; int err = task_info(mach_task_self(), TASK_VM_INFO, (int *)&vm_info, &count); - if (err == 0) { + if (err == 0 && vm_info.max_address != 0) { return vm_info.max_address - 1; } else { // xnu cannot provide vm address limit @@ -1060,14 +1115,16 @@ void CheckNoDeepBind(const char *filename, int flag) { // Do nothing. } -// FIXME: implement on this platform. bool GetRandom(void *buffer, uptr length, bool blocking) { - UNIMPLEMENTED(); + if (!buffer || !length || length > 256) + return false; + // arc4random never fails. + arc4random_buf(buffer, length); + return true; } -// FIXME: implement on this platform. u32 GetNumberOfCPUs() { - UNIMPLEMENTED(); + return (u32)sysconf(_SC_NPROCESSORS_ONLN); } } // namespace __sanitizer diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h index e022a2c0363c..52825f8cf819 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h @@ -40,6 +40,11 @@ enum MacosVersion { MACOS_VERSION_MOUNTAIN_LION, MACOS_VERSION_MAVERICKS, MACOS_VERSION_YOSEMITE, + MACOS_VERSION_EL_CAPITAN, + MACOS_VERSION_SIERRA, + MACOS_VERSION_HIGH_SIERRA, + MACOS_VERSION_HIGH_SIERRA_DOT_RELEASE_4, + MACOS_VERSION_MOJAVE, MACOS_VERSION_UNKNOWN_NEWER }; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc index e69d6f94b0e1..44c914cea9f9 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc @@ -30,9 +30,27 @@ // https://github.com/gperftools/gperftools. namespace __sanitizer { + extern malloc_zone_t sanitizer_zone; + +struct sanitizer_malloc_introspection_t : public malloc_introspection_t { + // IMPORTANT: Do not change the order, alignment, or types of these fields to + // maintain binary compatibility. You should only add fields to this struct. + + // Used to track changes to the allocator that will affect + // zone enumeration. + u64 allocator_enumeration_version; +}; + +u64 GetMallocZoneAllocatorEnumerationVersion() { + // This represents the current allocator ABI version. + // This field should be incremented every time the Allocator + // ABI changes in a way that breaks allocator enumeration. + return 0; } +} // namespace __sanitizer + INTERCEPTOR(malloc_zone_t *, malloc_create_zone, vm_size_t start_size, unsigned zone_flags) { COMMON_MALLOC_ENTER(); @@ -247,6 +265,13 @@ void *__sanitizer_mz_memalign(malloc_zone_t *zone, size_t align, size_t size) { return p; } +// This public API exists purely for testing purposes. +extern "C" +SANITIZER_INTERFACE_ATTRIBUTE +malloc_zone_t* __sanitizer_mz_default_zone() { + return &sanitizer_zone; +} + // This function is currently unused, and we build with -Werror. #if 0 void __sanitizer_mz_free_definite_size( @@ -302,8 +327,8 @@ boolean_t mi_zone_locked(malloc_zone_t *zone) { namespace COMMON_MALLOC_NAMESPACE { -void ReplaceSystemMalloc() { - static malloc_introspection_t sanitizer_zone_introspection; +void InitMallocZoneFields() { + static sanitizer_malloc_introspection_t sanitizer_zone_introspection; // Ok to use internal_memset, these places are not performance-critical. internal_memset(&sanitizer_zone_introspection, 0, sizeof(sanitizer_zone_introspection)); @@ -318,6 +343,10 @@ void ReplaceSystemMalloc() { sanitizer_zone_introspection.statistics = &mi_statistics; sanitizer_zone_introspection.zone_locked = &mi_zone_locked; + // Set current allocator enumeration version. + sanitizer_zone_introspection.allocator_enumeration_version = + GetMallocZoneAllocatorEnumerationVersion(); + internal_memset(&sanitizer_zone, 0, sizeof(malloc_zone_t)); // Use version 6 for OSX >= 10.6. @@ -335,6 +364,10 @@ void ReplaceSystemMalloc() { sanitizer_zone.free_definite_size = 0; sanitizer_zone.memalign = &__sanitizer_mz_memalign; sanitizer_zone.introspect = &sanitizer_zone_introspection; +} + +void ReplaceSystemMalloc() { + InitMallocZoneFields(); // Register the zone. malloc_zone_register(&sanitizer_zone); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_netbsd.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_netbsd.cc new file mode 100644 index 000000000000..80d0855efab0 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_netbsd.cc @@ -0,0 +1,335 @@ +//===-- sanitizer_netbsd.cc -----------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is shared between Sanitizer run-time libraries and implements +// NetBSD-specific functions from sanitizer_libc.h. +//===----------------------------------------------------------------------===// + +#include "sanitizer_platform.h" + +#if SANITIZER_NETBSD + +#include "sanitizer_common.h" +#include "sanitizer_flags.h" +#include "sanitizer_getauxval.h" +#include "sanitizer_internal_defs.h" +#include "sanitizer_libc.h" +#include "sanitizer_linux.h" +#include "sanitizer_mutex.h" +#include "sanitizer_placement_new.h" +#include "sanitizer_procmaps.h" + +#include <sys/param.h> +#include <sys/types.h> + +#include <sys/exec.h> +#include <sys/mman.h> +#include <sys/ptrace.h> +#include <sys/resource.h> +#include <sys/stat.h> +#include <sys/syscall.h> +#include <sys/sysctl.h> +#include <sys/time.h> + +#include <dlfcn.h> +#include <errno.h> +#include <fcntl.h> +#include <limits.h> +#include <link.h> +#include <lwp.h> +#include <pthread.h> +#include <sched.h> +#include <signal.h> +#include <ucontext.h> +#include <unistd.h> + +extern "C" void *__mmap(void *, size_t, int, int, int, int, + off_t) SANITIZER_WEAK_ATTRIBUTE; +extern "C" int __sysctl(const int *, unsigned int, void *, size_t *, + const void *, size_t) SANITIZER_WEAK_ATTRIBUTE; +extern "C" int _sys_close(int) SANITIZER_WEAK_ATTRIBUTE; +extern "C" int _sys_open(const char *, int, ...) SANITIZER_WEAK_ATTRIBUTE; +extern "C" ssize_t _sys_read(int, void *, size_t) SANITIZER_WEAK_ATTRIBUTE; +extern "C" ssize_t _sys_write(int, const void *, + size_t) SANITIZER_WEAK_ATTRIBUTE; +extern "C" int __ftruncate(int, int, off_t) SANITIZER_WEAK_ATTRIBUTE; +extern "C" ssize_t _sys_readlink(const char *, char *, + size_t) SANITIZER_WEAK_ATTRIBUTE; +extern "C" int _sys_sched_yield() SANITIZER_WEAK_ATTRIBUTE; +extern "C" int _sys___nanosleep50(const void *, + void *) SANITIZER_WEAK_ATTRIBUTE; +extern "C" int _sys_execve(const char *, char *const[], + char *const[]) SANITIZER_WEAK_ATTRIBUTE; +extern "C" off_t __lseek(int, int, off_t, int) SANITIZER_WEAK_ATTRIBUTE; +extern "C" int __fork() SANITIZER_WEAK_ATTRIBUTE; +extern "C" int _sys___sigprocmask14(int, const void *, + void *) SANITIZER_WEAK_ATTRIBUTE; +extern "C" int _sys___wait450(int wpid, int *, int, + void *) SANITIZER_WEAK_ATTRIBUTE; + +namespace __sanitizer { + +static void *GetRealLibcAddress(const char *symbol) { + void *real = dlsym(RTLD_NEXT, symbol); + if (!real) + real = dlsym(RTLD_DEFAULT, symbol); + if (!real) { + Printf("GetRealLibcAddress failed for symbol=%s", symbol); + Die(); + } + return real; +} + +#define _REAL(func, ...) real##_##func(__VA_ARGS__) +#define DEFINE__REAL(ret_type, func, ...) \ + static ret_type (*real_##func)(__VA_ARGS__) = NULL; \ + if (!real_##func) { \ + real_##func = (ret_type(*)(__VA_ARGS__))GetRealLibcAddress(#func); \ + } \ + CHECK(real_##func); + +// --------------- sanitizer_libc.h +uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd, + OFF_T offset) { + CHECK(&__mmap); + return (uptr)__mmap(addr, length, prot, flags, fd, 0, offset); +} + +uptr internal_munmap(void *addr, uptr length) { + DEFINE__REAL(int, munmap, void *a, uptr b); + return _REAL(munmap, addr, length); +} + +int internal_mprotect(void *addr, uptr length, int prot) { + DEFINE__REAL(int, mprotect, void *a, uptr b, int c); + return _REAL(mprotect, addr, length, prot); +} + +uptr internal_close(fd_t fd) { + CHECK(&_sys_close); + return _sys_close(fd); +} + +uptr internal_open(const char *filename, int flags) { + CHECK(&_sys_open); + return _sys_open(filename, flags); +} + +uptr internal_open(const char *filename, int flags, u32 mode) { + CHECK(&_sys_open); + return _sys_open(filename, flags, mode); +} + +uptr internal_read(fd_t fd, void *buf, uptr count) { + sptr res; + CHECK(&_sys_read); + HANDLE_EINTR(res, (sptr)_sys_read(fd, buf, (size_t)count)); + return res; +} + +uptr internal_write(fd_t fd, const void *buf, uptr count) { + sptr res; + CHECK(&_sys_write); + HANDLE_EINTR(res, (sptr)_sys_write(fd, buf, count)); + return res; +} + +uptr internal_ftruncate(fd_t fd, uptr size) { + sptr res; + CHECK(&__ftruncate); + HANDLE_EINTR(res, __ftruncate(fd, 0, (s64)size)); + return res; +} + +uptr internal_stat(const char *path, void *buf) { + DEFINE__REAL(int, __stat50, const char *a, void *b); + return _REAL(__stat50, path, buf); +} + +uptr internal_lstat(const char *path, void *buf) { + DEFINE__REAL(int, __lstat50, const char *a, void *b); + return _REAL(__lstat50, path, buf); +} + +uptr internal_fstat(fd_t fd, void *buf) { + DEFINE__REAL(int, __fstat50, int a, void *b); + return _REAL(__fstat50, fd, buf); +} + +uptr internal_filesize(fd_t fd) { + struct stat st; + if (internal_fstat(fd, &st)) + return -1; + return (uptr)st.st_size; +} + +uptr internal_dup(int oldfd) { + DEFINE__REAL(int, dup, int a); + return _REAL(dup, oldfd); +} + +uptr internal_dup2(int oldfd, int newfd) { + DEFINE__REAL(int, dup2, int a, int b); + return _REAL(dup2, oldfd, newfd); +} + +uptr internal_readlink(const char *path, char *buf, uptr bufsize) { + CHECK(&_sys_readlink); + return (uptr)_sys_readlink(path, buf, bufsize); +} + +uptr internal_unlink(const char *path) { + DEFINE__REAL(int, unlink, const char *a); + return _REAL(unlink, path); +} + +uptr internal_rename(const char *oldpath, const char *newpath) { + DEFINE__REAL(int, rename, const char *a, const char *b); + return _REAL(rename, oldpath, newpath); +} + +uptr internal_sched_yield() { + CHECK(&_sys_sched_yield); + return _sys_sched_yield(); +} + +void internal__exit(int exitcode) { + DEFINE__REAL(void, _exit, int a); + _REAL(_exit, exitcode); + Die(); // Unreachable. +} + +unsigned int internal_sleep(unsigned int seconds) { + struct timespec ts; + ts.tv_sec = seconds; + ts.tv_nsec = 0; + CHECK(&_sys___nanosleep50); + int res = _sys___nanosleep50(&ts, &ts); + if (res) + return ts.tv_sec; + return 0; +} + +uptr internal_execve(const char *filename, char *const argv[], + char *const envp[]) { + CHECK(&_sys_execve); + return _sys_execve(filename, argv, envp); +} + +tid_t GetTid() { + DEFINE__REAL(int, _lwp_self); + return _REAL(_lwp_self); +} + +int TgKill(pid_t pid, tid_t tid, int sig) { + DEFINE__REAL(int, _lwp_kill, int a, int b); + (void)pid; + return _REAL(_lwp_kill, tid, sig); +} + +u64 NanoTime() { + timeval tv; + DEFINE__REAL(int, __gettimeofday50, void *a, void *b); + internal_memset(&tv, 0, sizeof(tv)); + _REAL(__gettimeofday50, &tv, 0); + return (u64)tv.tv_sec * 1000 * 1000 * 1000 + tv.tv_usec * 1000; +} + +uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) { + DEFINE__REAL(int, __clock_gettime50, __sanitizer_clockid_t a, void *b); + return _REAL(__clock_gettime50, clk_id, tp); +} + +uptr internal_ptrace(int request, int pid, void *addr, void *data) { + Printf("internal_ptrace not implemented for NetBSD"); + Die(); + return 0; +} + +uptr internal_waitpid(int pid, int *status, int options) { + CHECK(&_sys___wait450); + return _sys___wait450(pid, status, options, 0 /* rusage */); +} + +uptr internal_getpid() { + DEFINE__REAL(int, getpid); + return _REAL(getpid); +} + +uptr internal_getppid() { + DEFINE__REAL(int, getppid); + return _REAL(getppid); +} + +uptr internal_getdents(fd_t fd, void *dirp, unsigned int count) { + DEFINE__REAL(int, __getdents30, int a, void *b, size_t c); + return _REAL(__getdents30, fd, dirp, count); +} + +uptr internal_lseek(fd_t fd, OFF_T offset, int whence) { + CHECK(&__lseek); + return __lseek(fd, 0, offset, whence); +} + +uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5) { + Printf("internal_prctl not implemented for NetBSD"); + Die(); + return 0; +} + +uptr internal_sigaltstack(const void *ss, void *oss) { + DEFINE__REAL(int, __sigaltstack14, const void *a, void *b); + return _REAL(__sigaltstack14, ss, oss); +} + +int internal_fork() { + CHECK(&__fork); + return __fork(); +} + +int internal_sysctl(const int *name, unsigned int namelen, void *oldp, + uptr *oldlenp, const void *newp, uptr newlen) { + CHECK(&__sysctl); + return __sysctl(name, namelen, oldp, (size_t *)oldlenp, newp, (size_t)newlen); +} + +int internal_sysctlbyname(const char *sname, void *oldp, uptr *oldlenp, + const void *newp, uptr newlen) { + DEFINE__REAL(int, sysctlbyname, const char *a, void *b, size_t *c, + const void *d, size_t e); + return _REAL(sysctlbyname, sname, oldp, (size_t *)oldlenp, newp, + (size_t)newlen); +} + +uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set, + __sanitizer_sigset_t *oldset) { + CHECK(&_sys___sigprocmask14); + return _sys___sigprocmask14(how, set, oldset); +} + +void internal_sigfillset(__sanitizer_sigset_t *set) { + DEFINE__REAL(int, __sigfillset14, const void *a); + (void)_REAL(__sigfillset14, set); +} + +void internal_sigemptyset(__sanitizer_sigset_t *set) { + DEFINE__REAL(int, __sigemptyset14, const void *a); + (void)_REAL(__sigemptyset14, set); +} + +uptr intrnal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, + int *parent_tidptr, void *newtls, int *child_tidptr) { + Printf("internal_clone not implemented for NetBSD"); + Die(); + return 0; +} + +} // namespace __sanitizer + +#endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_openbsd.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_openbsd.cc index 2aea7cb14875..f0d071e51f02 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_openbsd.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_openbsd.cc @@ -51,12 +51,19 @@ int internal_mprotect(void *addr, uptr length, int prot) { return mprotect(addr, length, prot); } +int internal_sysctlbyname(const char *sname, void *oldp, uptr *oldlenp, + const void *newp, uptr newlen) { + Printf("internal_sysctlbyname not implemented for OpenBSD"); + Die(); + return 0; +} + uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) { // On OpenBSD we cannot get the full path struct kinfo_proc kp; - size_t kl; + uptr kl; const int Mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PID, getpid()}; - if (sysctl(Mib, ARRAY_SIZE(Mib), &kp, &kl, NULL, 0) != -1) + if (internal_sysctl(Mib, ARRAY_SIZE(Mib), &kp, &kl, NULL, 0) != -1) return internal_snprintf(buf, (KI_MAXCOMLEN < buf_len ? KI_MAXCOMLEN : buf_len), "%s", kp.p_comm); @@ -64,23 +71,23 @@ uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) { } static void GetArgsAndEnv(char ***argv, char ***envp) { - size_t nargv; - size_t nenv; + uptr nargv; + uptr nenv; int argvmib[4] = {CTL_KERN, KERN_PROC_ARGS, getpid(), KERN_PROC_ARGV}; int envmib[4] = {CTL_KERN, KERN_PROC_ARGS, getpid(), KERN_PROC_ENV}; - if (sysctl(argvmib, 4, NULL, &nargv, NULL, 0) == -1) { + if (internal_sysctl(argvmib, 4, NULL, &nargv, NULL, 0) == -1) { Printf("sysctl KERN_PROC_NARGV failed\n"); Die(); } - if (sysctl(envmib, 4, NULL, &nenv, NULL, 0) == -1) { + if (internal_sysctl(envmib, 4, NULL, &nenv, NULL, 0) == -1) { Printf("sysctl KERN_PROC_NENV failed\n"); Die(); } - if (sysctl(argvmib, 4, &argv, &nargv, NULL, 0) == -1) { + if (internal_sysctl(argvmib, 4, &argv, &nargv, NULL, 0) == -1) { Printf("sysctl KERN_PROC_ARGV failed\n"); Die(); } - if (sysctl(envmib, 4, &envp, &nenv, NULL, 0) == -1) { + if (internal_sysctl(envmib, 4, &envp, &nenv, NULL, 0) == -1) { Printf("sysctl KERN_PROC_ENV failed\n"); Die(); } @@ -92,6 +99,12 @@ char **GetArgv() { return argv; } +char **GetEnviron() { + char **argv, **envp; + GetArgsAndEnv(&argv, &envp); + return envp; +} + void ReExec() { UNIMPLEMENTED(); } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform.h index d81e25580d9e..82bb1af7746b 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform.h @@ -235,7 +235,12 @@ #if defined(__mips__) # define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 40) #elif defined(__aarch64__) -# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 48) +# if SANITIZER_MAC +// Darwin iOS/ARM64 has a 36-bit VMA, 64GiB VM +# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 36) +# else +# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 48) +# endif #else # define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47) #endif @@ -270,12 +275,6 @@ # define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 12) #endif -// Assume obsolete RPC headers are available by default -#if !defined(HAVE_RPC_XDR_H) && !defined(HAVE_TIRPC_RPC_XDR_H) -# define HAVE_RPC_XDR_H (SANITIZER_LINUX && !SANITIZER_ANDROID) -# define HAVE_TIRPC_RPC_XDR_H 0 -#endif - /// \macro MSC_PREREQ /// \brief Is the compiler MSVC of at least the specified version? /// The common \param version values to check for are: @@ -337,4 +336,13 @@ #define SANITIZER_SYMBOLIZER_MARKUP 0 #endif +// Enable ability to support sanitizer initialization that is +// compatible with the sanitizer library being loaded via +// `dlopen()`. +#if SANITIZER_MAC +#define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 1 +#else +#define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 0 +#endif + #endif // SANITIZER_PLATFORM_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h index f95539a73c69..4d146651b78d 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h @@ -33,8 +33,9 @@ #endif #if SI_POSIX +# include "sanitizer_platform_limits_freebsd.h" # include "sanitizer_platform_limits_netbsd.h" -#include "sanitizer_platform_limits_openbsd.h" +# include "sanitizer_platform_limits_openbsd.h" # include "sanitizer_platform_limits_posix.h" # include "sanitizer_platform_limits_solaris.h" #endif @@ -207,9 +208,13 @@ #define SANITIZER_INTERCEPT_GETPWENT \ (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_MAC || SI_LINUX_NOT_ANDROID || \ SI_SOLARIS) +#define SANITIZER_INTERCEPT_FGETGRENT_R \ + (SI_FREEBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_FGETPWENT SI_LINUX_NOT_ANDROID || SI_SOLARIS #define SANITIZER_INTERCEPT_GETPWENT_R \ (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) +#define SANITIZER_INTERCEPT_FGETPWENT_R \ + (SI_FREEBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_SETPWENT \ (SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_CLOCK_GETTIME \ @@ -356,7 +361,9 @@ #define SANITIZER_INTERCEPT_TEMPNAM SI_POSIX #define SANITIZER_INTERCEPT_SINCOS SI_LINUX || SI_SOLARIS #define SANITIZER_INTERCEPT_REMQUO SI_POSIX +#define SANITIZER_INTERCEPT_REMQUOL (SI_POSIX && !SI_NETBSD) #define SANITIZER_INTERCEPT_LGAMMA SI_POSIX +#define SANITIZER_INTERCEPT_LGAMMAL (SI_POSIX && !SI_NETBSD) #define SANITIZER_INTERCEPT_LGAMMA_R (SI_FREEBSD || SI_LINUX || SI_SOLARIS) #define SANITIZER_INTERCEPT_LGAMMAL_R SI_LINUX_NOT_ANDROID || SI_SOLARIS #define SANITIZER_INTERCEPT_DRAND48_R SI_LINUX_NOT_ANDROID @@ -380,7 +387,7 @@ #define SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP \ (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_PTHREAD_GETNAME_NP \ - (SI_FREEBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) + (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_TLS_GET_ADDR \ (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS) @@ -510,5 +517,43 @@ #define SANITIZER_INTERCEPT_TTYENT SI_NETBSD #define SANITIZER_INTERCEPT_PROTOENT SI_NETBSD #define SANITIZER_INTERCEPT_NETENT SI_NETBSD +#define SANITIZER_INTERCEPT_SETVBUF (SI_NETBSD || SI_FREEBSD || \ + SI_LINUX || SI_MAC) +#define SANITIZER_INTERCEPT_GETMNTINFO (SI_NETBSD || SI_FREEBSD || SI_MAC) +#define SANITIZER_INTERCEPT_MI_VECTOR_HASH SI_NETBSD +#define SANITIZER_INTERCEPT_GETVFSSTAT SI_NETBSD +#define SANITIZER_INTERCEPT_REGEX (SI_NETBSD || SI_FREEBSD || SI_LINUX) +#define SANITIZER_INTERCEPT_REGEXSUB SI_NETBSD +#define SANITIZER_INTERCEPT_FTS (SI_NETBSD || SI_FREEBSD) +#define SANITIZER_INTERCEPT_SYSCTL (SI_NETBSD || SI_FREEBSD || SI_MAC) +#define SANITIZER_INTERCEPT_ASYSCTL SI_NETBSD +#define SANITIZER_INTERCEPT_SYSCTLGETMIBINFO SI_NETBSD +#define SANITIZER_INTERCEPT_NL_LANGINFO (SI_NETBSD || SI_FREEBSD || SI_MAC) +#define SANITIZER_INTERCEPT_MODCTL SI_NETBSD +#define SANITIZER_INTERCEPT_CAPSICUM SI_FREEBSD +#define SANITIZER_INTERCEPT_STRTONUM (SI_NETBSD || SI_FREEBSD) +#define SANITIZER_INTERCEPT_FPARSELN SI_NETBSD +#define SANITIZER_INTERCEPT_STATVFS1 SI_NETBSD +#define SANITIZER_INTERCEPT_STRTOI SI_NETBSD +#define SANITIZER_INTERCEPT_CAPSICUM SI_FREEBSD +#define SANITIZER_INTERCEPT_SHA1 SI_NETBSD +#define SANITIZER_INTERCEPT_MD4 SI_NETBSD +#define SANITIZER_INTERCEPT_RMD160 SI_NETBSD +#define SANITIZER_INTERCEPT_MD5 SI_NETBSD +#define SANITIZER_INTERCEPT_FSEEK (SI_NETBSD || SI_FREEBSD) +#define SANITIZER_INTERCEPT_MD2 SI_NETBSD +#define SANITIZER_INTERCEPT_SHA2 SI_NETBSD +#define SANITIZER_INTERCEPT_CDB SI_NETBSD +#define SANITIZER_INTERCEPT_VIS (SI_NETBSD || SI_FREEBSD) +#define SANITIZER_INTERCEPT_POPEN SI_POSIX +#define SANITIZER_INTERCEPT_POPENVE SI_NETBSD +#define SANITIZER_INTERCEPT_PCLOSE SI_POSIX +#define SANITIZER_INTERCEPT_FUNOPEN (SI_NETBSD || SI_FREEBSD) +#define SANITIZER_INTERCEPT_FUNOPEN2 SI_NETBSD +#define SANITIZER_INTERCEPT_GETFSENT (SI_FREEBSD || SI_NETBSD || SI_MAC) +#define SANITIZER_INTERCEPT_ARC4RANDOM (SI_FREEBSD || SI_NETBSD) +#define SANITIZER_INTERCEPT_FDEVNAME SI_FREEBSD +#define SANITIZER_INTERCEPT_GETUSERSHELL (SI_POSIX && !SI_POSIX) +#define SANITIZER_INTERCEPT_SL_INIT (SI_FREEBSD || SI_NETBSD) #endif // #ifndef SANITIZER_PLATFORM_INTERCEPTORS_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cc new file mode 100644 index 000000000000..377a62cab6c4 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cc @@ -0,0 +1,526 @@ +//===-- sanitizer_platform_limits_freebsd.cc ------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of Sanitizer common code. +// +// Sizes and layouts of platform-specific FreeBSD data structures. +//===----------------------------------------------------------------------===// + +#include "sanitizer_platform.h" + +#if SANITIZER_FREEBSD + +#include <arpa/inet.h> +#include <dirent.h> +#include <fts.h> +#include <fstab.h> +#include <grp.h> +#include <limits.h> +#include <net/if.h> +#include <netdb.h> +#include <poll.h> +#include <pthread.h> +#include <pwd.h> +#include <regex.h> +#include <signal.h> +#include <stddef.h> +#include <sys/mman.h> +#include <sys/capsicum.h> +#include <sys/resource.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <sys/times.h> +#include <sys/types.h> +#include <sys/utsname.h> +#include <termios.h> +#include <time.h> + +#include <net/route.h> +#include <sys/mount.h> +#include <sys/sockio.h> +#include <sys/socket.h> +#include <sys/filio.h> +#include <sys/signal.h> +#include <sys/timespec.h> +#include <sys/timeb.h> +#include <sys/mqueue.h> +#include <sys/msg.h> +#include <sys/ipc.h> +#include <sys/msg.h> +#include <sys/statvfs.h> +#include <sys/soundcard.h> +#include <sys/mtio.h> +#include <sys/consio.h> +#include <sys/kbio.h> +#include <sys/link_elf.h> +#include <netinet/ip_mroute.h> +#include <netinet/in.h> +#include <net/ethernet.h> +#include <net/ppp_defs.h> +#include <glob.h> +#include <stdio.h> +#include <stringlist.h> +#include <term.h> +#include <utmpx.h> +#include <wchar.h> +#include <vis.h> + +#define _KERNEL // to declare 'shminfo' structure +# include <sys/shm.h> +#undef _KERNEL + +#undef INLINE // to avoid clashes with sanitizers' definitions + +#undef IOC_DIRMASK + +# include <utime.h> +# include <sys/ptrace.h> +# include <semaphore.h> + +#include <ifaddrs.h> +#include <sys/ucontext.h> +#include <wordexp.h> + +// Include these after system headers to avoid name clashes and ambiguities. +#include "sanitizer_internal_defs.h" +#include "sanitizer_platform_limits_freebsd.h" + +namespace __sanitizer { + unsigned struct_cap_rights_sz = sizeof(cap_rights_t); + unsigned struct_utsname_sz = sizeof(struct utsname); + unsigned struct_stat_sz = sizeof(struct stat); + unsigned struct_rusage_sz = sizeof(struct rusage); + unsigned struct_tm_sz = sizeof(struct tm); + unsigned struct_passwd_sz = sizeof(struct passwd); + unsigned struct_group_sz = sizeof(struct group); + unsigned siginfo_t_sz = sizeof(siginfo_t); + unsigned struct_sigaction_sz = sizeof(struct sigaction); + unsigned struct_itimerval_sz = sizeof(struct itimerval); + unsigned pthread_t_sz = sizeof(pthread_t); + unsigned pthread_mutex_t_sz = sizeof(pthread_mutex_t); + unsigned pthread_cond_t_sz = sizeof(pthread_cond_t); + unsigned pid_t_sz = sizeof(pid_t); + unsigned timeval_sz = sizeof(timeval); + unsigned uid_t_sz = sizeof(uid_t); + unsigned gid_t_sz = sizeof(gid_t); + unsigned fpos_t_sz = sizeof(fpos_t); + unsigned mbstate_t_sz = sizeof(mbstate_t); + unsigned sigset_t_sz = sizeof(sigset_t); + unsigned struct_timezone_sz = sizeof(struct timezone); + unsigned struct_tms_sz = sizeof(struct tms); + unsigned struct_sigevent_sz = sizeof(struct sigevent); + unsigned struct_sched_param_sz = sizeof(struct sched_param); + unsigned struct_statfs_sz = sizeof(struct statfs); + unsigned struct_sockaddr_sz = sizeof(struct sockaddr); + unsigned ucontext_t_sz = sizeof(ucontext_t); + unsigned struct_rlimit_sz = sizeof(struct rlimit); + unsigned struct_timespec_sz = sizeof(struct timespec); + unsigned struct_utimbuf_sz = sizeof(struct utimbuf); + unsigned struct_itimerspec_sz = sizeof(struct itimerspec); + unsigned struct_timeb_sz = sizeof(struct timeb); + unsigned struct_msqid_ds_sz = sizeof(struct msqid_ds); + unsigned struct_mq_attr_sz = sizeof(struct mq_attr); + unsigned struct_statvfs_sz = sizeof(struct statvfs); + unsigned struct_shminfo_sz = sizeof(struct shminfo); + unsigned struct_shm_info_sz = sizeof(struct shm_info); + unsigned struct_regmatch_sz = sizeof(regmatch_t); + unsigned struct_regex_sz = sizeof(regex_t); + unsigned struct_fstab_sz = sizeof(struct fstab); + unsigned struct_FTS_sz = sizeof(FTS); + unsigned struct_FTSENT_sz = sizeof(FTSENT); + unsigned struct_StringList_sz = sizeof(StringList); + + const uptr sig_ign = (uptr)SIG_IGN; + const uptr sig_dfl = (uptr)SIG_DFL; + const uptr sig_err = (uptr)SIG_ERR; + const uptr sa_siginfo = (uptr)SA_SIGINFO; + + int shmctl_ipc_stat = (int)IPC_STAT; + int shmctl_ipc_info = (int)IPC_INFO; + int shmctl_shm_info = (int)SHM_INFO; + int shmctl_shm_stat = (int)SHM_STAT; + unsigned struct_utmpx_sz = sizeof(struct utmpx); + + int map_fixed = MAP_FIXED; + + int af_inet = (int)AF_INET; + int af_inet6 = (int)AF_INET6; + + uptr __sanitizer_in_addr_sz(int af) { + if (af == AF_INET) + return sizeof(struct in_addr); + else if (af == AF_INET6) + return sizeof(struct in6_addr); + else + return 0; + } + + unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); + int glob_nomatch = GLOB_NOMATCH; + int glob_altdirfunc = GLOB_ALTDIRFUNC; + + unsigned path_max = PATH_MAX; + + // ioctl arguments + unsigned struct_ifreq_sz = sizeof(struct ifreq); + unsigned struct_termios_sz = sizeof(struct termios); + unsigned struct_winsize_sz = sizeof(struct winsize); +#if SOUND_VERSION >= 0x040000 + unsigned struct_copr_buffer_sz = 0; + unsigned struct_copr_debug_buf_sz = 0; + unsigned struct_copr_msg_sz = 0; +#else + unsigned struct_copr_buffer_sz = sizeof(struct copr_buffer); + unsigned struct_copr_debug_buf_sz = sizeof(struct copr_debug_buf); + unsigned struct_copr_msg_sz = sizeof(struct copr_msg); +#endif + unsigned struct_midi_info_sz = sizeof(struct midi_info); + unsigned struct_mtget_sz = sizeof(struct mtget); + unsigned struct_mtop_sz = sizeof(struct mtop); + unsigned struct_sbi_instrument_sz = sizeof(struct sbi_instrument); + unsigned struct_seq_event_rec_sz = sizeof(struct seq_event_rec); + unsigned struct_synth_info_sz = sizeof(struct synth_info); + unsigned struct_audio_buf_info_sz = sizeof(struct audio_buf_info); + unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats); + unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req); + unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req); + const unsigned long __sanitizer_bufsiz = BUFSIZ; + + const unsigned IOCTL_NOT_PRESENT = 0; + + unsigned IOCTL_FIOASYNC = FIOASYNC; + unsigned IOCTL_FIOCLEX = FIOCLEX; + unsigned IOCTL_FIOGETOWN = FIOGETOWN; + unsigned IOCTL_FIONBIO = FIONBIO; + unsigned IOCTL_FIONCLEX = FIONCLEX; + unsigned IOCTL_FIOSETOWN = FIOSETOWN; + unsigned IOCTL_SIOCADDMULTI = SIOCADDMULTI; + unsigned IOCTL_SIOCATMARK = SIOCATMARK; + unsigned IOCTL_SIOCDELMULTI = SIOCDELMULTI; + unsigned IOCTL_SIOCGIFADDR = SIOCGIFADDR; + unsigned IOCTL_SIOCGIFBRDADDR = SIOCGIFBRDADDR; + unsigned IOCTL_SIOCGIFCONF = SIOCGIFCONF; + unsigned IOCTL_SIOCGIFDSTADDR = SIOCGIFDSTADDR; + unsigned IOCTL_SIOCGIFFLAGS = SIOCGIFFLAGS; + unsigned IOCTL_SIOCGIFMETRIC = SIOCGIFMETRIC; + unsigned IOCTL_SIOCGIFMTU = SIOCGIFMTU; + unsigned IOCTL_SIOCGIFNETMASK = SIOCGIFNETMASK; + unsigned IOCTL_SIOCGPGRP = SIOCGPGRP; + unsigned IOCTL_SIOCSIFADDR = SIOCSIFADDR; + unsigned IOCTL_SIOCSIFBRDADDR = SIOCSIFBRDADDR; + unsigned IOCTL_SIOCSIFDSTADDR = SIOCSIFDSTADDR; + unsigned IOCTL_SIOCSIFFLAGS = SIOCSIFFLAGS; + unsigned IOCTL_SIOCSIFMETRIC = SIOCSIFMETRIC; + unsigned IOCTL_SIOCSIFMTU = SIOCSIFMTU; + unsigned IOCTL_SIOCSIFNETMASK = SIOCSIFNETMASK; + unsigned IOCTL_SIOCSPGRP = SIOCSPGRP; + unsigned IOCTL_TIOCCONS = TIOCCONS; + unsigned IOCTL_TIOCEXCL = TIOCEXCL; + unsigned IOCTL_TIOCGETD = TIOCGETD; + unsigned IOCTL_TIOCGPGRP = TIOCGPGRP; + unsigned IOCTL_TIOCGWINSZ = TIOCGWINSZ; + unsigned IOCTL_TIOCMBIC = TIOCMBIC; + unsigned IOCTL_TIOCMBIS = TIOCMBIS; + unsigned IOCTL_TIOCMGET = TIOCMGET; + unsigned IOCTL_TIOCMSET = TIOCMSET; + unsigned IOCTL_TIOCNOTTY = TIOCNOTTY; + unsigned IOCTL_TIOCNXCL = TIOCNXCL; + unsigned IOCTL_TIOCOUTQ = TIOCOUTQ; + unsigned IOCTL_TIOCPKT = TIOCPKT; + unsigned IOCTL_TIOCSCTTY = TIOCSCTTY; + unsigned IOCTL_TIOCSETD = TIOCSETD; + unsigned IOCTL_TIOCSPGRP = TIOCSPGRP; + unsigned IOCTL_TIOCSTI = TIOCSTI; + unsigned IOCTL_TIOCSWINSZ = TIOCSWINSZ; + unsigned IOCTL_SIOCGETSGCNT = SIOCGETSGCNT; + unsigned IOCTL_SIOCGETVIFCNT = SIOCGETVIFCNT; + unsigned IOCTL_MTIOCGET = MTIOCGET; + unsigned IOCTL_MTIOCTOP = MTIOCTOP; + unsigned IOCTL_SNDCTL_DSP_GETBLKSIZE = SNDCTL_DSP_GETBLKSIZE; + unsigned IOCTL_SNDCTL_DSP_GETFMTS = SNDCTL_DSP_GETFMTS; + unsigned IOCTL_SNDCTL_DSP_NONBLOCK = SNDCTL_DSP_NONBLOCK; + unsigned IOCTL_SNDCTL_DSP_POST = SNDCTL_DSP_POST; + unsigned IOCTL_SNDCTL_DSP_RESET = SNDCTL_DSP_RESET; + unsigned IOCTL_SNDCTL_DSP_SETFMT = SNDCTL_DSP_SETFMT; + unsigned IOCTL_SNDCTL_DSP_SETFRAGMENT = SNDCTL_DSP_SETFRAGMENT; + unsigned IOCTL_SNDCTL_DSP_SPEED = SNDCTL_DSP_SPEED; + unsigned IOCTL_SNDCTL_DSP_STEREO = SNDCTL_DSP_STEREO; + unsigned IOCTL_SNDCTL_DSP_SUBDIVIDE = SNDCTL_DSP_SUBDIVIDE; + unsigned IOCTL_SNDCTL_DSP_SYNC = SNDCTL_DSP_SYNC; + unsigned IOCTL_SNDCTL_FM_4OP_ENABLE = SNDCTL_FM_4OP_ENABLE; + unsigned IOCTL_SNDCTL_FM_LOAD_INSTR = SNDCTL_FM_LOAD_INSTR; + unsigned IOCTL_SNDCTL_MIDI_INFO = SNDCTL_MIDI_INFO; + unsigned IOCTL_SNDCTL_MIDI_PRETIME = SNDCTL_MIDI_PRETIME; + unsigned IOCTL_SNDCTL_SEQ_CTRLRATE = SNDCTL_SEQ_CTRLRATE; + unsigned IOCTL_SNDCTL_SEQ_GETINCOUNT = SNDCTL_SEQ_GETINCOUNT; + unsigned IOCTL_SNDCTL_SEQ_GETOUTCOUNT = SNDCTL_SEQ_GETOUTCOUNT; + unsigned IOCTL_SNDCTL_SEQ_NRMIDIS = SNDCTL_SEQ_NRMIDIS; + unsigned IOCTL_SNDCTL_SEQ_NRSYNTHS = SNDCTL_SEQ_NRSYNTHS; + unsigned IOCTL_SNDCTL_SEQ_OUTOFBAND = SNDCTL_SEQ_OUTOFBAND; + unsigned IOCTL_SNDCTL_SEQ_PANIC = SNDCTL_SEQ_PANIC; + unsigned IOCTL_SNDCTL_SEQ_PERCMODE = SNDCTL_SEQ_PERCMODE; + unsigned IOCTL_SNDCTL_SEQ_RESET = SNDCTL_SEQ_RESET; + unsigned IOCTL_SNDCTL_SEQ_RESETSAMPLES = SNDCTL_SEQ_RESETSAMPLES; + unsigned IOCTL_SNDCTL_SEQ_SYNC = SNDCTL_SEQ_SYNC; + unsigned IOCTL_SNDCTL_SEQ_TESTMIDI = SNDCTL_SEQ_TESTMIDI; + unsigned IOCTL_SNDCTL_SEQ_THRESHOLD = SNDCTL_SEQ_THRESHOLD; + unsigned IOCTL_SNDCTL_SYNTH_INFO = SNDCTL_SYNTH_INFO; + unsigned IOCTL_SNDCTL_SYNTH_MEMAVL = SNDCTL_SYNTH_MEMAVL; + unsigned IOCTL_SNDCTL_TMR_CONTINUE = SNDCTL_TMR_CONTINUE; + unsigned IOCTL_SNDCTL_TMR_METRONOME = SNDCTL_TMR_METRONOME; + unsigned IOCTL_SNDCTL_TMR_SELECT = SNDCTL_TMR_SELECT; + unsigned IOCTL_SNDCTL_TMR_SOURCE = SNDCTL_TMR_SOURCE; + unsigned IOCTL_SNDCTL_TMR_START = SNDCTL_TMR_START; + unsigned IOCTL_SNDCTL_TMR_STOP = SNDCTL_TMR_STOP; + unsigned IOCTL_SNDCTL_TMR_TEMPO = SNDCTL_TMR_TEMPO; + unsigned IOCTL_SNDCTL_TMR_TIMEBASE = SNDCTL_TMR_TIMEBASE; + unsigned IOCTL_SOUND_MIXER_READ_ALTPCM = SOUND_MIXER_READ_ALTPCM; + unsigned IOCTL_SOUND_MIXER_READ_BASS = SOUND_MIXER_READ_BASS; + unsigned IOCTL_SOUND_MIXER_READ_CAPS = SOUND_MIXER_READ_CAPS; + unsigned IOCTL_SOUND_MIXER_READ_CD = SOUND_MIXER_READ_CD; + unsigned IOCTL_SOUND_MIXER_READ_DEVMASK = SOUND_MIXER_READ_DEVMASK; + unsigned IOCTL_SOUND_MIXER_READ_ENHANCE = SOUND_MIXER_READ_ENHANCE; + unsigned IOCTL_SOUND_MIXER_READ_IGAIN = SOUND_MIXER_READ_IGAIN; + unsigned IOCTL_SOUND_MIXER_READ_IMIX = SOUND_MIXER_READ_IMIX; + unsigned IOCTL_SOUND_MIXER_READ_LINE = SOUND_MIXER_READ_LINE; + unsigned IOCTL_SOUND_MIXER_READ_LINE1 = SOUND_MIXER_READ_LINE1; + unsigned IOCTL_SOUND_MIXER_READ_LINE2 = SOUND_MIXER_READ_LINE2; + unsigned IOCTL_SOUND_MIXER_READ_LINE3 = SOUND_MIXER_READ_LINE3; + unsigned IOCTL_SOUND_MIXER_READ_LOUD = SOUND_MIXER_READ_LOUD; + unsigned IOCTL_SOUND_MIXER_READ_MIC = SOUND_MIXER_READ_MIC; + unsigned IOCTL_SOUND_MIXER_READ_MUTE = SOUND_MIXER_READ_MUTE; + unsigned IOCTL_SOUND_MIXER_READ_OGAIN = SOUND_MIXER_READ_OGAIN; + unsigned IOCTL_SOUND_MIXER_READ_PCM = SOUND_MIXER_READ_PCM; + unsigned IOCTL_SOUND_MIXER_READ_RECLEV = SOUND_MIXER_READ_RECLEV; + unsigned IOCTL_SOUND_MIXER_READ_RECMASK = SOUND_MIXER_READ_RECMASK; + unsigned IOCTL_SOUND_MIXER_READ_RECSRC = SOUND_MIXER_READ_RECSRC; + unsigned IOCTL_SOUND_MIXER_READ_SPEAKER = SOUND_MIXER_READ_SPEAKER; + unsigned IOCTL_SOUND_MIXER_READ_STEREODEVS = SOUND_MIXER_READ_STEREODEVS; + unsigned IOCTL_SOUND_MIXER_READ_SYNTH = SOUND_MIXER_READ_SYNTH; + unsigned IOCTL_SOUND_MIXER_READ_TREBLE = SOUND_MIXER_READ_TREBLE; + unsigned IOCTL_SOUND_MIXER_READ_VOLUME = SOUND_MIXER_READ_VOLUME; + unsigned IOCTL_SOUND_MIXER_WRITE_ALTPCM = SOUND_MIXER_WRITE_ALTPCM; + unsigned IOCTL_SOUND_MIXER_WRITE_BASS = SOUND_MIXER_WRITE_BASS; + unsigned IOCTL_SOUND_MIXER_WRITE_CD = SOUND_MIXER_WRITE_CD; + unsigned IOCTL_SOUND_MIXER_WRITE_ENHANCE = SOUND_MIXER_WRITE_ENHANCE; + unsigned IOCTL_SOUND_MIXER_WRITE_IGAIN = SOUND_MIXER_WRITE_IGAIN; + unsigned IOCTL_SOUND_MIXER_WRITE_IMIX = SOUND_MIXER_WRITE_IMIX; + unsigned IOCTL_SOUND_MIXER_WRITE_LINE = SOUND_MIXER_WRITE_LINE; + unsigned IOCTL_SOUND_MIXER_WRITE_LINE1 = SOUND_MIXER_WRITE_LINE1; + unsigned IOCTL_SOUND_MIXER_WRITE_LINE2 = SOUND_MIXER_WRITE_LINE2; + unsigned IOCTL_SOUND_MIXER_WRITE_LINE3 = SOUND_MIXER_WRITE_LINE3; + unsigned IOCTL_SOUND_MIXER_WRITE_LOUD = SOUND_MIXER_WRITE_LOUD; + unsigned IOCTL_SOUND_MIXER_WRITE_MIC = SOUND_MIXER_WRITE_MIC; + unsigned IOCTL_SOUND_MIXER_WRITE_MUTE = SOUND_MIXER_WRITE_MUTE; + unsigned IOCTL_SOUND_MIXER_WRITE_OGAIN = SOUND_MIXER_WRITE_OGAIN; + unsigned IOCTL_SOUND_MIXER_WRITE_PCM = SOUND_MIXER_WRITE_PCM; + unsigned IOCTL_SOUND_MIXER_WRITE_RECLEV = SOUND_MIXER_WRITE_RECLEV; + unsigned IOCTL_SOUND_MIXER_WRITE_RECSRC = SOUND_MIXER_WRITE_RECSRC; + unsigned IOCTL_SOUND_MIXER_WRITE_SPEAKER = SOUND_MIXER_WRITE_SPEAKER; + unsigned IOCTL_SOUND_MIXER_WRITE_SYNTH = SOUND_MIXER_WRITE_SYNTH; + unsigned IOCTL_SOUND_MIXER_WRITE_TREBLE = SOUND_MIXER_WRITE_TREBLE; + unsigned IOCTL_SOUND_MIXER_WRITE_VOLUME = SOUND_MIXER_WRITE_VOLUME; + unsigned IOCTL_VT_ACTIVATE = VT_ACTIVATE; + unsigned IOCTL_VT_GETMODE = VT_GETMODE; + unsigned IOCTL_VT_OPENQRY = VT_OPENQRY; + unsigned IOCTL_VT_RELDISP = VT_RELDISP; + unsigned IOCTL_VT_SETMODE = VT_SETMODE; + unsigned IOCTL_VT_WAITACTIVE = VT_WAITACTIVE; + unsigned IOCTL_GIO_SCRNMAP = GIO_SCRNMAP; + unsigned IOCTL_KDDISABIO = KDDISABIO; + unsigned IOCTL_KDENABIO = KDENABIO; + unsigned IOCTL_KDGETLED = KDGETLED; + unsigned IOCTL_KDGETMODE = KDGETMODE; + unsigned IOCTL_KDGKBMODE = KDGKBMODE; + unsigned IOCTL_KDGKBTYPE = KDGKBTYPE; + unsigned IOCTL_KDMKTONE = KDMKTONE; + unsigned IOCTL_KDSETLED = KDSETLED; + unsigned IOCTL_KDSETMODE = KDSETMODE; + unsigned IOCTL_KDSKBMODE = KDSKBMODE; + unsigned IOCTL_KIOCSOUND = KIOCSOUND; + unsigned IOCTL_PIO_SCRNMAP = PIO_SCRNMAP; + unsigned IOCTL_SNDCTL_DSP_GETISPACE = SNDCTL_DSP_GETISPACE; + + const int si_SEGV_MAPERR = SEGV_MAPERR; + const int si_SEGV_ACCERR = SEGV_ACCERR; + const int unvis_valid = UNVIS_VALID; + const int unvis_validpush = UNVIS_VALIDPUSH; +} // namespace __sanitizer + +using namespace __sanitizer; + +COMPILER_CHECK(sizeof(__sanitizer_pthread_attr_t) >= sizeof(pthread_attr_t)); + +COMPILER_CHECK(sizeof(socklen_t) == sizeof(unsigned)); +CHECK_TYPE_SIZE(pthread_key_t); + +// There are more undocumented fields in dl_phdr_info that we are not interested +// in. +COMPILER_CHECK(sizeof(__sanitizer_dl_phdr_info) <= sizeof(dl_phdr_info)); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_addr); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_name); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phdr); +CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phnum); + +CHECK_TYPE_SIZE(glob_t); +CHECK_SIZE_AND_OFFSET(glob_t, gl_pathc); +CHECK_SIZE_AND_OFFSET(glob_t, gl_pathv); +CHECK_SIZE_AND_OFFSET(glob_t, gl_offs); +CHECK_SIZE_AND_OFFSET(glob_t, gl_flags); +CHECK_SIZE_AND_OFFSET(glob_t, gl_closedir); +CHECK_SIZE_AND_OFFSET(glob_t, gl_readdir); +CHECK_SIZE_AND_OFFSET(glob_t, gl_opendir); +CHECK_SIZE_AND_OFFSET(glob_t, gl_lstat); +CHECK_SIZE_AND_OFFSET(glob_t, gl_stat); + +CHECK_TYPE_SIZE(addrinfo); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_flags); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_family); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_socktype); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_addrlen); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_canonname); +CHECK_SIZE_AND_OFFSET(addrinfo, ai_addr); + +CHECK_TYPE_SIZE(hostent); +CHECK_SIZE_AND_OFFSET(hostent, h_name); +CHECK_SIZE_AND_OFFSET(hostent, h_aliases); +CHECK_SIZE_AND_OFFSET(hostent, h_addrtype); +CHECK_SIZE_AND_OFFSET(hostent, h_length); +CHECK_SIZE_AND_OFFSET(hostent, h_addr_list); + +CHECK_TYPE_SIZE(iovec); +CHECK_SIZE_AND_OFFSET(iovec, iov_base); +CHECK_SIZE_AND_OFFSET(iovec, iov_len); + +CHECK_TYPE_SIZE(msghdr); +CHECK_SIZE_AND_OFFSET(msghdr, msg_name); +CHECK_SIZE_AND_OFFSET(msghdr, msg_namelen); +CHECK_SIZE_AND_OFFSET(msghdr, msg_iov); +CHECK_SIZE_AND_OFFSET(msghdr, msg_iovlen); +CHECK_SIZE_AND_OFFSET(msghdr, msg_control); +CHECK_SIZE_AND_OFFSET(msghdr, msg_controllen); +CHECK_SIZE_AND_OFFSET(msghdr, msg_flags); + +CHECK_TYPE_SIZE(cmsghdr); +CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_len); +CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_level); +CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_type); + +COMPILER_CHECK(sizeof(__sanitizer_dirent) <= sizeof(dirent)); +CHECK_SIZE_AND_OFFSET(dirent, d_ino); +CHECK_SIZE_AND_OFFSET(dirent, d_reclen); + +CHECK_TYPE_SIZE(ifconf); +CHECK_SIZE_AND_OFFSET(ifconf, ifc_len); +CHECK_SIZE_AND_OFFSET(ifconf, ifc_ifcu); + +CHECK_TYPE_SIZE(pollfd); +CHECK_SIZE_AND_OFFSET(pollfd, fd); +CHECK_SIZE_AND_OFFSET(pollfd, events); +CHECK_SIZE_AND_OFFSET(pollfd, revents); + +CHECK_TYPE_SIZE(nfds_t); + +CHECK_TYPE_SIZE(sigset_t); + +COMPILER_CHECK(sizeof(__sanitizer_sigaction) == sizeof(struct sigaction)); +// Can't write checks for sa_handler and sa_sigaction due to them being +// preprocessor macros. +CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_mask); + +CHECK_TYPE_SIZE(wordexp_t); +CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordc); +CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordv); +CHECK_SIZE_AND_OFFSET(wordexp_t, we_offs); + +CHECK_TYPE_SIZE(tm); +CHECK_SIZE_AND_OFFSET(tm, tm_sec); +CHECK_SIZE_AND_OFFSET(tm, tm_min); +CHECK_SIZE_AND_OFFSET(tm, tm_hour); +CHECK_SIZE_AND_OFFSET(tm, tm_mday); +CHECK_SIZE_AND_OFFSET(tm, tm_mon); +CHECK_SIZE_AND_OFFSET(tm, tm_year); +CHECK_SIZE_AND_OFFSET(tm, tm_wday); +CHECK_SIZE_AND_OFFSET(tm, tm_yday); +CHECK_SIZE_AND_OFFSET(tm, tm_isdst); +CHECK_SIZE_AND_OFFSET(tm, tm_gmtoff); +CHECK_SIZE_AND_OFFSET(tm, tm_zone); + +CHECK_TYPE_SIZE(ether_addr); + +CHECK_TYPE_SIZE(ipc_perm); +CHECK_SIZE_AND_OFFSET(ipc_perm, key); +CHECK_SIZE_AND_OFFSET(ipc_perm, seq); +CHECK_SIZE_AND_OFFSET(ipc_perm, uid); +CHECK_SIZE_AND_OFFSET(ipc_perm, gid); +CHECK_SIZE_AND_OFFSET(ipc_perm, cuid); +CHECK_SIZE_AND_OFFSET(ipc_perm, cgid); + +CHECK_TYPE_SIZE(shmid_ds); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_perm); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_segsz); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_atime); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_dtime); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_ctime); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_cpid); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_lpid); +CHECK_SIZE_AND_OFFSET(shmid_ds, shm_nattch); + +CHECK_TYPE_SIZE(clock_t); + +CHECK_TYPE_SIZE(ifaddrs); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_next); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_name); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_addr); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_netmask); +#undef ifa_dstaddr +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_dstaddr); +CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_data); + +CHECK_TYPE_SIZE(timeb); +CHECK_SIZE_AND_OFFSET(timeb, time); +CHECK_SIZE_AND_OFFSET(timeb, millitm); +CHECK_SIZE_AND_OFFSET(timeb, timezone); +CHECK_SIZE_AND_OFFSET(timeb, dstflag); + +CHECK_TYPE_SIZE(passwd); +CHECK_SIZE_AND_OFFSET(passwd, pw_name); +CHECK_SIZE_AND_OFFSET(passwd, pw_passwd); +CHECK_SIZE_AND_OFFSET(passwd, pw_uid); +CHECK_SIZE_AND_OFFSET(passwd, pw_gid); +CHECK_SIZE_AND_OFFSET(passwd, pw_dir); +CHECK_SIZE_AND_OFFSET(passwd, pw_shell); + +CHECK_SIZE_AND_OFFSET(passwd, pw_gecos); + +CHECK_TYPE_SIZE(group); +CHECK_SIZE_AND_OFFSET(group, gr_name); +CHECK_SIZE_AND_OFFSET(group, gr_passwd); +CHECK_SIZE_AND_OFFSET(group, gr_gid); +CHECK_SIZE_AND_OFFSET(group, gr_mem); + +#if HAVE_RPC_XDR_H +CHECK_TYPE_SIZE(XDR); +CHECK_SIZE_AND_OFFSET(XDR, x_op); +CHECK_SIZE_AND_OFFSET(XDR, x_ops); +CHECK_SIZE_AND_OFFSET(XDR, x_public); +CHECK_SIZE_AND_OFFSET(XDR, x_private); +CHECK_SIZE_AND_OFFSET(XDR, x_base); +CHECK_SIZE_AND_OFFSET(XDR, x_handy); +COMPILER_CHECK(__sanitizer_XDR_ENCODE == XDR_ENCODE); +COMPILER_CHECK(__sanitizer_XDR_DECODE == XDR_DECODE); +COMPILER_CHECK(__sanitizer_XDR_FREE == XDR_FREE); +#endif + +CHECK_TYPE_SIZE(sem_t); + +COMPILER_CHECK(sizeof(__sanitizer_cap_rights_t) >= sizeof(cap_rights_t)); +#endif // SANITIZER_FREEBSD diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h new file mode 100644 index 000000000000..588bead2ac12 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h @@ -0,0 +1,657 @@ +//===-- sanitizer_platform_limits_freebsd.h -------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of Sanitizer common code. +// +// Sizes and layouts of platform-specific FreeBSD data structures. +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_PLATFORM_LIMITS_FREEBSD_H +#define SANITIZER_PLATFORM_LIMITS_FREEBSD_H + +#if SANITIZER_FREEBSD + +#include "sanitizer_internal_defs.h" +#include "sanitizer_platform.h" + +#include "sanitizer_platform_limits_posix.h" + +// FreeBSD's dlopen() returns a pointer to an Obj_Entry structure that +// incorporates the map structure. +# define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \ + ((link_map*)((handle) == nullptr ? nullptr : ((char*)(handle) + 560))) +// Get sys/_types.h, because that tells us whether 64-bit inodes are +// used in struct dirent below. +#include <sys/_types.h> + +namespace __sanitizer { + extern unsigned struct_utsname_sz; + extern unsigned struct_stat_sz; +#if defined(__powerpc64__) + const unsigned struct___old_kernel_stat_sz = 0; +#else + const unsigned struct___old_kernel_stat_sz = 32; +#endif + extern unsigned struct_rusage_sz; + extern unsigned siginfo_t_sz; + extern unsigned struct_itimerval_sz; + extern unsigned pthread_t_sz; + extern unsigned pthread_mutex_t_sz; + extern unsigned pthread_cond_t_sz; + extern unsigned pid_t_sz; + extern unsigned timeval_sz; + extern unsigned uid_t_sz; + extern unsigned gid_t_sz; + extern unsigned fpos_t_sz; + extern unsigned mbstate_t_sz; + extern unsigned struct_timezone_sz; + extern unsigned struct_tms_sz; + extern unsigned struct_itimerspec_sz; + extern unsigned struct_sigevent_sz; + extern unsigned struct_sched_param_sz; + extern unsigned struct_statfs64_sz; + extern unsigned struct_statfs_sz; + extern unsigned struct_sockaddr_sz; + extern unsigned ucontext_t_sz; + extern unsigned struct_rlimit_sz; + extern unsigned struct_utimbuf_sz; + extern unsigned struct_timespec_sz; + extern unsigned struct_regmatch_sz; + extern unsigned struct_regex_sz; + extern unsigned struct_FTS_sz; + extern unsigned struct_FTSENT_sz; + extern const int unvis_valid; + extern const int unvis_validpush; + + struct __sanitizer_iocb { + u64 aio_data; + u32 aio_key_or_aio_reserved1; // Simply crazy. + u32 aio_reserved1_or_aio_key; // Luckily, we don't need these. + u16 aio_lio_opcode; + s16 aio_reqprio; + u32 aio_fildes; + u64 aio_buf; + u64 aio_nbytes; + s64 aio_offset; + u64 aio_reserved2; + u64 aio_reserved3; + }; + + struct __sanitizer_io_event { + u64 data; + u64 obj; + u64 res; + u64 res2; + }; + + const unsigned iocb_cmd_pread = 0; + const unsigned iocb_cmd_pwrite = 1; + const unsigned iocb_cmd_preadv = 7; + const unsigned iocb_cmd_pwritev = 8; + + struct __sanitizer___sysctl_args { + int *name; + int nlen; + void *oldval; + uptr *oldlenp; + void *newval; + uptr newlen; + unsigned long ___unused[4]; + }; + + struct __sanitizer_ipc_perm { + unsigned int cuid; + unsigned int cgid; + unsigned int uid; + unsigned int gid; + unsigned short mode; + unsigned short seq; + long key; + }; + + struct __sanitizer_shmid_ds { + __sanitizer_ipc_perm shm_perm; + unsigned long shm_segsz; + unsigned int shm_lpid; + unsigned int shm_cpid; + int shm_nattch; + unsigned long shm_atime; + unsigned long shm_dtime; + unsigned long shm_ctime; + }; + + extern unsigned struct_msqid_ds_sz; + extern unsigned struct_mq_attr_sz; + extern unsigned struct_timeb_sz; + extern unsigned struct_statvfs_sz; + + struct __sanitizer_iovec { + void *iov_base; + uptr iov_len; + }; + + struct __sanitizer_ifaddrs { + struct __sanitizer_ifaddrs *ifa_next; + char *ifa_name; + unsigned int ifa_flags; + void *ifa_addr; // (struct sockaddr *) + void *ifa_netmask; // (struct sockaddr *) +# undef ifa_dstaddr + void *ifa_dstaddr; // (struct sockaddr *) + void *ifa_data; + }; + + typedef unsigned __sanitizer_pthread_key_t; + + struct __sanitizer_passwd { + char *pw_name; + char *pw_passwd; + int pw_uid; + int pw_gid; + long pw_change; + char *pw_class; + char *pw_gecos; + char *pw_dir; + char *pw_shell; + long pw_expire; + int pw_fields; + }; + + struct __sanitizer_group { + char *gr_name; + char *gr_passwd; + int gr_gid; + char **gr_mem; + }; + +#if defined(__LP64___) + typedef long long __sanitizer_time_t; +#else + typedef long __sanitizer_time_t; +#endif + + typedef long __sanitizer_suseconds_t; + + struct __sanitizer_timeval { + __sanitizer_time_t tv_sec; + __sanitizer_suseconds_t tv_usec; + }; + + struct __sanitizer_itimerval { + struct __sanitizer_timeval it_interval; + struct __sanitizer_timeval it_value; + }; + + struct __sanitizer_timeb { + __sanitizer_time_t time; + unsigned short millitm; + short timezone; + short dstflag; + }; + + struct __sanitizer_ether_addr { + u8 octet[6]; + }; + + struct __sanitizer_tm { + int tm_sec; + int tm_min; + int tm_hour; + int tm_mday; + int tm_mon; + int tm_year; + int tm_wday; + int tm_yday; + int tm_isdst; + long int tm_gmtoff; + const char *tm_zone; + }; + + struct __sanitizer_msghdr { + void *msg_name; + unsigned msg_namelen; + struct __sanitizer_iovec *msg_iov; + unsigned msg_iovlen; + void *msg_control; + unsigned msg_controllen; + int msg_flags; + }; + + struct __sanitizer_cmsghdr { + unsigned cmsg_len; + int cmsg_level; + int cmsg_type; + }; + + struct __sanitizer_dirent { +#if defined(__INO64) + unsigned long long d_fileno; + unsigned long long d_off; +#else + unsigned int d_fileno; +#endif + unsigned short d_reclen; + // more fields that we don't care about + }; + +// 'clock_t' is 32 bits wide on x64 FreeBSD + typedef int __sanitizer_clock_t; + typedef int __sanitizer_clockid_t; + +#if defined(_LP64) || defined(__x86_64__) || defined(__powerpc__)\ + || defined(__mips__) + typedef unsigned __sanitizer___kernel_uid_t; + typedef unsigned __sanitizer___kernel_gid_t; +#else + typedef unsigned short __sanitizer___kernel_uid_t; + typedef unsigned short __sanitizer___kernel_gid_t; +#endif + typedef long long __sanitizer___kernel_off_t; + +#if defined(__powerpc__) || defined(__mips__) + typedef unsigned int __sanitizer___kernel_old_uid_t; + typedef unsigned int __sanitizer___kernel_old_gid_t; +#else + typedef unsigned short __sanitizer___kernel_old_uid_t; + typedef unsigned short __sanitizer___kernel_old_gid_t; +#endif + + typedef long long __sanitizer___kernel_loff_t; + typedef struct { + unsigned long fds_bits[1024 / (8 * sizeof(long))]; + } __sanitizer___kernel_fd_set; + + // This thing depends on the platform. We are only interested in the upper + // limit. Verified with a compiler assert in .cc. + const int pthread_attr_t_max_sz = 128; + union __sanitizer_pthread_attr_t { + char size[pthread_attr_t_max_sz]; // NOLINT + void *align; + }; + + const unsigned old_sigset_t_sz = sizeof(unsigned long); + + struct __sanitizer_sigset_t { + // uint32_t * 4 + unsigned int __bits[4]; + }; + + typedef __sanitizer_sigset_t __sanitizer_kernel_sigset_t; + + struct __sanitizer_siginfo { + // The size is determined by looking at sizeof of real siginfo_t on linux. + u64 opaque[128 / sizeof(u64)]; + }; + + using __sanitizer_sighandler_ptr = void (*)(int sig); + using __sanitizer_sigactionhandler_ptr = + void (*)(int sig, __sanitizer_siginfo *siginfo, void *uctx); + + struct __sanitizer_sigaction { + union { + __sanitizer_sigactionhandler_ptr sigaction; + __sanitizer_sighandler_ptr handler; + }; + int sa_flags; + __sanitizer_sigset_t sa_mask; + }; + + struct __sanitizer_sem_t { + u32 data[4]; + }; + + extern const uptr sig_ign; + extern const uptr sig_dfl; + extern const uptr sig_err; + extern const uptr sa_siginfo; + + extern int af_inet; + extern int af_inet6; + uptr __sanitizer_in_addr_sz(int af); + + struct __sanitizer_dl_phdr_info { + uptr dlpi_addr; + const char *dlpi_name; + const void *dlpi_phdr; + short dlpi_phnum; + }; + + extern unsigned struct_ElfW_Phdr_sz; + + struct __sanitizer_addrinfo { + int ai_flags; + int ai_family; + int ai_socktype; + int ai_protocol; + unsigned ai_addrlen; + char *ai_canonname; + void *ai_addr; + struct __sanitizer_addrinfo *ai_next; + }; + + struct __sanitizer_hostent { + char *h_name; + char **h_aliases; + int h_addrtype; + int h_length; + char **h_addr_list; + }; + + struct __sanitizer_pollfd { + int fd; + short events; + short revents; + }; + + typedef unsigned __sanitizer_nfds_t; + + struct __sanitizer_glob_t { + uptr gl_pathc; + uptr gl_matchc; + uptr gl_offs; + int gl_flags; + char **gl_pathv; + int (*gl_errfunc)(const char*, int); + void (*gl_closedir)(void *dirp); + struct dirent *(*gl_readdir)(void *dirp); + void *(*gl_opendir)(const char*); + int (*gl_lstat)(const char*, void* /* struct stat* */); + int (*gl_stat)(const char*, void* /* struct stat* */); + }; + + extern int glob_nomatch; + extern int glob_altdirfunc; + + extern unsigned path_max; + + struct __sanitizer_wordexp_t { + uptr we_wordc; + char **we_wordv; + uptr we_offs; + char *we_strings; + uptr we_nbytes; + }; + + typedef void __sanitizer_FILE; + + extern unsigned struct_shminfo_sz; + extern unsigned struct_shm_info_sz; + extern int shmctl_ipc_stat; + extern int shmctl_ipc_info; + extern int shmctl_shm_info; + extern int shmctl_shm_stat; + + extern unsigned struct_utmpx_sz; + + extern int map_fixed; + + // ioctl arguments + struct __sanitizer_ifconf { + int ifc_len; + union { + void *ifcu_req; + } ifc_ifcu; + }; + +#define IOC_NRBITS 8 +#define IOC_TYPEBITS 8 +#if defined(__powerpc__) || defined(__powerpc64__) || defined(__mips__) +#define IOC_SIZEBITS 13 +#define IOC_DIRBITS 3 +#define IOC_NONE 1U +#define IOC_WRITE 4U +#define IOC_READ 2U +#else +#define IOC_SIZEBITS 14 +#define IOC_DIRBITS 2 +#define IOC_NONE 0U +#define IOC_WRITE 1U +#define IOC_READ 2U +#endif +#define IOC_NRMASK ((1 << IOC_NRBITS) - 1) +#define IOC_TYPEMASK ((1 << IOC_TYPEBITS) - 1) +#define IOC_SIZEMASK ((1 << IOC_SIZEBITS) - 1) +#if defined(IOC_DIRMASK) +#undef IOC_DIRMASK +#endif +#define IOC_DIRMASK ((1 << IOC_DIRBITS) - 1) +#define IOC_NRSHIFT 0 +#define IOC_TYPESHIFT (IOC_NRSHIFT + IOC_NRBITS) +#define IOC_SIZESHIFT (IOC_TYPESHIFT + IOC_TYPEBITS) +#define IOC_DIRSHIFT (IOC_SIZESHIFT + IOC_SIZEBITS) +#define EVIOC_EV_MAX 0x1f +#define EVIOC_ABS_MAX 0x3f + +#define IOC_DIR(nr) (((nr) >> IOC_DIRSHIFT) & IOC_DIRMASK) +#define IOC_TYPE(nr) (((nr) >> IOC_TYPESHIFT) & IOC_TYPEMASK) +#define IOC_NR(nr) (((nr) >> IOC_NRSHIFT) & IOC_NRMASK) +#define IOC_SIZE(nr) (((nr) >> IOC_SIZESHIFT) & IOC_SIZEMASK) + + extern unsigned struct_ifreq_sz; + extern unsigned struct_termios_sz; + extern unsigned struct_winsize_sz; + + extern unsigned struct_copr_buffer_sz; + extern unsigned struct_copr_debug_buf_sz; + extern unsigned struct_copr_msg_sz; + extern unsigned struct_midi_info_sz; + extern unsigned struct_mtget_sz; + extern unsigned struct_mtop_sz; + extern unsigned struct_rtentry_sz; + extern unsigned struct_sbi_instrument_sz; + extern unsigned struct_seq_event_rec_sz; + extern unsigned struct_synth_info_sz; + extern unsigned struct_vt_mode_sz; + + extern const unsigned long __sanitizer_bufsiz; + extern unsigned struct_audio_buf_info_sz; + extern unsigned struct_ppp_stats_sz; + extern unsigned struct_sioc_sg_req_sz; + extern unsigned struct_sioc_vif_req_sz; + + // ioctl request identifiers + + // A special value to mark ioctls that are not present on the target platform, + // when it can not be determined without including any system headers. + extern const unsigned IOCTL_NOT_PRESENT; + + extern unsigned IOCTL_FIOASYNC; + extern unsigned IOCTL_FIOCLEX; + extern unsigned IOCTL_FIOGETOWN; + extern unsigned IOCTL_FIONBIO; + extern unsigned IOCTL_FIONCLEX; + extern unsigned IOCTL_FIOSETOWN; + extern unsigned IOCTL_SIOCADDMULTI; + extern unsigned IOCTL_SIOCATMARK; + extern unsigned IOCTL_SIOCDELMULTI; + extern unsigned IOCTL_SIOCGIFADDR; + extern unsigned IOCTL_SIOCGIFBRDADDR; + extern unsigned IOCTL_SIOCGIFCONF; + extern unsigned IOCTL_SIOCGIFDSTADDR; + extern unsigned IOCTL_SIOCGIFFLAGS; + extern unsigned IOCTL_SIOCGIFMETRIC; + extern unsigned IOCTL_SIOCGIFMTU; + extern unsigned IOCTL_SIOCGIFNETMASK; + extern unsigned IOCTL_SIOCGPGRP; + extern unsigned IOCTL_SIOCSIFADDR; + extern unsigned IOCTL_SIOCSIFBRDADDR; + extern unsigned IOCTL_SIOCSIFDSTADDR; + extern unsigned IOCTL_SIOCSIFFLAGS; + extern unsigned IOCTL_SIOCSIFMETRIC; + extern unsigned IOCTL_SIOCSIFMTU; + extern unsigned IOCTL_SIOCSIFNETMASK; + extern unsigned IOCTL_SIOCSPGRP; + extern unsigned IOCTL_TIOCCONS; + extern unsigned IOCTL_TIOCEXCL; + extern unsigned IOCTL_TIOCGETD; + extern unsigned IOCTL_TIOCGPGRP; + extern unsigned IOCTL_TIOCGWINSZ; + extern unsigned IOCTL_TIOCMBIC; + extern unsigned IOCTL_TIOCMBIS; + extern unsigned IOCTL_TIOCMGET; + extern unsigned IOCTL_TIOCMSET; + extern unsigned IOCTL_TIOCNOTTY; + extern unsigned IOCTL_TIOCNXCL; + extern unsigned IOCTL_TIOCOUTQ; + extern unsigned IOCTL_TIOCPKT; + extern unsigned IOCTL_TIOCSCTTY; + extern unsigned IOCTL_TIOCSETD; + extern unsigned IOCTL_TIOCSPGRP; + extern unsigned IOCTL_TIOCSTI; + extern unsigned IOCTL_TIOCSWINSZ; + extern unsigned IOCTL_SIOCGETSGCNT; + extern unsigned IOCTL_SIOCGETVIFCNT; + extern unsigned IOCTL_MTIOCGET; + extern unsigned IOCTL_MTIOCTOP; + extern unsigned IOCTL_SIOCADDRT; + extern unsigned IOCTL_SIOCDELRT; + extern unsigned IOCTL_SNDCTL_DSP_GETBLKSIZE; + extern unsigned IOCTL_SNDCTL_DSP_GETFMTS; + extern unsigned IOCTL_SNDCTL_DSP_NONBLOCK; + extern unsigned IOCTL_SNDCTL_DSP_POST; + extern unsigned IOCTL_SNDCTL_DSP_RESET; + extern unsigned IOCTL_SNDCTL_DSP_SETFMT; + extern unsigned IOCTL_SNDCTL_DSP_SETFRAGMENT; + extern unsigned IOCTL_SNDCTL_DSP_SPEED; + extern unsigned IOCTL_SNDCTL_DSP_STEREO; + extern unsigned IOCTL_SNDCTL_DSP_SUBDIVIDE; + extern unsigned IOCTL_SNDCTL_DSP_SYNC; + extern unsigned IOCTL_SNDCTL_FM_4OP_ENABLE; + extern unsigned IOCTL_SNDCTL_FM_LOAD_INSTR; + extern unsigned IOCTL_SNDCTL_MIDI_INFO; + extern unsigned IOCTL_SNDCTL_MIDI_PRETIME; + extern unsigned IOCTL_SNDCTL_SEQ_CTRLRATE; + extern unsigned IOCTL_SNDCTL_SEQ_GETINCOUNT; + extern unsigned IOCTL_SNDCTL_SEQ_GETOUTCOUNT; + extern unsigned IOCTL_SNDCTL_SEQ_NRMIDIS; + extern unsigned IOCTL_SNDCTL_SEQ_NRSYNTHS; + extern unsigned IOCTL_SNDCTL_SEQ_OUTOFBAND; + extern unsigned IOCTL_SNDCTL_SEQ_PANIC; + extern unsigned IOCTL_SNDCTL_SEQ_PERCMODE; + extern unsigned IOCTL_SNDCTL_SEQ_RESET; + extern unsigned IOCTL_SNDCTL_SEQ_RESETSAMPLES; + extern unsigned IOCTL_SNDCTL_SEQ_SYNC; + extern unsigned IOCTL_SNDCTL_SEQ_TESTMIDI; + extern unsigned IOCTL_SNDCTL_SEQ_THRESHOLD; + extern unsigned IOCTL_SNDCTL_SYNTH_INFO; + extern unsigned IOCTL_SNDCTL_SYNTH_MEMAVL; + extern unsigned IOCTL_SNDCTL_TMR_CONTINUE; + extern unsigned IOCTL_SNDCTL_TMR_METRONOME; + extern unsigned IOCTL_SNDCTL_TMR_SELECT; + extern unsigned IOCTL_SNDCTL_TMR_SOURCE; + extern unsigned IOCTL_SNDCTL_TMR_START; + extern unsigned IOCTL_SNDCTL_TMR_STOP; + extern unsigned IOCTL_SNDCTL_TMR_TEMPO; + extern unsigned IOCTL_SNDCTL_TMR_TIMEBASE; + extern unsigned IOCTL_SOUND_MIXER_READ_ALTPCM; + extern unsigned IOCTL_SOUND_MIXER_READ_BASS; + extern unsigned IOCTL_SOUND_MIXER_READ_CAPS; + extern unsigned IOCTL_SOUND_MIXER_READ_CD; + extern unsigned IOCTL_SOUND_MIXER_READ_DEVMASK; + extern unsigned IOCTL_SOUND_MIXER_READ_ENHANCE; + extern unsigned IOCTL_SOUND_MIXER_READ_IGAIN; + extern unsigned IOCTL_SOUND_MIXER_READ_IMIX; + extern unsigned IOCTL_SOUND_MIXER_READ_LINE1; + extern unsigned IOCTL_SOUND_MIXER_READ_LINE2; + extern unsigned IOCTL_SOUND_MIXER_READ_LINE3; + extern unsigned IOCTL_SOUND_MIXER_READ_LINE; + extern unsigned IOCTL_SOUND_MIXER_READ_LOUD; + extern unsigned IOCTL_SOUND_MIXER_READ_MIC; + extern unsigned IOCTL_SOUND_MIXER_READ_MUTE; + extern unsigned IOCTL_SOUND_MIXER_READ_OGAIN; + extern unsigned IOCTL_SOUND_MIXER_READ_PCM; + extern unsigned IOCTL_SOUND_MIXER_READ_RECLEV; + extern unsigned IOCTL_SOUND_MIXER_READ_RECMASK; + extern unsigned IOCTL_SOUND_MIXER_READ_RECSRC; + extern unsigned IOCTL_SOUND_MIXER_READ_SPEAKER; + extern unsigned IOCTL_SOUND_MIXER_READ_STEREODEVS; + extern unsigned IOCTL_SOUND_MIXER_READ_SYNTH; + extern unsigned IOCTL_SOUND_MIXER_READ_TREBLE; + extern unsigned IOCTL_SOUND_MIXER_READ_VOLUME; + extern unsigned IOCTL_SOUND_MIXER_WRITE_ALTPCM; + extern unsigned IOCTL_SOUND_MIXER_WRITE_BASS; + extern unsigned IOCTL_SOUND_MIXER_WRITE_CD; + extern unsigned IOCTL_SOUND_MIXER_WRITE_ENHANCE; + extern unsigned IOCTL_SOUND_MIXER_WRITE_IGAIN; + extern unsigned IOCTL_SOUND_MIXER_WRITE_IMIX; + extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE1; + extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE2; + extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE3; + extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE; + extern unsigned IOCTL_SOUND_MIXER_WRITE_LOUD; + extern unsigned IOCTL_SOUND_MIXER_WRITE_MIC; + extern unsigned IOCTL_SOUND_MIXER_WRITE_MUTE; + extern unsigned IOCTL_SOUND_MIXER_WRITE_OGAIN; + extern unsigned IOCTL_SOUND_MIXER_WRITE_PCM; + extern unsigned IOCTL_SOUND_MIXER_WRITE_RECLEV; + extern unsigned IOCTL_SOUND_MIXER_WRITE_RECSRC; + extern unsigned IOCTL_SOUND_MIXER_WRITE_SPEAKER; + extern unsigned IOCTL_SOUND_MIXER_WRITE_SYNTH; + extern unsigned IOCTL_SOUND_MIXER_WRITE_TREBLE; + extern unsigned IOCTL_SOUND_MIXER_WRITE_VOLUME; + extern unsigned IOCTL_SOUND_PCM_READ_BITS; + extern unsigned IOCTL_SOUND_PCM_READ_CHANNELS; + extern unsigned IOCTL_SOUND_PCM_READ_FILTER; + extern unsigned IOCTL_SOUND_PCM_READ_RATE; + extern unsigned IOCTL_SOUND_PCM_WRITE_CHANNELS; + extern unsigned IOCTL_SOUND_PCM_WRITE_FILTER; + extern unsigned IOCTL_VT_ACTIVATE; + extern unsigned IOCTL_VT_GETMODE; + extern unsigned IOCTL_VT_OPENQRY; + extern unsigned IOCTL_VT_RELDISP; + extern unsigned IOCTL_VT_SETMODE; + extern unsigned IOCTL_VT_WAITACTIVE; + extern unsigned IOCTL_GIO_SCRNMAP; + extern unsigned IOCTL_KDDISABIO; + extern unsigned IOCTL_KDENABIO; + extern unsigned IOCTL_KDGETLED; + extern unsigned IOCTL_KDGETMODE; + extern unsigned IOCTL_KDGKBMODE; + extern unsigned IOCTL_KDGKBTYPE; + extern unsigned IOCTL_KDMKTONE; + extern unsigned IOCTL_KDSETLED; + extern unsigned IOCTL_KDSETMODE; + extern unsigned IOCTL_KDSKBMODE; + + extern const int si_SEGV_MAPERR; + extern const int si_SEGV_ACCERR; + + struct __sanitizer_cap_rights { + u64 cr_rights[2]; + }; + + typedef struct __sanitizer_cap_rights __sanitizer_cap_rights_t; + extern unsigned struct_cap_rights_sz; + + extern unsigned struct_fstab_sz; + extern unsigned struct_StringList_sz; +} // namespace __sanitizer + +#define CHECK_TYPE_SIZE(TYPE) \ + COMPILER_CHECK(sizeof(__sanitizer_##TYPE) == sizeof(TYPE)) + +#define CHECK_SIZE_AND_OFFSET(CLASS, MEMBER) \ + COMPILER_CHECK(sizeof(((__sanitizer_##CLASS *) NULL)->MEMBER) == \ + sizeof(((CLASS *) NULL)->MEMBER)); \ + COMPILER_CHECK(offsetof(__sanitizer_##CLASS, MEMBER) == \ + offsetof(CLASS, MEMBER)) + +// For sigaction, which is a function and struct at the same time, +// and thus requires explicit "struct" in sizeof() expression. +#define CHECK_STRUCT_SIZE_AND_OFFSET(CLASS, MEMBER) \ + COMPILER_CHECK(sizeof(((struct __sanitizer_##CLASS *) NULL)->MEMBER) == \ + sizeof(((struct CLASS *) NULL)->MEMBER)); \ + COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) == \ + offsetof(struct CLASS, MEMBER)) + +#define SIGACTION_SYMNAME sigaction + +#endif + +#endif // SANITIZER_FREEBSD diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc index 3c6bb2090fa5..b23b430d9e5b 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc @@ -15,9 +15,80 @@ #include "sanitizer_platform.h" #if SANITIZER_NETBSD + +#define _KMEMUSER +#define RAY_DO_SIGLEV + +// clang-format off #include <sys/param.h> #include <sys/types.h> - +#include <sys/sysctl.h> +#include <sys/disk.h> +#include <sys/disklabel.h> +#include <sys/mount.h> +#include <sys/agpio.h> +#include <sys/ataio.h> +#include <sys/audioio.h> +#include <sys/cdbr.h> +#include <sys/cdio.h> +#include <sys/chio.h> +#include <sys/clockctl.h> +#include <sys/cpuio.h> +#include <sys/dkio.h> +#include <sys/drvctlio.h> +#include <sys/dvdio.h> +#include <sys/envsys.h> +#include <sys/event.h> +#include <sys/fdio.h> +#include <sys/filio.h> +#include <sys/gpio.h> +#include <sys/ioctl.h> +#include <sys/ioctl_compat.h> +#include <sys/joystick.h> +#include <sys/ksyms.h> +#include <sys/lua.h> +#include <sys/midiio.h> +#include <sys/mtio.h> +#include <sys/power.h> +#include <sys/radioio.h> +#include <sys/rndio.h> +#include <sys/scanio.h> +#include <sys/scsiio.h> +#include <sys/sockio.h> +#include <sys/timepps.h> +#include <sys/ttycom.h> +#include <sys/verified_exec.h> +#include <sys/videoio.h> +#include <sys/wdog.h> +#include <sys/event.h> +#include <sys/filio.h> +#include <sys/ipc.h> +#include <sys/mman.h> +#include <sys/module.h> +#include <sys/mount.h> +#include <sys/mqueue.h> +#include <sys/msg.h> +#include <sys/mtio.h> +#include <sys/ptrace.h> +#include <sys/resource.h> +#include <sys/sem.h> +#include <sys/sha1.h> +#include <sys/sha2.h> +#include <sys/shm.h> +#include <sys/signal.h> +#include <sys/socket.h> +#include <sys/sockio.h> +#include <sys/soundcard.h> +#include <sys/stat.h> +#include <sys/statvfs.h> +#include <sys/time.h> +#include <sys/timeb.h> +#include <sys/times.h> +#include <sys/timespec.h> +#include <sys/timex.h> +#include <sys/types.h> +#include <sys/ucontext.h> +#include <sys/utsname.h> #include <altq/altq.h> #include <altq/altq_afmap.h> #include <altq/altq_blue.h> @@ -47,47 +118,41 @@ #include <dev/ic/icp_ioctl.h> #include <dev/ic/isp_ioctl.h> #include <dev/ic/mlxio.h> +#include <dev/ic/qemufwcfgio.h> #include <dev/ic/nvmeio.h> #include <dev/ir/irdaio.h> #include <dev/isa/isvio.h> #include <dev/isa/satlinkio.h> #include <dev/isa/wtreg.h> #include <dev/iscsi/iscsi_ioctl.h> +#include <dev/nvmm/nvmm_ioctl.h> #include <dev/ofw/openfirmio.h> #include <dev/pci/amrio.h> - #include <dev/pci/mlyreg.h> #include <dev/pci/mlyio.h> - #include <dev/pci/pciio.h> #include <dev/pci/tweio.h> #include <dev/pcmcia/if_cnwioctl.h> -#include <dirent.h> -#include <glob.h> -#include <grp.h> -#include <ifaddrs.h> -#include <limits.h> -#include <link_elf.h> -#include <net/if.h> -#include <net/if_ether.h> +#include <net/bpf.h> +#include <net/if_gre.h> #include <net/ppp_defs.h> -#include <net/route.h> -#include <netdb.h> -#include <netinet/in.h> +#include <net/if_ppp.h> +#include <net/if_pppoe.h> +#include <net/if_sppp.h> +#include <net/if_srt.h> +#include <net/if_tap.h> +#include <net/if_tun.h> +#include <net/npf.h> +#include <net/pfvar.h> +#include <net/slip.h> +#include <netbt/hci.h> #include <netinet/ip_compat.h> #include <netinet/ip_fil.h> -#include <netinet/ip_mroute.h> -#include <poll.h> -#include <pthread.h> -#include <pwd.h> -#include <semaphore.h> -#include <signal.h> -#include <stddef.h> -#include <stdio.h> -#include <sys/disk.h> -#include <sys/disklabel.h> -#include <sys/mount.h> -#define RAY_DO_SIGLEV +#include <netinet/ip_nat.h> +#include <netinet/ip_proxy.h> +#include <netinet6/in6_var.h> +#include <netinet6/nd6.h> +#include <netsmb/smb_dev.h> #include <dev/biovar.h> #include <dev/bluetooth/btdev.h> #include <dev/bluetooth/btsco.h> @@ -97,7 +162,9 @@ #include <dev/kttcpio.h> #include <dev/lockstat.h> #include <dev/md.h> +#include <net/if_ether.h> #include <dev/pcmcia/if_rayreg.h> +#include <stdio.h> #include <dev/raidframe/raidframeio.h> #include <dev/sbus/mbppio.h> #include <dev/scsipi/ses.h> @@ -115,86 +182,30 @@ #include <dev/vndvar.h> #include <dev/wscons/wsconsio.h> #include <dev/wscons/wsdisplay_usl_io.h> -#include <net/bpf.h> -#include <net/if_atm.h> -#include <net/if_gre.h> -#include <net/if_ppp.h> -#include <net/if_pppoe.h> -#include <net/if_sppp.h> -#include <net/if_srt.h> -#include <net/if_tap.h> -#include <net/if_tun.h> -#include <net/npf.h> -#include <net/pfvar.h> -#include <net/slip.h> -#include <netbt/hci.h> -#include <netinet/ip_nat.h> -#include <netinet/ip_proxy.h> -#include <netinet6/in6_var.h> -#include <netinet6/nd6.h> -#include <netnatm/natm.h> -#include <netsmb/smb_dev.h> +#include <fs/autofs/autofs_ioctl.h> +#include <dirent.h> +#include <glob.h> +#include <grp.h> +#include <ifaddrs.h> +#include <limits.h> +#include <link_elf.h> +#include <net/if.h> +#include <net/route.h> +#include <netdb.h> +#include <netinet/in.h> +#include <netinet/ip_mroute.h> +#include <netinet/sctp_uio.h> +#include <poll.h> +#include <pthread.h> +#include <pwd.h> +#include <semaphore.h> +#include <signal.h> +#include <stddef.h> +#include <md2.h> +#include <md4.h> +#include <md5.h> +#include <rmd160.h> #include <soundcard.h> -#include <sys/agpio.h> -#include <sys/ataio.h> -#include <sys/audioio.h> -#include <sys/cdio.h> -#include <sys/chio.h> -#include <sys/clockctl.h> -#include <sys/cpuio.h> -#include <sys/dkio.h> -#include <sys/drvctlio.h> -#include <sys/dvdio.h> -#include <sys/envsys.h> -#include <sys/event.h> -#include <sys/fdio.h> -#include <sys/filio.h> -#include <sys/gpio.h> -#include <sys/ioctl.h> -#include <sys/ioctl_compat.h> -#include <sys/joystick.h> -#include <sys/ksyms.h> -#include <sys/lua.h> -#include <sys/midiio.h> -#include <sys/mtio.h> -#include <sys/power.h> -#include <sys/radioio.h> -#include <sys/rndio.h> -#include <sys/scanio.h> -#include <sys/scsiio.h> -#include <sys/sockio.h> -#include <sys/timepps.h> -#include <sys/ttycom.h> -#include <sys/verified_exec.h> -#include <sys/videoio.h> -#include <sys/wdog.h> -//#include <xen/xenio.h> -#include <sys/event.h> -#include <sys/filio.h> -#include <sys/ipc.h> -#include <sys/mman.h> -#include <sys/mount.h> -#include <sys/mqueue.h> -#include <sys/msg.h> -#include <sys/mtio.h> -#include <sys/ptrace.h> -#include <sys/resource.h> -#include <sys/sem.h> -#include <sys/shm.h> -#include <sys/signal.h> -#include <sys/socket.h> -#include <sys/sockio.h> -#include <sys/soundcard.h> -#include <sys/stat.h> -#include <sys/statvfs.h> -#include <sys/time.h> -#include <sys/timeb.h> -#include <sys/times.h> -#include <sys/timespec.h> -#include <sys/timex.h> -#include <sys/types.h> -#include <sys/ucontext.h> -#include <sys/utsname.h> #include <term.h> #include <termios.h> #include <time.h> @@ -202,8 +213,15 @@ #include <utime.h> #include <utmp.h> #include <utmpx.h> +#include <vis.h> #include <wchar.h> #include <wordexp.h> +#include <ttyent.h> +#include <fts.h> +#include <regex.h> +#include <fstab.h> +#include <stringlist.h> +// clang-format on // Include these after system headers to avoid name clashes and ambiguities. #include "sanitizer_internal_defs.h" @@ -238,6 +256,11 @@ unsigned struct_rlimit_sz = sizeof(struct rlimit); unsigned struct_timespec_sz = sizeof(struct timespec); unsigned struct_sembuf_sz = sizeof(struct sembuf); unsigned struct_kevent_sz = sizeof(struct kevent); +unsigned struct_FTS_sz = sizeof(FTS); +unsigned struct_FTSENT_sz = sizeof(FTSENT); +unsigned struct_regex_sz = sizeof(regex_t); +unsigned struct_regmatch_sz = sizeof(regmatch_t); +unsigned struct_fstab_sz = sizeof(struct fstab); unsigned struct_utimbuf_sz = sizeof(struct utimbuf); unsigned struct_itimerspec_sz = sizeof(struct itimerspec); unsigned struct_timex_sz = sizeof(struct timex); @@ -251,6 +274,8 @@ const uptr sig_dfl = (uptr)SIG_DFL; const uptr sig_err = (uptr)SIG_ERR; const uptr sa_siginfo = (uptr)SA_SIGINFO; +const unsigned long __sanitizer_bufsiz = BUFSIZ; + int ptrace_pt_io = PT_IO; int ptrace_pt_lwpinfo = PT_LWPINFO; int ptrace_pt_set_event_mask = PT_SET_EVENT_MASK; @@ -339,6 +364,14 @@ unsigned path_max = PATH_MAX; int struct_ttyent_sz = sizeof(struct ttyent); +struct __sanitizer_nvlist_ref_t { + void *buf; + uptr len; + int flags; +}; + +typedef __sanitizer_nvlist_ref_t nvlist_ref_t; + // ioctl arguments unsigned struct_altqreq_sz = sizeof(altqreq); unsigned struct_amr_user_ioctl_sz = sizeof(amr_user_ioctl); @@ -350,7 +383,6 @@ unsigned struct_atabusiodetach_args_sz = sizeof(atabusiodetach_args); unsigned struct_atabusioscan_args_sz = sizeof(atabusioscan_args); unsigned struct_ath_diag_sz = sizeof(ath_diag); unsigned struct_atm_flowmap_sz = sizeof(atm_flowmap); -unsigned struct_atm_pseudoioctl_sz = sizeof(atm_pseudoioctl); unsigned struct_audio_buf_info_sz = sizeof(audio_buf_info); unsigned struct_audio_device_sz = sizeof(audio_device); unsigned struct_audio_encoding_sz = sizeof(audio_encoding); @@ -596,7 +628,6 @@ unsigned struct_priq_delete_filter_sz = sizeof(priq_delete_filter); unsigned struct_priq_interface_sz = sizeof(priq_interface); unsigned struct_priq_modify_class_sz = sizeof(priq_modify_class); unsigned struct_ptmget_sz = sizeof(ptmget); -unsigned struct_pvctxreq_sz = sizeof(pvctxreq); unsigned struct_radio_info_sz = sizeof(radio_info); unsigned struct_red_conf_sz = sizeof(red_conf); unsigned struct_red_interface_sz = sizeof(red_interface); @@ -656,6 +687,9 @@ unsigned struct_usb_config_desc_sz = sizeof(usb_config_desc); unsigned struct_usb_ctl_report_desc_sz = sizeof(usb_ctl_report_desc); unsigned struct_usb_ctl_report_sz = sizeof(usb_ctl_report); unsigned struct_usb_ctl_request_sz = sizeof(usb_ctl_request); +unsigned struct_autofs_daemon_request_sz = sizeof(autofs_daemon_request); +unsigned struct_autofs_daemon_done_sz = sizeof(autofs_daemon_done); +unsigned struct_sctp_connectx_addrs_sz = sizeof(sctp_connectx_addrs); unsigned struct_usb_device_info_old_sz = sizeof(usb_device_info_old); unsigned struct_usb_device_info_sz = sizeof(usb_device_info); unsigned struct_usb_device_stats_sz = sizeof(usb_device_stats); @@ -797,6 +831,8 @@ unsigned struct_RF_SparetWait_sz = sizeof(RF_SparetWait_t); unsigned struct_RF_ComponentLabel_sz = sizeof(RF_ComponentLabel_t); unsigned struct_RF_SingleComponent_sz = sizeof(RF_SingleComponent_t); unsigned struct_RF_ProgressInfo_sz = sizeof(RF_ProgressInfo_t); +unsigned struct_nvlist_ref_sz = sizeof(struct __sanitizer_nvlist_ref_t); +unsigned struct_StringList_sz = sizeof(StringList); const unsigned IOCTL_NOT_PRESENT = 0; @@ -1061,6 +1097,7 @@ unsigned IOCTL_MLX_REBUILDSTAT = MLX_REBUILDSTAT; unsigned IOCTL_MLX_GET_SYSDRIVE = MLX_GET_SYSDRIVE; unsigned IOCTL_MLX_GET_CINFO = MLX_GET_CINFO; unsigned IOCTL_NVME_PASSTHROUGH_CMD = NVME_PASSTHROUGH_CMD; +unsigned IOCTL_FWCFGIO_SET_INDEX = FWCFGIO_SET_INDEX; unsigned IOCTL_IRDA_RESET_PARAMS = IRDA_RESET_PARAMS; unsigned IOCTL_IRDA_SET_PARAMS = IRDA_SET_PARAMS; unsigned IOCTL_IRDA_GET_SPEEDMASK = IRDA_GET_SPEEDMASK; @@ -1390,6 +1427,24 @@ unsigned IOCTL_SPKRTONE = SPKRTONE; unsigned IOCTL_SPKRTUNE = SPKRTUNE; unsigned IOCTL_SPKRGETVOL = SPKRGETVOL; unsigned IOCTL_SPKRSETVOL = SPKRSETVOL; +#if 0 /* interfaces are WIP */ +unsigned IOCTL_NVMM_IOC_CAPABILITY = NVMM_IOC_CAPABILITY; +unsigned IOCTL_NVMM_IOC_MACHINE_CREATE = NVMM_IOC_MACHINE_CREATE; +unsigned IOCTL_NVMM_IOC_MACHINE_DESTROY = NVMM_IOC_MACHINE_DESTROY; +unsigned IOCTL_NVMM_IOC_MACHINE_CONFIGURE = NVMM_IOC_MACHINE_CONFIGURE; +unsigned IOCTL_NVMM_IOC_VCPU_CREATE = NVMM_IOC_VCPU_CREATE; +unsigned IOCTL_NVMM_IOC_VCPU_DESTROY = NVMM_IOC_VCPU_DESTROY; +unsigned IOCTL_NVMM_IOC_VCPU_SETSTATE = NVMM_IOC_VCPU_SETSTATE; +unsigned IOCTL_NVMM_IOC_VCPU_GETSTATE = NVMM_IOC_VCPU_GETSTATE; +unsigned IOCTL_NVMM_IOC_VCPU_INJECT = NVMM_IOC_VCPU_INJECT; +unsigned IOCTL_NVMM_IOC_VCPU_RUN = NVMM_IOC_VCPU_RUN; +unsigned IOCTL_NVMM_IOC_GPA_MAP = NVMM_IOC_GPA_MAP; +unsigned IOCTL_NVMM_IOC_GPA_UNMAP = NVMM_IOC_GPA_UNMAP; +unsigned IOCTL_NVMM_IOC_HVA_MAP = NVMM_IOC_HVA_MAP; +unsigned IOCTL_NVMM_IOC_HVA_UNMAP = NVMM_IOC_HVA_UNMAP; +#endif +unsigned IOCTL_AUTOFSREQUEST = AUTOFSREQUEST; +unsigned IOCTL_AUTOFSDONE = AUTOFSDONE; unsigned IOCTL_BIOCGBLEN = BIOCGBLEN; unsigned IOCTL_BIOCSBLEN = BIOCSBLEN; unsigned IOCTL_BIOCSETF = BIOCSETF; @@ -1408,19 +1463,12 @@ unsigned IOCTL_BIOCGHDRCMPLT = BIOCGHDRCMPLT; unsigned IOCTL_BIOCSHDRCMPLT = BIOCSHDRCMPLT; unsigned IOCTL_BIOCSDLT = BIOCSDLT; unsigned IOCTL_BIOCGDLTLIST = BIOCGDLTLIST; -unsigned IOCTL_BIOCGSEESENT = BIOCGSEESENT; -unsigned IOCTL_BIOCSSEESENT = BIOCSSEESENT; +unsigned IOCTL_BIOCGDIRECTION = BIOCGDIRECTION; +unsigned IOCTL_BIOCSDIRECTION = BIOCSDIRECTION; unsigned IOCTL_BIOCSRTIMEOUT = BIOCSRTIMEOUT; unsigned IOCTL_BIOCGRTIMEOUT = BIOCGRTIMEOUT; unsigned IOCTL_BIOCGFEEDBACK = BIOCGFEEDBACK; unsigned IOCTL_BIOCSFEEDBACK = BIOCSFEEDBACK; -unsigned IOCTL_SIOCRAWATM = SIOCRAWATM; -unsigned IOCTL_SIOCATMENA = SIOCATMENA; -unsigned IOCTL_SIOCATMDIS = SIOCATMDIS; -unsigned IOCTL_SIOCSPVCTX = SIOCSPVCTX; -unsigned IOCTL_SIOCGPVCTX = SIOCGPVCTX; -unsigned IOCTL_SIOCSPVCSIF = SIOCSPVCSIF; -unsigned IOCTL_SIOCGPVCSIF = SIOCGPVCSIF; unsigned IOCTL_GRESADDRS = GRESADDRS; unsigned IOCTL_GRESADDRD = GRESADDRD; unsigned IOCTL_GREGADDRS = GREGADDRS; @@ -1575,6 +1623,8 @@ unsigned IOCTL_SIOCRMNAT = SIOCRMNAT; unsigned IOCTL_SIOCGNATS = SIOCGNATS; unsigned IOCTL_SIOCGNATL = SIOCGNATL; unsigned IOCTL_SIOCPURGENAT = SIOCPURGENAT; +unsigned IOCTL_SIOCCONNECTX = SIOCCONNECTX; +unsigned IOCTL_SIOCCONNECTXDEL = SIOCCONNECTXDEL; unsigned IOCTL_SIOCSIFINFO_FLAGS = SIOCSIFINFO_FLAGS; unsigned IOCTL_SIOCAADDRCTL_POLICY = SIOCAADDRCTL_POLICY; unsigned IOCTL_SIOCDADDRCTL_POLICY = SIOCDADDRCTL_POLICY; @@ -1719,6 +1769,8 @@ unsigned IOCTL_FDIOCGETFORMAT = FDIOCGETFORMAT; unsigned IOCTL_FDIOCFORMAT_TRACK = FDIOCFORMAT_TRACK; unsigned IOCTL_FIOCLEX = FIOCLEX; unsigned IOCTL_FIONCLEX = FIONCLEX; +unsigned IOCTL_FIOSEEKDATA = FIOSEEKDATA; +unsigned IOCTL_FIOSEEKHOLE = FIOSEEKHOLE; unsigned IOCTL_FIONREAD = FIONREAD; unsigned IOCTL_FIONBIO = FIONBIO; unsigned IOCTL_FIOASYNC = FIOASYNC; @@ -1804,8 +1856,6 @@ unsigned IOCTL_MTIOCSLOCATE = MTIOCSLOCATE; unsigned IOCTL_MTIOCHLOCATE = MTIOCHLOCATE; unsigned IOCTL_POWER_EVENT_RECVDICT = POWER_EVENT_RECVDICT; unsigned IOCTL_POWER_IOC_GET_TYPE = POWER_IOC_GET_TYPE; -unsigned IOCTL_POWER_IOC_GET_TYPE_WITH_LOSSAGE = - POWER_IOC_GET_TYPE_WITH_LOSSAGE; unsigned IOCTL_RIOCGINFO = RIOCGINFO; unsigned IOCTL_RIOCSINFO = RIOCSINFO; unsigned IOCTL_RIOCSSRCH = RIOCSSRCH; @@ -1840,6 +1890,7 @@ unsigned IOCTL_SIOCGLOWAT = SIOCGLOWAT; unsigned IOCTL_SIOCATMARK = SIOCATMARK; unsigned IOCTL_SIOCSPGRP = SIOCSPGRP; unsigned IOCTL_SIOCGPGRP = SIOCGPGRP; +unsigned IOCTL_SIOCPEELOFF = SIOCPEELOFF; unsigned IOCTL_SIOCADDRT = SIOCADDRT; unsigned IOCTL_SIOCDELRT = SIOCDELRT; unsigned IOCTL_SIOCSIFADDR = SIOCSIFADDR; @@ -1897,6 +1948,10 @@ unsigned IOCTL_SIOCGLINKSTR = SIOCGLINKSTR; unsigned IOCTL_SIOCSLINKSTR = SIOCSLINKSTR; unsigned IOCTL_SIOCGETHERCAP = SIOCGETHERCAP; unsigned IOCTL_SIOCGIFINDEX = SIOCGIFINDEX; +unsigned IOCTL_SIOCSETHERCAP = SIOCSETHERCAP; +unsigned IOCTL_SIOCGUMBINFO = SIOCGUMBINFO; +unsigned IOCTL_SIOCSUMBPARAM = SIOCSUMBPARAM; +unsigned IOCTL_SIOCGUMBPARAM = SIOCGUMBPARAM; unsigned IOCTL_SIOCSETPFSYNC = SIOCSETPFSYNC; unsigned IOCTL_SIOCGETPFSYNC = SIOCGETPFSYNC; unsigned IOCTL_PPS_IOC_CREATE = PPS_IOC_CREATE; @@ -2063,6 +2118,44 @@ unsigned IOCTL_SNDCTL_DSP_SILENCE = SNDCTL_DSP_SILENCE; const int si_SEGV_MAPERR = SEGV_MAPERR; const int si_SEGV_ACCERR = SEGV_ACCERR; + +const int modctl_load = MODCTL_LOAD; +const int modctl_unload = MODCTL_UNLOAD; +const int modctl_stat = MODCTL_STAT; +const int modctl_exists = MODCTL_EXISTS; + +const unsigned SHA1_CTX_sz = sizeof(SHA1_CTX); +const unsigned SHA1_return_length = SHA1_DIGEST_STRING_LENGTH; + +const unsigned MD4_CTX_sz = sizeof(MD4_CTX); +const unsigned MD4_return_length = MD4_DIGEST_STRING_LENGTH; + +const unsigned RMD160_CTX_sz = sizeof(RMD160_CTX); +const unsigned RMD160_return_length = RMD160_DIGEST_STRING_LENGTH; + +const unsigned MD5_CTX_sz = sizeof(MD5_CTX); +const unsigned MD5_return_length = MD5_DIGEST_STRING_LENGTH; + +const unsigned fpos_t_sz = sizeof(fpos_t); + +const unsigned MD2_CTX_sz = sizeof(MD2_CTX); +const unsigned MD2_return_length = MD2_DIGEST_STRING_LENGTH; + +#define SHA2_CONST(LEN) \ + const unsigned SHA##LEN##_CTX_sz = sizeof(SHA##LEN##_CTX); \ + const unsigned SHA##LEN##_return_length = SHA##LEN##_DIGEST_STRING_LENGTH; \ + const unsigned SHA##LEN##_block_length = SHA##LEN##_BLOCK_LENGTH; \ + const unsigned SHA##LEN##_digest_length = SHA##LEN##_DIGEST_LENGTH + +SHA2_CONST(224); +SHA2_CONST(256); +SHA2_CONST(384); +SHA2_CONST(512); + +#undef SHA2_CONST + +const int unvis_valid = UNVIS_VALID; +const int unvis_validpush = UNVIS_VALIDPUSH; } // namespace __sanitizer using namespace __sanitizer; @@ -2153,6 +2246,29 @@ CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordc); CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordv); CHECK_SIZE_AND_OFFSET(wordexp_t, we_offs); +COMPILER_CHECK(sizeof(__sanitizer_FILE) <= sizeof(FILE)); +CHECK_SIZE_AND_OFFSET(FILE, _p); +CHECK_SIZE_AND_OFFSET(FILE, _r); +CHECK_SIZE_AND_OFFSET(FILE, _w); +CHECK_SIZE_AND_OFFSET(FILE, _flags); +CHECK_SIZE_AND_OFFSET(FILE, _file); +CHECK_SIZE_AND_OFFSET(FILE, _bf); +CHECK_SIZE_AND_OFFSET(FILE, _lbfsize); +CHECK_SIZE_AND_OFFSET(FILE, _cookie); +CHECK_SIZE_AND_OFFSET(FILE, _close); +CHECK_SIZE_AND_OFFSET(FILE, _read); +CHECK_SIZE_AND_OFFSET(FILE, _seek); +CHECK_SIZE_AND_OFFSET(FILE, _write); +CHECK_SIZE_AND_OFFSET(FILE, _ext); +CHECK_SIZE_AND_OFFSET(FILE, _up); +CHECK_SIZE_AND_OFFSET(FILE, _ur); +CHECK_SIZE_AND_OFFSET(FILE, _ubuf); +CHECK_SIZE_AND_OFFSET(FILE, _nbuf); +CHECK_SIZE_AND_OFFSET(FILE, _flush); +CHECK_SIZE_AND_OFFSET(FILE, _lb_unused); +CHECK_SIZE_AND_OFFSET(FILE, _blksize); +CHECK_SIZE_AND_OFFSET(FILE, _offset); + CHECK_TYPE_SIZE(tm); CHECK_SIZE_AND_OFFSET(tm, tm_sec); CHECK_SIZE_AND_OFFSET(tm, tm_min); @@ -2224,4 +2340,10 @@ CHECK_SIZE_AND_OFFSET(group, gr_passwd); CHECK_SIZE_AND_OFFSET(group, gr_gid); CHECK_SIZE_AND_OFFSET(group, gr_mem); +CHECK_TYPE_SIZE(modctl_load_t); +CHECK_SIZE_AND_OFFSET(modctl_load_t, ml_filename); +CHECK_SIZE_AND_OFFSET(modctl_load_t, ml_flags); +CHECK_SIZE_AND_OFFSET(modctl_load_t, ml_props); +CHECK_SIZE_AND_OFFSET(modctl_load_t, ml_propslen); + #endif // SANITIZER_NETBSD diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h index 1718e3b1e8ea..0c0c8a837b8b 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h @@ -25,10 +25,10 @@ #if defined(__x86_64__) #define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \ - _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, 312) + _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, 264) #elif defined(__i386__) #define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \ - _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, 164) + _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, 136) #endif namespace __sanitizer { @@ -60,6 +60,29 @@ extern unsigned struct_timespec_sz; extern unsigned struct_sembuf_sz; extern unsigned struct_kevent_sz; +extern unsigned struct_FTS_sz; +extern unsigned struct_FTSENT_sz; + +extern unsigned struct_regex_sz; +extern unsigned struct_regmatch_sz; + +extern unsigned struct_fstab_sz; + +struct __sanitizer_regmatch { + OFF_T rm_so; + OFF_T rm_eo; +}; + +typedef struct __sanitizer_modctl_load { + const char *ml_filename; + int ml_flags; + const char *ml_props; + uptr ml_propslen; +} __sanitizer_modctl_load_t; +extern const int modctl_load; +extern const int modctl_unload; +extern const int modctl_stat; +extern const int modctl_exists; union __sanitizer_sigval { int sival_int; @@ -425,8 +448,36 @@ struct __sanitizer_wordexp_t { uptr we_nbytes; }; -typedef char __sanitizer_FILE; -#define SANITIZER_HAS_STRUCT_FILE 0 +struct __sanitizer_FILE { + unsigned char *_p; + int _r; + int _w; + unsigned short _flags; + short _file; + struct { + unsigned char *_base; + int _size; + } _bf; + int _lbfsize; + void *_cookie; + int (*_close)(void *ptr); + u64 (*_read)(void *, void *, uptr); + u64 (*_seek)(void *, u64, int); + uptr (*_write)(void *, const void *, uptr); + struct { + unsigned char *_base; + int _size; + } _ext; + unsigned char *_up; + int _ur; + unsigned char _ubuf[3]; + unsigned char _nbuf[1]; + int (*_flush)(void *ptr); + char _lb_unused[sizeof(uptr)]; + int _blksize; + u64 _offset; +}; +#define SANITIZER_HAS_STRUCT_FILE 1 extern int shmctl_ipc_stat; @@ -460,6 +511,8 @@ struct __sanitizer_ttyent { char *ty_class; }; +extern const unsigned long __sanitizer_bufsiz; + #define IOC_NRBITS 8 #define IOC_TYPEBITS 8 #define IOC_SIZEBITS 14 @@ -496,7 +549,6 @@ extern unsigned struct_atabusiodetach_args_sz; extern unsigned struct_atabusioscan_args_sz; extern unsigned struct_ath_diag_sz; extern unsigned struct_atm_flowmap_sz; -extern unsigned struct_atm_pseudoioctl_sz; extern unsigned struct_audio_buf_info_sz; extern unsigned struct_audio_device_sz; extern unsigned struct_audio_encoding_sz; @@ -799,6 +851,9 @@ extern unsigned struct_usb_config_desc_sz; extern unsigned struct_usb_ctl_report_desc_sz; extern unsigned struct_usb_ctl_report_sz; extern unsigned struct_usb_ctl_request_sz; +extern unsigned struct_autofs_daemon_request_sz; +extern unsigned struct_autofs_daemon_done_sz; +extern unsigned struct_sctp_connectx_addrs_sz; extern unsigned struct_usb_device_info_old_sz; extern unsigned struct_usb_device_info_sz; extern unsigned struct_usb_device_stats_sz; @@ -933,6 +988,8 @@ extern unsigned struct_RF_SparetWait_sz; extern unsigned struct_RF_ComponentLabel_sz; extern unsigned struct_RF_SingleComponent_sz; extern unsigned struct_RF_ProgressInfo_sz; +extern unsigned struct_nvlist_ref_sz; +extern unsigned struct_StringList_sz; // A special value to mark ioctls that are not present on the target platform, @@ -1201,6 +1258,7 @@ extern unsigned IOCTL_MLX_REBUILDSTAT; extern unsigned IOCTL_MLX_GET_SYSDRIVE; extern unsigned IOCTL_MLX_GET_CINFO; extern unsigned IOCTL_NVME_PASSTHROUGH_CMD; +extern unsigned IOCTL_FWCFGIO_SET_INDEX; extern unsigned IOCTL_IRDA_RESET_PARAMS; extern unsigned IOCTL_IRDA_SET_PARAMS; extern unsigned IOCTL_IRDA_GET_SPEEDMASK; @@ -1522,6 +1580,24 @@ extern unsigned IOCTL_SPKRTONE; extern unsigned IOCTL_SPKRTUNE; extern unsigned IOCTL_SPKRGETVOL; extern unsigned IOCTL_SPKRSETVOL; +#if 0 /* interfaces are WIP */ +extern unsigned IOCTL_NVMM_IOC_CAPABILITY; +extern unsigned IOCTL_NVMM_IOC_MACHINE_CREATE; +extern unsigned IOCTL_NVMM_IOC_MACHINE_DESTROY; +extern unsigned IOCTL_NVMM_IOC_MACHINE_CONFIGURE; +extern unsigned IOCTL_NVMM_IOC_VCPU_CREATE; +extern unsigned IOCTL_NVMM_IOC_VCPU_DESTROY; +extern unsigned IOCTL_NVMM_IOC_VCPU_SETSTATE; +extern unsigned IOCTL_NVMM_IOC_VCPU_GETSTATE; +extern unsigned IOCTL_NVMM_IOC_VCPU_INJECT; +extern unsigned IOCTL_NVMM_IOC_VCPU_RUN; +extern unsigned IOCTL_NVMM_IOC_GPA_MAP; +extern unsigned IOCTL_NVMM_IOC_GPA_UNMAP; +extern unsigned IOCTL_NVMM_IOC_HVA_MAP; +extern unsigned IOCTL_NVMM_IOC_HVA_UNMAP; +#endif +extern unsigned IOCTL_AUTOFSREQUEST; +extern unsigned IOCTL_AUTOFSDONE; extern unsigned IOCTL_BIOCGBLEN; extern unsigned IOCTL_BIOCSBLEN; extern unsigned IOCTL_BIOCSETF; @@ -1540,19 +1616,12 @@ extern unsigned IOCTL_BIOCGHDRCMPLT; extern unsigned IOCTL_BIOCSHDRCMPLT; extern unsigned IOCTL_BIOCSDLT; extern unsigned IOCTL_BIOCGDLTLIST; -extern unsigned IOCTL_BIOCGSEESENT; -extern unsigned IOCTL_BIOCSSEESENT; +extern unsigned IOCTL_BIOCGDIRECTION; +extern unsigned IOCTL_BIOCSDIRECTION; extern unsigned IOCTL_BIOCSRTIMEOUT; extern unsigned IOCTL_BIOCGRTIMEOUT; extern unsigned IOCTL_BIOCGFEEDBACK; extern unsigned IOCTL_BIOCSFEEDBACK; -extern unsigned IOCTL_SIOCRAWATM; -extern unsigned IOCTL_SIOCATMENA; -extern unsigned IOCTL_SIOCATMDIS; -extern unsigned IOCTL_SIOCSPVCTX; -extern unsigned IOCTL_SIOCGPVCTX; -extern unsigned IOCTL_SIOCSPVCSIF; -extern unsigned IOCTL_SIOCGPVCSIF; extern unsigned IOCTL_GRESADDRS; extern unsigned IOCTL_GRESADDRD; extern unsigned IOCTL_GREGADDRS; @@ -1707,6 +1776,8 @@ extern unsigned IOCTL_SIOCRMNAT; extern unsigned IOCTL_SIOCGNATS; extern unsigned IOCTL_SIOCGNATL; extern unsigned IOCTL_SIOCPURGENAT; +extern unsigned IOCTL_SIOCCONNECTX; +extern unsigned IOCTL_SIOCCONNECTXDEL; extern unsigned IOCTL_SIOCSIFINFO_FLAGS; extern unsigned IOCTL_SIOCAADDRCTL_POLICY; extern unsigned IOCTL_SIOCDADDRCTL_POLICY; @@ -1851,6 +1922,8 @@ extern unsigned IOCTL_FDIOCGETFORMAT; extern unsigned IOCTL_FDIOCFORMAT_TRACK; extern unsigned IOCTL_FIOCLEX; extern unsigned IOCTL_FIONCLEX; +extern unsigned IOCTL_FIOSEEKDATA; +extern unsigned IOCTL_FIOSEEKHOLE; extern unsigned IOCTL_FIONREAD; extern unsigned IOCTL_FIONBIO; extern unsigned IOCTL_FIOASYNC; @@ -1936,7 +2009,6 @@ extern unsigned IOCTL_MTIOCSLOCATE; extern unsigned IOCTL_MTIOCHLOCATE; extern unsigned IOCTL_POWER_EVENT_RECVDICT; extern unsigned IOCTL_POWER_IOC_GET_TYPE; -extern unsigned IOCTL_POWER_IOC_GET_TYPE_WITH_LOSSAGE; extern unsigned IOCTL_RIOCGINFO; extern unsigned IOCTL_RIOCSINFO; extern unsigned IOCTL_RIOCSSRCH; @@ -1971,6 +2043,7 @@ extern unsigned IOCTL_SIOCGLOWAT; extern unsigned IOCTL_SIOCATMARK; extern unsigned IOCTL_SIOCSPGRP; extern unsigned IOCTL_SIOCGPGRP; +extern unsigned IOCTL_SIOCPEELOFF; extern unsigned IOCTL_SIOCADDRT; extern unsigned IOCTL_SIOCDELRT; extern unsigned IOCTL_SIOCSIFADDR; @@ -2028,6 +2101,10 @@ extern unsigned IOCTL_SIOCGLINKSTR; extern unsigned IOCTL_SIOCSLINKSTR; extern unsigned IOCTL_SIOCGETHERCAP; extern unsigned IOCTL_SIOCGIFINDEX; +extern unsigned IOCTL_SIOCSETHERCAP; +extern unsigned IOCTL_SIOCGUMBINFO; +extern unsigned IOCTL_SIOCSUMBPARAM; +extern unsigned IOCTL_SIOCGUMBPARAM; extern unsigned IOCTL_SIOCSETPFSYNC; extern unsigned IOCTL_SIOCGETPFSYNC; extern unsigned IOCTL_PPS_IOC_CREATE; @@ -2194,6 +2271,74 @@ extern unsigned IOCTL_SNDCTL_DSP_SILENCE; extern const int si_SEGV_MAPERR; extern const int si_SEGV_ACCERR; + +extern const unsigned SHA1_CTX_sz; +extern const unsigned SHA1_return_length; + +extern const unsigned MD4_CTX_sz; +extern const unsigned MD4_return_length; + +extern const unsigned RMD160_CTX_sz; +extern const unsigned RMD160_return_length; + +extern const unsigned MD5_CTX_sz; +extern const unsigned MD5_return_length; + +extern const unsigned fpos_t_sz; + +extern const unsigned MD2_CTX_sz; +extern const unsigned MD2_return_length; + +#define SHA2_EXTERN(LEN) \ + extern const unsigned SHA##LEN##_CTX_sz; \ + extern const unsigned SHA##LEN##_return_length; \ + extern const unsigned SHA##LEN##_block_length; \ + extern const unsigned SHA##LEN##_digest_length + +SHA2_EXTERN(224); +SHA2_EXTERN(256); +SHA2_EXTERN(384); +SHA2_EXTERN(512); + +#undef SHA2_EXTERN + +extern const int unvis_valid; +extern const int unvis_validpush; + +struct __sanitizer_cdbr { + void (*unmap)(void *, void *, uptr); + void *cookie; + u8 *mmap_base; + uptr mmap_size; + + u8 *hash_base; + u8 *offset_base; + u8 *data_base; + + u32 data_size; + u32 entries; + u32 entries_index; + u32 seed; + + u8 offset_size; + u8 index_size; + + u32 entries_m; + u32 entries_index_m; + u8 entries_s1, entries_s2; + u8 entries_index_s1, entries_index_s2; +}; + +struct __sanitizer_cdbw { + uptr data_counter; + uptr data_allocated; + uptr data_size; + uptr *data_len; + void **data_ptr; + uptr hash_size; + void *hash; + uptr key_counter; +}; } // namespace __sanitizer #define CHECK_TYPE_SIZE(TYPE) \ diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc index 54da635d71f0..ecc69bcea79b 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc @@ -14,7 +14,7 @@ #include "sanitizer_platform.h" -#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_MAC +#if SANITIZER_LINUX || SANITIZER_MAC // Tests in this file assume that off_t-dependent data structures match the // libc ABI. For example, struct dirent here is what readdir() function (as // exported from libc) returns, and not the user-facing "dirent", which @@ -45,7 +45,8 @@ #include <termios.h> #include <time.h> #include <wchar.h> -#if !SANITIZER_MAC && !SANITIZER_FREEBSD +#include <regex.h> +#if !SANITIZER_MAC #include <utmp.h> #endif @@ -54,6 +55,7 @@ #endif #if !SANITIZER_ANDROID +#include <fstab.h> #include <sys/mount.h> #include <sys/timeb.h> #include <utmpx.h> @@ -78,43 +80,11 @@ #include <net/if_arp.h> #endif -#if SANITIZER_FREEBSD -# include <sys/mount.h> -# include <sys/sockio.h> -# include <sys/socket.h> -# include <sys/filio.h> -# include <sys/signal.h> -# include <sys/timespec.h> -# include <sys/timex.h> -# include <sys/mqueue.h> -# include <sys/msg.h> -# include <sys/ipc.h> -# include <sys/msg.h> -# include <sys/statvfs.h> -# include <sys/soundcard.h> -# include <sys/mtio.h> -# include <sys/consio.h> -# include <sys/kbio.h> -# include <sys/link_elf.h> -# include <netinet/ip_mroute.h> -# include <netinet/in.h> -# include <net/ethernet.h> -# include <net/ppp_defs.h> -# include <glob.h> -# include <term.h> - -#define _KERNEL // to declare 'shminfo' structure -# include <sys/shm.h> -#undef _KERNEL - -#undef INLINE // to avoid clashes with sanitizers' definitions -#endif - -#if SANITIZER_FREEBSD || SANITIZER_IOS +#if SANITIZER_IOS #undef IOC_DIRMASK #endif -#if SANITIZER_LINUX || SANITIZER_FREEBSD +#if SANITIZER_LINUX # include <utime.h> # include <sys/ptrace.h> # if defined(__mips64) || defined(__aarch64__) || defined(__arm__) @@ -146,8 +116,6 @@ typedef struct user_fpregs elf_fpregset_t; #include <netrom/netrom.h> #if HAVE_RPC_XDR_H # include <rpc/xdr.h> -#elif HAVE_TIRPC_RPC_XDR_H -# include <tirpc/rpc/xdr.h> #endif #include <scsi/scsi.h> #include <sys/mtio.h> @@ -198,9 +166,9 @@ typedef struct user_fpregs elf_fpregset_t; namespace __sanitizer { unsigned struct_utsname_sz = sizeof(struct utsname); unsigned struct_stat_sz = sizeof(struct stat); -#if !SANITIZER_IOS && !SANITIZER_FREEBSD +#if !SANITIZER_IOS unsigned struct_stat64_sz = sizeof(struct stat64); -#endif // !SANITIZER_IOS && !SANITIZER_FREEBSD +#endif // !SANITIZER_IOS unsigned struct_rusage_sz = sizeof(struct rusage); unsigned struct_tm_sz = sizeof(struct tm); unsigned struct_passwd_sz = sizeof(struct passwd); @@ -221,13 +189,15 @@ namespace __sanitizer { unsigned struct_tms_sz = sizeof(struct tms); unsigned struct_sigevent_sz = sizeof(struct sigevent); unsigned struct_sched_param_sz = sizeof(struct sched_param); - + unsigned struct_regex_sz = sizeof(regex_t); + unsigned struct_regmatch_sz = sizeof(regmatch_t); #if SANITIZER_MAC && !SANITIZER_IOS unsigned struct_statfs64_sz = sizeof(struct statfs64); #endif // SANITIZER_MAC && !SANITIZER_IOS #if !SANITIZER_ANDROID + unsigned struct_fstab_sz = sizeof(struct fstab); unsigned struct_statfs_sz = sizeof(struct statfs); unsigned struct_sockaddr_sz = sizeof(struct sockaddr); unsigned ucontext_t_sz = sizeof(ucontext_t); @@ -244,12 +214,12 @@ namespace __sanitizer { unsigned struct_oldold_utsname_sz = sizeof(struct oldold_utsname); #endif // SANITIZER_LINUX -#if SANITIZER_LINUX || SANITIZER_FREEBSD +#if SANITIZER_LINUX unsigned struct_rlimit_sz = sizeof(struct rlimit); unsigned struct_timespec_sz = sizeof(struct timespec); unsigned struct_utimbuf_sz = sizeof(struct utimbuf); unsigned struct_itimerspec_sz = sizeof(struct itimerspec); -#endif // SANITIZER_LINUX || SANITIZER_FREEBSD +#endif // SANITIZER_LINUX #if SANITIZER_LINUX && !SANITIZER_ANDROID // Use pre-computed size of struct ustat to avoid <sys/ustat.h> which @@ -269,12 +239,12 @@ namespace __sanitizer { unsigned struct_statvfs64_sz = sizeof(struct statvfs64); #endif // SANITIZER_LINUX && !SANITIZER_ANDROID -#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#if SANITIZER_LINUX && !SANITIZER_ANDROID unsigned struct_timex_sz = sizeof(struct timex); unsigned struct_msqid_ds_sz = sizeof(struct msqid_ds); unsigned struct_mq_attr_sz = sizeof(struct mq_attr); unsigned struct_statvfs_sz = sizeof(struct statvfs); -#endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#endif // SANITIZER_LINUX && !SANITIZER_ANDROID const uptr sig_ign = (uptr)SIG_IGN; const uptr sig_dfl = (uptr)SIG_DFL; @@ -286,7 +256,7 @@ namespace __sanitizer { #endif -#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#if SANITIZER_LINUX && !SANITIZER_ANDROID unsigned struct_shminfo_sz = sizeof(struct shminfo); unsigned struct_shm_info_sz = sizeof(struct shm_info); int shmctl_ipc_stat = (int)IPC_STAT; @@ -322,7 +292,7 @@ unsigned struct_ElfW_Phdr_sz = sizeof(ElfW(Phdr)); unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); #endif -#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#if SANITIZER_LINUX && !SANITIZER_ANDROID int glob_nomatch = GLOB_NOMATCH; int glob_altdirfunc = GLOB_ALTDIRFUNC; #endif @@ -447,7 +417,7 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); unsigned struct_vt_stat_sz = sizeof(struct vt_stat); #endif // SANITIZER_LINUX -#if SANITIZER_LINUX || SANITIZER_FREEBSD +#if SANITIZER_LINUX #if SOUND_VERSION >= 0x040000 unsigned struct_copr_buffer_sz = 0; unsigned struct_copr_debug_buf_sz = 0; @@ -464,7 +434,7 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); unsigned struct_seq_event_rec_sz = sizeof(struct seq_event_rec); unsigned struct_synth_info_sz = sizeof(struct synth_info); unsigned struct_vt_mode_sz = sizeof(struct vt_mode); -#endif // SANITIZER_LINUX || SANITIZER_FREEBSD +#endif // SANITIZER_LINUX #if SANITIZER_LINUX && !SANITIZER_ANDROID unsigned struct_ax25_parms_struct_sz = sizeof(struct ax25_parms_struct); @@ -491,7 +461,7 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); unsigned struct_unimapinit_sz = sizeof(struct unimapinit); #endif // SANITIZER_LINUX && !SANITIZER_ANDROID -#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#if SANITIZER_LINUX && !SANITIZER_ANDROID unsigned struct_audio_buf_info_sz = sizeof(struct audio_buf_info); unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats); #endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID @@ -501,6 +471,8 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req); #endif + const unsigned long __sanitizer_bufsiz = BUFSIZ; + const unsigned IOCTL_NOT_PRESENT = 0; unsigned IOCTL_FIOASYNC = FIOASYNC; @@ -547,7 +519,7 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); unsigned IOCTL_TIOCSPGRP = TIOCSPGRP; unsigned IOCTL_TIOCSTI = TIOCSTI; unsigned IOCTL_TIOCSWINSZ = TIOCSWINSZ; -#if ((SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID) +#if SANITIZER_LINUX && !SANITIZER_ANDROID unsigned IOCTL_SIOCGETSGCNT = SIOCGETSGCNT; unsigned IOCTL_SIOCGETVIFCNT = SIOCGETVIFCNT; #endif @@ -737,9 +709,6 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); unsigned IOCTL_VT_RESIZE = VT_RESIZE; unsigned IOCTL_VT_RESIZEX = VT_RESIZEX; unsigned IOCTL_VT_SENDSIG = VT_SENDSIG; -#endif // SANITIZER_LINUX - -#if SANITIZER_LINUX || SANITIZER_FREEBSD unsigned IOCTL_MTIOCGET = MTIOCGET; unsigned IOCTL_MTIOCTOP = MTIOCTOP; unsigned IOCTL_SNDCTL_DSP_GETBLKSIZE = SNDCTL_DSP_GETBLKSIZE; @@ -832,7 +801,7 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); unsigned IOCTL_VT_RELDISP = VT_RELDISP; unsigned IOCTL_VT_SETMODE = VT_SETMODE; unsigned IOCTL_VT_WAITACTIVE = VT_WAITACTIVE; -#endif // SANITIZER_LINUX || SANITIZER_FREEBSD +#endif // SANITIZER_LINUX #if SANITIZER_LINUX && !SANITIZER_ANDROID unsigned IOCTL_CYGETDEFTHRESH = CYGETDEFTHRESH; @@ -925,7 +894,7 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); unsigned IOCTL_TIOCSSERIAL = TIOCSSERIAL; #endif // SANITIZER_LINUX && !SANITIZER_ANDROID -#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#if SANITIZER_LINUX && !SANITIZER_ANDROID unsigned IOCTL_GIO_SCRNMAP = GIO_SCRNMAP; unsigned IOCTL_KDDISABIO = KDDISABIO; unsigned IOCTL_KDENABIO = KDENABIO; @@ -1207,7 +1176,7 @@ CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_data); #endif #if SANITIZER_LINUX -COMPILER_CHECK(sizeof(__sanitizer_mallinfo) == sizeof(struct mallinfo)); +COMPILER_CHECK(sizeof(__sanitizer_struct_mallinfo) == sizeof(struct mallinfo)); #endif #if !SANITIZER_ANDROID @@ -1243,7 +1212,7 @@ CHECK_SIZE_AND_OFFSET(group, gr_passwd); CHECK_SIZE_AND_OFFSET(group, gr_gid); CHECK_SIZE_AND_OFFSET(group, gr_mem); -#if HAVE_RPC_XDR_H || HAVE_TIRPC_RPC_XDR_H +#if HAVE_RPC_XDR_H CHECK_TYPE_SIZE(XDR); CHECK_SIZE_AND_OFFSET(XDR, x_op); CHECK_SIZE_AND_OFFSET(XDR, x_ops); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h index 476b9be736a5..bc4ddce0a81c 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h @@ -15,22 +15,12 @@ #ifndef SANITIZER_PLATFORM_LIMITS_POSIX_H #define SANITIZER_PLATFORM_LIMITS_POSIX_H -#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC +#if SANITIZER_LINUX || SANITIZER_MAC #include "sanitizer_internal_defs.h" #include "sanitizer_platform.h" -#if SANITIZER_FREEBSD -// FreeBSD's dlopen() returns a pointer to an Obj_Entry structure that -// incorporates the map structure. -# define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \ - ((link_map*)((handle) == nullptr ? nullptr : ((char*)(handle) + 560))) -// Get sys/_types.h, because that tells us whether 64-bit inodes are -// used in struct dirent below. -#include <sys/_types.h> -#else # define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) ((link_map*)(handle)) -#endif // !SANITIZER_FREEBSD #ifndef __GLIBC_PREREQ #define __GLIBC_PREREQ(x, y) 0 @@ -39,7 +29,7 @@ namespace __sanitizer { extern unsigned struct_utsname_sz; extern unsigned struct_stat_sz; -#if !SANITIZER_FREEBSD && !SANITIZER_IOS +#if !SANITIZER_IOS extern unsigned struct_stat64_sz; #endif extern unsigned struct_rusage_sz; @@ -59,8 +49,11 @@ namespace __sanitizer { extern unsigned struct_sigevent_sz; extern unsigned struct_sched_param_sz; extern unsigned struct_statfs64_sz; + extern unsigned struct_regex_sz; + extern unsigned struct_regmatch_sz; #if !SANITIZER_ANDROID + extern unsigned struct_fstab_sz; extern unsigned struct_statfs_sz; extern unsigned struct_sockaddr_sz; extern unsigned ucontext_t_sz; @@ -127,7 +120,7 @@ namespace __sanitizer { const unsigned struct_kexec_segment_sz = 4 * sizeof(unsigned long); #endif // SANITIZER_LINUX -#if SANITIZER_LINUX || SANITIZER_FREEBSD +#if SANITIZER_LINUX #if defined(__powerpc64__) || defined(__riscv) || defined(__s390__) const unsigned struct___old_kernel_stat_sz = 0; @@ -184,20 +177,18 @@ namespace __sanitizer { int data; #elif SANITIZER_LINUX uptr data[4]; -#elif SANITIZER_FREEBSD - u32 data[4]; #endif }; -#endif // SANITIZER_LINUX || SANITIZER_FREEBSD +#endif // SANITIZER_LINUX #if SANITIZER_ANDROID - struct __sanitizer_mallinfo { + struct __sanitizer_struct_mallinfo { uptr v[10]; }; #endif #if SANITIZER_LINUX && !SANITIZER_ANDROID - struct __sanitizer_mallinfo { + struct __sanitizer_struct_mallinfo { int v[10]; }; @@ -310,35 +301,14 @@ namespace __sanitizer { #endif #endif }; -#elif SANITIZER_FREEBSD - struct __sanitizer_ipc_perm { - unsigned int cuid; - unsigned int cgid; - unsigned int uid; - unsigned int gid; - unsigned short mode; - unsigned short seq; - long key; - }; - - struct __sanitizer_shmid_ds { - __sanitizer_ipc_perm shm_perm; - unsigned long shm_segsz; - unsigned int shm_lpid; - unsigned int shm_cpid; - int shm_nattch; - unsigned long shm_atime; - unsigned long shm_dtime; - unsigned long shm_ctime; - }; #endif -#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#if SANITIZER_LINUX && !SANITIZER_ANDROID extern unsigned struct_msqid_ds_sz; extern unsigned struct_mq_attr_sz; extern unsigned struct_timex_sz; extern unsigned struct_statvfs_sz; -#endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#endif // SANITIZER_LINUX && !SANITIZER_ANDROID struct __sanitizer_iovec { void *iov_base; @@ -388,7 +358,7 @@ namespace __sanitizer { char *pw_passwd; int pw_uid; int pw_gid; -#if SANITIZER_MAC || SANITIZER_FREEBSD +#if SANITIZER_MAC long pw_change; char *pw_class; #endif @@ -397,12 +367,9 @@ namespace __sanitizer { #endif char *pw_dir; char *pw_shell; -#if SANITIZER_MAC || SANITIZER_FREEBSD +#if SANITIZER_MAC long pw_expire; #endif -#if SANITIZER_FREEBSD - int pw_fields; -#endif }; struct __sanitizer_group { @@ -472,7 +439,7 @@ namespace __sanitizer { }; #endif -#if SANITIZER_MAC || SANITIZER_FREEBSD +#if SANITIZER_MAC struct __sanitizer_msghdr { void *msg_name; unsigned msg_namelen; @@ -518,17 +485,6 @@ namespace __sanitizer { unsigned short d_reclen; // more fields that we don't care about }; -#elif SANITIZER_FREEBSD - struct __sanitizer_dirent { -#if defined(__INO64) - unsigned long long d_fileno; - unsigned long long d_off; -#else - unsigned int d_fileno; -#endif - unsigned short d_reclen; - // more fields that we don't care about - }; #elif SANITIZER_ANDROID || defined(__x86_64__) struct __sanitizer_dirent { unsigned long long d_ino; @@ -554,20 +510,17 @@ namespace __sanitizer { }; #endif -// 'clock_t' is 32 bits wide on x64 FreeBSD -#if SANITIZER_FREEBSD - typedef int __sanitizer_clock_t; -#elif defined(__x86_64__) && !defined(_LP64) +#if defined(__x86_64__) && !defined(_LP64) typedef long long __sanitizer_clock_t; #else typedef long __sanitizer_clock_t; #endif -#if SANITIZER_LINUX || SANITIZER_FREEBSD +#if SANITIZER_LINUX typedef int __sanitizer_clockid_t; #endif -#if SANITIZER_LINUX || SANITIZER_FREEBSD +#if SANITIZER_LINUX #if defined(_LP64) || defined(__x86_64__) || defined(__powerpc__)\ || defined(__mips__) typedef unsigned __sanitizer___kernel_uid_t; @@ -617,11 +570,6 @@ namespace __sanitizer { // The size is determined by looking at sizeof of real sigset_t on linux. uptr val[128 / sizeof(uptr)]; }; -#elif SANITIZER_FREEBSD - struct __sanitizer_sigset_t { - // uint32_t * 4 - unsigned int __bits[4]; - }; #endif struct __sanitizer_siginfo { @@ -711,9 +659,7 @@ namespace __sanitizer { }; #endif // !SANITIZER_ANDROID -#if SANITIZER_FREEBSD - typedef __sanitizer_sigset_t __sanitizer_kernel_sigset_t; -#elif defined(__mips__) +#if defined(__mips__) struct __sanitizer_kernel_sigset_t { uptr sig[2]; }; @@ -759,7 +705,7 @@ namespace __sanitizer { extern int af_inet6; uptr __sanitizer_in_addr_sz(int af); -#if SANITIZER_LINUX || SANITIZER_FREEBSD +#if SANITIZER_LINUX struct __sanitizer_dl_phdr_info { uptr dlpi_addr; const char *dlpi_name; @@ -775,7 +721,7 @@ namespace __sanitizer { int ai_family; int ai_socktype; int ai_protocol; -#if SANITIZER_ANDROID || SANITIZER_MAC || SANITIZER_FREEBSD +#if SANITIZER_ANDROID || SANITIZER_MAC unsigned ai_addrlen; char *ai_canonname; void *ai_addr; @@ -801,7 +747,7 @@ namespace __sanitizer { short revents; }; -#if SANITIZER_ANDROID || SANITIZER_MAC || SANITIZER_FREEBSD +#if SANITIZER_ANDROID || SANITIZER_MAC typedef unsigned __sanitizer_nfds_t; #else typedef unsigned long __sanitizer_nfds_t; @@ -821,23 +767,9 @@ namespace __sanitizer { int (*gl_lstat)(const char *, void *); int (*gl_stat)(const char *, void *); }; -# elif SANITIZER_FREEBSD - struct __sanitizer_glob_t { - uptr gl_pathc; - uptr gl_matchc; - uptr gl_offs; - int gl_flags; - char **gl_pathv; - int (*gl_errfunc)(const char*, int); - void (*gl_closedir)(void *dirp); - struct dirent *(*gl_readdir)(void *dirp); - void *(*gl_opendir)(const char*); - int (*gl_lstat)(const char*, void* /* struct stat* */); - int (*gl_stat)(const char*, void* /* struct stat* */); - }; -# endif // SANITIZER_FREEBSD +# endif // SANITIZER_LINUX -# if SANITIZER_LINUX || SANITIZER_FREEBSD +# if SANITIZER_LINUX extern int glob_nomatch; extern int glob_altdirfunc; # endif @@ -849,10 +781,6 @@ namespace __sanitizer { uptr we_wordc; char **we_wordv; uptr we_offs; -#if SANITIZER_FREEBSD - char *we_strings; - uptr we_nbytes; -#endif }; #if SANITIZER_LINUX && !SANITIZER_ANDROID @@ -906,7 +834,7 @@ namespace __sanitizer { extern int ptrace_geteventmsg; #endif -#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#if SANITIZER_LINUX && !SANITIZER_ANDROID extern unsigned struct_shminfo_sz; extern unsigned struct_shm_info_sz; extern int shmctl_ipc_stat; @@ -1043,7 +971,7 @@ struct __sanitizer_cookie_io_functions_t { extern unsigned struct_vt_stat_sz; #endif // SANITIZER_LINUX -#if SANITIZER_LINUX || SANITIZER_FREEBSD +#if SANITIZER_LINUX extern unsigned struct_copr_buffer_sz; extern unsigned struct_copr_debug_buf_sz; extern unsigned struct_copr_msg_sz; @@ -1055,7 +983,7 @@ struct __sanitizer_cookie_io_functions_t { extern unsigned struct_seq_event_rec_sz; extern unsigned struct_synth_info_sz; extern unsigned struct_vt_mode_sz; -#endif // SANITIZER_LINUX || SANITIZER_FREEBSD +#endif // SANITIZER_LINUX #if SANITIZER_LINUX && !SANITIZER_ANDROID extern unsigned struct_ax25_parms_struct_sz; @@ -1077,7 +1005,9 @@ struct __sanitizer_cookie_io_functions_t { extern unsigned struct_unimapinit_sz; #endif // SANITIZER_LINUX && !SANITIZER_ANDROID -#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID + extern const unsigned long __sanitizer_bufsiz; + +#if SANITIZER_LINUX && !SANITIZER_ANDROID extern unsigned struct_audio_buf_info_sz; extern unsigned struct_ppp_stats_sz; #endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID @@ -1137,7 +1067,7 @@ struct __sanitizer_cookie_io_functions_t { extern unsigned IOCTL_TIOCSPGRP; extern unsigned IOCTL_TIOCSTI; extern unsigned IOCTL_TIOCSWINSZ; -#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#if SANITIZER_LINUX && !SANITIZER_ANDROID extern unsigned IOCTL_SIOCGETSGCNT; extern unsigned IOCTL_SIOCGETVIFCNT; #endif @@ -1299,8 +1229,6 @@ struct __sanitizer_cookie_io_functions_t { extern unsigned IOCTL_VT_RESIZE; extern unsigned IOCTL_VT_RESIZEX; extern unsigned IOCTL_VT_SENDSIG; -#endif // SANITIZER_LINUX -#if SANITIZER_LINUX || SANITIZER_FREEBSD extern unsigned IOCTL_MTIOCGET; extern unsigned IOCTL_MTIOCTOP; extern unsigned IOCTL_SIOCADDRT; @@ -1401,7 +1329,7 @@ struct __sanitizer_cookie_io_functions_t { extern unsigned IOCTL_VT_RELDISP; extern unsigned IOCTL_VT_SETMODE; extern unsigned IOCTL_VT_WAITACTIVE; -#endif // SANITIZER_LINUX || SANITIZER_FREEBSD +#endif // SANITIZER_LINUX #if SANITIZER_LINUX && !SANITIZER_ANDROID extern unsigned IOCTL_CYGETDEFTHRESH; @@ -1488,9 +1416,6 @@ struct __sanitizer_cookie_io_functions_t { extern unsigned IOCTL_TIOCSERGETMULTI; extern unsigned IOCTL_TIOCSERSETMULTI; extern unsigned IOCTL_TIOCSSERIAL; -#endif // SANITIZER_LINUX && !SANITIZER_ANDROID - -#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID extern unsigned IOCTL_GIO_SCRNMAP; extern unsigned IOCTL_KDDISABIO; extern unsigned IOCTL_KDENABIO; @@ -1529,6 +1454,6 @@ struct __sanitizer_cookie_io_functions_t { #define SIGACTION_SYMNAME sigaction -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC +#endif // SANITIZER_LINUX || SANITIZER_MAC #endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.h index c0aa4cc1b17e..08bd24cd23e0 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.h @@ -210,8 +210,7 @@ struct __sanitizer_cmsghdr { int cmsg_type; }; -#if SANITIZER_SOLARIS32 && 0 -// FIXME: need to deal with large file and non-large file cases +#if SANITIZER_SOLARIS && (defined(_LP64) || _FILE_OFFSET_BITS == 64) struct __sanitizer_dirent { unsigned long long d_ino; long long d_off; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc index f7dfc86f58c6..b965fb0da1aa 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc @@ -18,6 +18,7 @@ #include "sanitizer_common.h" #include "sanitizer_file.h" +#include "sanitizer_flags.h" #include "sanitizer_libc.h" #include "sanitizer_posix.h" #include "sanitizer_procmaps.h" @@ -157,6 +158,8 @@ void MprotectMallocZones(void *addr, int prot) {} #endif fd_t OpenFile(const char *filename, FileAccessMode mode, error_t *errno_p) { + if (ShouldMockFailureToOpen(filename)) + return kInvalidFd; int flags; switch (mode) { case RdOnly: flags = O_RDONLY; break; @@ -166,7 +169,7 @@ fd_t OpenFile(const char *filename, FileAccessMode mode, error_t *errno_p) { fd_t res = internal_open(filename, flags, 0660); if (internal_iserror(res, errno_p)) return kInvalidFd; - return res; + return ReserveStandardFds(res); } void CloseFile(fd_t fd) { @@ -193,11 +196,6 @@ bool WriteToFile(fd_t fd, const void *buff, uptr buff_size, uptr *bytes_written, return true; } -bool RenameFile(const char *oldpath, const char *newpath, error_t *error_p) { - uptr res = internal_rename(oldpath, newpath); - return !internal_iserror(res, error_p); -} - void *MapFileToMemory(const char *file_name, uptr *buff_size) { fd_t fd = OpenFile(file_name, RdOnly); CHECK(fd != kInvalidFd); @@ -235,6 +233,8 @@ static inline bool IntervalsAreSeparate(uptr start1, uptr end1, // memory). bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) { MemoryMappingLayout proc_maps(/*cache_enabled*/true); + if (proc_maps.Error()) + return true; // and hope for the best MemoryMappedSegment segment; while (proc_maps.Next(&segment)) { if (segment.start == segment.end) continue; // Empty range. @@ -274,13 +274,8 @@ bool IsAbsolutePath(const char *path) { void ReportFile::Write(const char *buffer, uptr length) { SpinMutexLock l(mu); - static const char *kWriteError = - "ReportFile::Write() can't output requested buffer!\n"; ReopenIfNecessary(); - if (length != internal_write(fd, buffer, length)) { - internal_write(fd, kWriteError, internal_strlen(kWriteError)); - Die(); - } + internal_write(fd, buffer, length); } bool GetCodeRangeForFile(const char *module, uptr *start, uptr *end) { @@ -328,6 +323,27 @@ const char *SignalContext::Describe() const { return "UNKNOWN SIGNAL"; } +fd_t ReserveStandardFds(fd_t fd) { + CHECK_GE(fd, 0); + if (fd > 2) + return fd; + bool used[3]; + internal_memset(used, 0, sizeof(used)); + while (fd <= 2) { + used[fd] = true; + fd = internal_dup(fd); + } + for (int i = 0; i <= 2; ++i) + if (used[i]) + internal_close(i); + return fd; +} + +bool ShouldMockFailureToOpen(const char *path) { + return common_flags()->test_only_emulate_no_memorymap && + internal_strncmp(path, "/proc/", 6) == 0; +} + } // namespace __sanitizer #endif // SANITIZER_POSIX diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h index da447002b66c..04a76445eab2 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h @@ -16,6 +16,7 @@ // ----------- ATTENTION ------------- // This header should NOT include any other headers from sanitizer runtime. #include "sanitizer_internal_defs.h" +#include "sanitizer_platform_limits_freebsd.h" #include "sanitizer_platform_limits_netbsd.h" #include "sanitizer_platform_limits_openbsd.h" #include "sanitizer_platform_limits_posix.h" @@ -48,6 +49,7 @@ uptr internal_filesize(fd_t fd); // -1 on error. uptr internal_stat(const char *path, void *buf); uptr internal_lstat(const char *path, void *buf); uptr internal_fstat(fd_t fd, void *buf); +uptr internal_dup(int oldfd); uptr internal_dup2(int oldfd, int newfd); uptr internal_readlink(const char *path, char *buf, uptr bufsize); uptr internal_unlink(const char *path); @@ -60,6 +62,11 @@ uptr internal_waitpid(int pid, int *status, int options); int internal_fork(); int internal_forkpty(int *amaster); +int internal_sysctl(const int *name, unsigned int namelen, void *oldp, + uptr *oldlenp, const void *newp, uptr newlen); +int internal_sysctlbyname(const char *sname, void *oldp, uptr *oldlenp, + const void *newp, uptr newlen); + // These functions call appropriate pthread_ functions directly, bypassing // the interceptor. They are weak and may not be present in some tools. SANITIZER_WEAK_ATTRIBUTE @@ -93,6 +100,11 @@ uptr internal_execve(const char *filename, char *const argv[], bool IsStateDetached(int state); +// Move the fd out of {0, 1, 2} range. +fd_t ReserveStandardFds(fd_t fd); + +bool ShouldMockFailureToOpen(const char *path); + } // namespace __sanitizer #endif // SANITIZER_POSIX_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc index 266e9bdba051..3006e60d89f3 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc @@ -94,10 +94,12 @@ static rlim_t getlim(int res) { } static void setlim(int res, rlim_t lim) { - // The following magic is to prevent clang from replacing it with memset. - volatile struct rlimit rlim; + struct rlimit rlim; + if (getrlimit(res, const_cast<struct rlimit *>(&rlim))) { + Report("ERROR: %s getrlimit() failed %d\n", SanitizerToolName, errno); + Die(); + } rlim.rlim_cur = lim; - rlim.rlim_max = lim; if (setrlimit(res, const_cast<struct rlimit *>(&rlim))) { Report("ERROR: %s setrlimit() failed %d\n", SanitizerToolName, errno); Die(); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h index 9fde040a11a6..acb7104f33e6 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h @@ -70,6 +70,7 @@ class MemoryMappingLayout { explicit MemoryMappingLayout(bool cache_enabled); ~MemoryMappingLayout(); bool Next(MemoryMappedSegment *segment); + bool Error() const; void Reset(); // In some cases, e.g. when running under a sandbox on Linux, ASan is unable // to obtain the memory mappings. It should fall back to pre-cached data diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_bsd.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_bsd.cc index e41dc987dcd7..362a424d737a 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_bsd.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_bsd.cc @@ -67,8 +67,8 @@ void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { #endif }; - size_t Size = 0; - int Err = sysctl(Mib, ARRAY_SIZE(Mib), NULL, &Size, NULL, 0); + uptr Size = 0; + int Err = internal_sysctl(Mib, ARRAY_SIZE(Mib), NULL, &Size, NULL, 0); CHECK_EQ(Err, 0); CHECK_GT(Size, 0); @@ -76,7 +76,7 @@ void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { size_t MmapedSize = Size * 4 / 3; void *VmMap = MmapOrDie(MmapedSize, "ReadProcMaps()"); Size = MmapedSize; - Err = sysctl(Mib, ARRAY_SIZE(Mib), VmMap, &Size, NULL, 0); + Err = internal_sysctl(Mib, ARRAY_SIZE(Mib), VmMap, &Size, NULL, 0); CHECK_EQ(Err, 0); proc_maps->data = (char *)VmMap; #else @@ -88,7 +88,7 @@ void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { if (Size > 0x10000) Size = 0x10000; Size = (Size / sizeof(struct kinfo_vmentry)) * sizeof(struct kinfo_vmentry); - Err = sysctl(Mib, ARRAY_SIZE(Mib), Mem, &Size, NULL, 0); + Err = internal_sysctl(Mib, ARRAY_SIZE(Mib), Mem, &Size, NULL, 0); CHECK_EQ(Err, 0); MmapedSize = Size; proc_maps->data = Mem; @@ -99,6 +99,7 @@ void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { } bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { + CHECK(!Error()); // can not fail char *last = data_.proc_self_maps.data + data_.proc_self_maps.len; if (data_.current >= last) return false; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc index 1f2b431c7ccd..17d61b6c5e8f 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc @@ -80,12 +80,14 @@ MemoryMappingLayout::MemoryMappingLayout(bool cache_enabled) { ReadProcMaps(&data_.proc_self_maps); if (cache_enabled && data_.proc_self_maps.mmaped_size == 0) LoadFromCache(); - CHECK_GT(data_.proc_self_maps.mmaped_size, 0); - CHECK_GT(data_.proc_self_maps.len, 0); Reset(); } +bool MemoryMappingLayout::Error() const { + return data_.current == nullptr; +} + MemoryMappingLayout::~MemoryMappingLayout() { // Only unmap the buffer if it is different from the cached one. Otherwise // it will be unmapped when the cache is refreshed. diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc index 633e9393cce0..cf9cb25ba061 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc @@ -31,6 +31,7 @@ static bool IsOneOf(char c, char c1, char c2) { } bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { + if (Error()) return false; // simulate empty maps char *last = data_.proc_self_maps.data + data_.proc_self_maps.len; if (data_.current >= last) return false; char *next_line = diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc index 0167ab18ba13..267c960b55d4 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc @@ -88,6 +88,10 @@ MemoryMappingLayout::MemoryMappingLayout(bool cache_enabled) { MemoryMappingLayout::~MemoryMappingLayout() { } +bool MemoryMappingLayout::Error() const { + return false; +} + // More information about Mach-O headers can be found in mach-o/loader.h // Each Mach-O image has a header (mach_header or mach_header_64) starting with // a magic number, and a list of linker load commands directly following the @@ -140,12 +144,6 @@ void MemoryMappingLayout::LoadFromCache() { // early in the process, when dyld is one of the only images loaded, // so it will be hit after only a few iterations. static mach_header *get_dyld_image_header() { - mach_port_name_t port; - if (task_for_pid(mach_task_self(), internal_getpid(), &port) != - KERN_SUCCESS) { - return nullptr; - } - unsigned depth = 1; vm_size_t size = 0; vm_address_t address = 0; @@ -154,7 +152,7 @@ static mach_header *get_dyld_image_header() { while (true) { struct vm_region_submap_info_64 info; - err = vm_region_recurse_64(port, &address, &size, &depth, + err = vm_region_recurse_64(mach_task_self(), &address, &size, &depth, (vm_region_info_t)&info, &count); if (err != KERN_SUCCESS) return nullptr; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cc index bfe83170f4e2..49bb46c31b08 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cc @@ -13,17 +13,24 @@ #include "sanitizer_common.h" #include "sanitizer_procmaps.h" +// Before Solaris 11.4, <procfs.h> doesn't work in a largefile environment. +#undef _FILE_OFFSET_BITS #include <procfs.h> #include <limits.h> namespace __sanitizer { void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { - ReadFileToBuffer("/proc/self/xmap", &proc_maps->data, &proc_maps->mmaped_size, - &proc_maps->len); + if (!ReadFileToBuffer("/proc/self/xmap", &proc_maps->data, + &proc_maps->mmaped_size, &proc_maps->len)) { + proc_maps->data = nullptr; + proc_maps->mmaped_size = 0; + proc_maps->len = 0; + } } bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { + if (Error()) return false; // simulate empty maps char *last = data_.proc_self_maps.data + data_.proc_self_maps.len; if (data_.current >= last) return false; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_ring_buffer.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_ring_buffer.h new file mode 100644 index 000000000000..d15f27fd4a83 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_ring_buffer.h @@ -0,0 +1,162 @@ +//===-- sanitizer_ring_buffer.h ---------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Simple ring buffer. +// +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_RING_BUFFER_H +#define SANITIZER_RING_BUFFER_H + +#include "sanitizer_common.h" + +namespace __sanitizer { +// RingBuffer<T>: fixed-size ring buffer optimized for speed of push(). +// T should be a POD type and sizeof(T) should be divisible by sizeof(void*). +// At creation, all elements are zero. +template<class T> +class RingBuffer { + public: + COMPILER_CHECK(sizeof(T) % sizeof(void *) == 0); + static RingBuffer *New(uptr Size) { + void *Ptr = MmapOrDie(SizeInBytes(Size), "RingBuffer"); + RingBuffer *RB = reinterpret_cast<RingBuffer*>(Ptr); + uptr End = reinterpret_cast<uptr>(Ptr) + SizeInBytes(Size); + RB->last_ = RB->next_ = reinterpret_cast<T*>(End - sizeof(T)); + return RB; + } + void Delete() { + UnmapOrDie(this, SizeInBytes(size())); + } + uptr size() const { + return last_ + 1 - + reinterpret_cast<T *>(reinterpret_cast<uptr>(this) + + 2 * sizeof(T *)); + } + + static uptr SizeInBytes(uptr Size) { + return Size * sizeof(T) + 2 * sizeof(T*); + } + + uptr SizeInBytes() { return SizeInBytes(size()); } + + void push(T t) { + *next_ = t; + next_--; + // The condition below works only if sizeof(T) is divisible by sizeof(T*). + if (next_ <= reinterpret_cast<T*>(&next_)) + next_ = last_; + } + + T operator[](uptr Idx) const { + CHECK_LT(Idx, size()); + sptr IdxNext = Idx + 1; + if (IdxNext > last_ - next_) + IdxNext -= size(); + return next_[IdxNext]; + } + + private: + RingBuffer() {} + ~RingBuffer() {} + RingBuffer(const RingBuffer&) = delete; + + // Data layout: + // LNDDDDDDDD + // D: data elements. + // L: last_, always points to the last data element. + // N: next_, initially equals to last_, is decremented on every push, + // wraps around if it's less or equal than its own address. + T *last_; + T *next_; + T data_[1]; // flexible array. +}; + +// A ring buffer with externally provided storage that encodes its state in 8 +// bytes. Has significant constraints on size and alignment of storage. +// See a comment in hwasan/hwasan_thread_list.h for the motivation behind this. +#if SANITIZER_WORDSIZE == 64 +template <class T> +class CompactRingBuffer { + // Top byte of long_ stores the buffer size in pages. + // Lower bytes store the address of the next buffer element. + static constexpr int kPageSizeBits = 12; + static constexpr int kSizeShift = 56; + static constexpr uptr kNextMask = (1ULL << kSizeShift) - 1; + + uptr GetStorageSize() const { return (long_ >> kSizeShift) << kPageSizeBits; } + + void Init(void *storage, uptr size) { + CHECK_EQ(sizeof(CompactRingBuffer<T>), sizeof(void *)); + CHECK(IsPowerOfTwo(size)); + CHECK_GE(size, 1 << kPageSizeBits); + CHECK_LE(size, 128 << kPageSizeBits); + CHECK_EQ(size % 4096, 0); + CHECK_EQ(size % sizeof(T), 0); + CHECK_EQ((uptr)storage % (size * 2), 0); + long_ = (uptr)storage | ((size >> kPageSizeBits) << kSizeShift); + } + + void SetNext(const T *next) { + long_ = (long_ & ~kNextMask) | (uptr)next; + } + + public: + CompactRingBuffer(void *storage, uptr size) { + Init(storage, size); + } + + // A copy constructor of sorts. + CompactRingBuffer(const CompactRingBuffer &other, void *storage) { + uptr size = other.GetStorageSize(); + internal_memcpy(storage, other.StartOfStorage(), size); + Init(storage, size); + uptr Idx = other.Next() - (const T *)other.StartOfStorage(); + SetNext((const T *)storage + Idx); + } + + T *Next() const { return (T *)(long_ & kNextMask); } + + void *StartOfStorage() const { + return (void *)((uptr)Next() & ~(GetStorageSize() - 1)); + } + + void *EndOfStorage() const { + return (void *)((uptr)StartOfStorage() + GetStorageSize()); + } + + uptr size() const { return GetStorageSize() / sizeof(T); } + + void push(T t) { + T *next = Next(); + *next = t; + next++; + next = (T *)((uptr)next & ~GetStorageSize()); + SetNext(next); + } + + T operator[](uptr Idx) const { + CHECK_LT(Idx, size()); + const T *Begin = (const T *)StartOfStorage(); + sptr StorageIdx = Next() - Begin; + StorageIdx -= (sptr)(Idx + 1); + if (StorageIdx < 0) + StorageIdx += size(); + return Begin[StorageIdx]; + } + + public: + ~CompactRingBuffer() {} + CompactRingBuffer(const CompactRingBuffer &) = delete; + + uptr long_; +}; +#endif +} // namespace __sanitizer + +#endif // SANITIZER_RING_BUFFER_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_rtems.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_rtems.cc index 4be367911e3f..76aebe419034 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_rtems.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_rtems.cc @@ -95,8 +95,10 @@ void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, *tls_addr = *tls_size = 0; } +void InitializePlatformEarly() {} void MaybeReexec() {} void CheckASLR() {} +void CheckMPROTECT() {} void DisableCoreDumperIfNecessary() {} void InstallDeadlySignalHandlers(SignalHandlerType handler) {} void SetAlternateSignalStack() {} @@ -226,11 +228,6 @@ bool WriteToFile(fd_t fd, const void *buff, uptr buff_size, uptr *bytes_written, return true; } -bool RenameFile(const char *oldpath, const char *newpath, error_t *error_p) { - uptr res = rename(oldpath, newpath); - return !internal_iserror(res, error_p); -} - void ReleaseMemoryPagesToOS(uptr beg, uptr end) {} void DumpProcessMap() {} @@ -240,6 +237,7 @@ bool IsAccessibleMemoryRange(uptr beg, uptr size) { } char **GetArgv() { return nullptr; } +char **GetEnviron() { return nullptr; } const char *GetEnv(const char *name) { return getenv(name); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_solaris.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_solaris.cc index a5db22994e0a..cc0201c7a37e 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_solaris.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_solaris.cc @@ -48,10 +48,21 @@ namespace __sanitizer { DECLARE__REAL(ret_type, func, __VA_ARGS__); \ ret_type internal_ ## func(__VA_ARGS__) +#if !defined(_LP64) && _FILE_OFFSET_BITS == 64 +#define _REAL64(func) _ ## func ## 64 +#else +#define _REAL64(func) _REAL(func) +#endif +#define DECLARE__REAL64(ret_type, func, ...) \ + extern "C" ret_type _REAL64(func)(__VA_ARGS__) +#define DECLARE__REAL_AND_INTERNAL64(ret_type, func, ...) \ + DECLARE__REAL64(ret_type, func, __VA_ARGS__); \ + ret_type internal_ ## func(__VA_ARGS__) + // ---------------------- sanitizer_libc.h -DECLARE__REAL_AND_INTERNAL(uptr, mmap, void *addr, uptr /*size_t*/ length, - int prot, int flags, int fd, OFF_T offset) { - return (uptr)_REAL(mmap)(addr, length, prot, flags, fd, offset); +DECLARE__REAL_AND_INTERNAL64(uptr, mmap, void *addr, uptr /*size_t*/ length, + int prot, int flags, int fd, OFF_T offset) { + return (uptr)_REAL64(mmap)(addr, length, prot, flags, fd, offset); } DECLARE__REAL_AND_INTERNAL(uptr, munmap, void *addr, uptr length) { @@ -66,19 +77,19 @@ DECLARE__REAL_AND_INTERNAL(uptr, close, fd_t fd) { return _REAL(close)(fd); } -extern "C" int _REAL(open)(const char *, int, ...); +extern "C" int _REAL64(open)(const char *, int, ...); uptr internal_open(const char *filename, int flags) { - return _REAL(open)(filename, flags); + return _REAL64(open)(filename, flags); } uptr internal_open(const char *filename, int flags, u32 mode) { - return _REAL(open)(filename, flags, mode); + return _REAL64(open)(filename, flags, mode); } uptr OpenFile(const char *filename, bool write) { - return internal_open(filename, - write ? O_WRONLY | O_CREAT : O_RDONLY, 0660); + return ReserveStandardFds( + internal_open(filename, write ? O_WRONLY | O_CREAT : O_RDONLY, 0660)); } DECLARE__REAL_AND_INTERNAL(uptr, read, fd_t fd, void *buf, uptr count) { @@ -94,16 +105,16 @@ DECLARE__REAL_AND_INTERNAL(uptr, ftruncate, fd_t fd, uptr size) { return ftruncate(fd, size); } -DECLARE__REAL_AND_INTERNAL(uptr, stat, const char *path, void *buf) { - return _REAL(stat)(path, (struct stat *)buf); +DECLARE__REAL_AND_INTERNAL64(uptr, stat, const char *path, void *buf) { + return _REAL64(stat)(path, (struct stat *)buf); } -DECLARE__REAL_AND_INTERNAL(uptr, lstat, const char *path, void *buf) { - return _REAL(lstat)(path, (struct stat *)buf); +DECLARE__REAL_AND_INTERNAL64(uptr, lstat, const char *path, void *buf) { + return _REAL64(lstat)(path, (struct stat *)buf); } -DECLARE__REAL_AND_INTERNAL(uptr, fstat, fd_t fd, void *buf) { - return _REAL(fstat)(fd, (struct stat *)buf); +DECLARE__REAL_AND_INTERNAL64(uptr, fstat, fd_t fd, void *buf) { + return _REAL64(fstat)(fd, (struct stat *)buf); } uptr internal_filesize(fd_t fd) { @@ -153,13 +164,13 @@ DECLARE__REAL_AND_INTERNAL(uptr, getpid, void) { } // FIXME: This might be wrong: _getdents doesn't take a struct linux_dirent *. -DECLARE__REAL_AND_INTERNAL(uptr, getdents, fd_t fd, struct linux_dirent *dirp, - unsigned int count) { - return _REAL(getdents)(fd, dirp, count); +DECLARE__REAL_AND_INTERNAL64(uptr, getdents, fd_t fd, struct linux_dirent *dirp, + unsigned int count) { + return _REAL64(getdents)(fd, dirp, count); } -DECLARE__REAL_AND_INTERNAL(uptr, lseek, fd_t fd, OFF_T offset, int whence) { - return _REAL(lseek)(fd, offset, whence); +DECLARE__REAL_AND_INTERNAL64(uptr, lseek, fd_t fd, OFF_T offset, int whence) { + return _REAL64(lseek)(fd, offset, whence); } // FIXME: This might be wrong: _sigfillset doesn't take a diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cc index 3bd5b677a1f8..6aab98485226 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cc @@ -26,7 +26,7 @@ struct StackDepotNode { u32 tag; uptr stack[1]; // [size] - static const u32 kTabSizeLog = 20; + static const u32 kTabSizeLog = SANITIZER_ANDROID ? 16 : 20; // Lower kTabSizeLog bits are equal for all items in one bucket. // We use these bits to store the per-stack use counter. static const u32 kUseCountBits = kTabSizeLog; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.h index cb7345002a40..e22ed2e38e57 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.h @@ -32,7 +32,7 @@ struct StackDepotHandle { void inc_use_count_unsafe(); }; -const int kStackDepotMaxUseCount = 1U << 20; +const int kStackDepotMaxUseCount = 1U << (SANITIZER_ANDROID ? 16 : 20); StackDepotStats *StackDepotGetStats(); u32 StackDepotPut(StackTrace stack); diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h index 562d2e9f737c..450a40a9069f 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h @@ -19,7 +19,7 @@ namespace __sanitizer { static const u32 kStackTraceMax = 256; -#if SANITIZER_LINUX && (defined(__sparc__) || defined(__mips__)) +#if defined(__sparc__) || (SANITIZER_LINUX && defined(__mips__)) # define SANITIZER_CAN_FAST_UNWIND 0 #elif SANITIZER_WINDOWS # define SANITIZER_CAN_FAST_UNWIND 0 diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc index 747a4a701728..c87b18e1b696 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc @@ -114,11 +114,25 @@ void __sanitizer_symbolize_pc(uptr pc, const char *fmt, char *out_buf, return; } InternalScopedString frame_desc(GetPageSizeCached()); - RenderFrame(&frame_desc, fmt, 0, frame->info, - common_flags()->symbolize_vs_style, - common_flags()->strip_path_prefix); - internal_strncpy(out_buf, frame_desc.data(), out_buf_size); - out_buf[out_buf_size - 1] = 0; + uptr frame_num = 0; + // Reserve one byte for the final 0. + char *out_end = out_buf + out_buf_size - 1; + for (SymbolizedStack *cur = frame; cur && out_buf < out_end; + cur = cur->next) { + frame_desc.clear(); + RenderFrame(&frame_desc, fmt, frame_num++, cur->info, + common_flags()->symbolize_vs_style, + common_flags()->strip_path_prefix); + if (!frame_desc.length()) + continue; + // Reserve one byte for the terminating 0. + uptr n = out_end - out_buf - 1; + internal_strncpy(out_buf, frame_desc.data(), n); + out_buf += __sanitizer::Min<uptr>(n, frame_desc.length()); + *out_buf++ = 0; + } + CHECK(out_buf <= out_end); + *out_buf = 0; } SANITIZER_INTERFACE_ATTRIBUTE diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cc index ac0731d46a6b..f2b337433427 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cc @@ -98,6 +98,8 @@ static const char *DemangleFunctionName(const char *function) { return "pthread_equal"; if (!internal_strcmp(function, "__libc_thr_curcpu")) return "pthread_curcpu_np"; + if (!internal_strcmp(function, "__libc_thr_sigsetmask")) + return "pthread_sigmask"; #endif return function; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_sparc.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_sparc.cc index 9f9920ece80b..f41a3cefb350 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_sparc.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_sparc.cc @@ -15,7 +15,7 @@ // This file is ported to Sparc v8, but it should be easy to port to // Sparc v9. -#if defined(__sparcv8__) +#if defined(__sparcv8__) || defined(__sparcv8) || defined(__sparc_v8__) #include "sanitizer_common.h" #include "sanitizer_stacktrace.h" @@ -55,4 +55,5 @@ void BufferedStackTrace::FastUnwindStack(uptr pc, uptr bp, uptr stack_top, } // namespace __sanitizer -#endif // !defined(__sparcv8__) +#endif // !defined(__sparcv8__) && !defined(__sparcv8) && + // !defined(__sparc_v8__) diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.h index 3c1c864c0615..b241b9dbc3c8 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.h @@ -35,6 +35,9 @@ constexpr const char *kFormatData = "{{{data:%p}}}"; // One frame in a backtrace (printed on a line by itself). constexpr const char *kFormatFrame = "{{{bt:%u:%p}}}"; +// Dump trigger element. +#define FORMAT_DUMPFILE "{{{dumpfile:%s:%s}}}" + } // namespace __sanitizer #endif // SANITIZER_SYMBOLIZER_FUCHSIA_H diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc index e4ed1b4deee0..ddfb26397f44 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc @@ -11,46 +11,21 @@ // //===----------------------------------------------------------------------===// -#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD || \ - SANITIZER_OPENBSD || SANITIZER_SOLARIS +// NetBSD uses libc calls directly +#if !SANITIZER_NETBSD + +#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_OPENBSD || SANITIZER_SOLARIS # define SYSCALL(name) SYS_ ## name #else # define SYSCALL(name) __NR_ ## name #endif -#if SANITIZER_NETBSD -// We use 3 kinds of internal_syscall's for different types of retval in order -// to address differences in calling conventions (e.g. registers to place the -// return value in). -// - internal_syscall for 32-bit length (int, pid_t) -// - internal_syscall64 for 64-bit length (off_t) -// - internal_syscall_ptr for pointer and (s)size_t -# define internal_syscall syscall -# define internal_syscall64 __syscall -// Handle syscall renames manually -# define SYS_stat SYS___stat50 -# define SYS_lstat SYS___lstat50 -# define SYS_fstat SYS___fstat50 -# define SYS_gettimeofday SYS___gettimeofday50 -# define SYS_wait4 SYS___wait450 -# define SYS_getdents SYS___getdents30 -# define SYS_sigaltstack SYS___sigaltstack14 -# define SYS_sigprocmask SYS___sigprocmask14 -# define SYS_nanosleep SYS___nanosleep50 -# define SYS_clock_gettime SYS___clock_gettime50 -# if SANITIZER_WORDSIZE == 64 -# define internal_syscall_ptr __syscall -# else -# define internal_syscall_ptr syscall -# endif -#elif defined(__x86_64__) && (SANITIZER_FREEBSD || SANITIZER_MAC) +#if defined(__x86_64__) && (SANITIZER_FREEBSD || SANITIZER_MAC) # define internal_syscall __syscall -# define internal_syscall64 __syscall -# define internal_syscall_ptr __syscall # else # define internal_syscall syscall -# define internal_syscall64 syscall -# define internal_syscall_ptr syscall +#endif + #endif bool internal_iserror(uptr retval, int *rverrno) { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc index 1f05ed9b6c1c..7ab1d7641449 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc @@ -127,9 +127,6 @@ static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, #define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__) -#define internal_syscall_ptr internal_syscall -#define internal_syscall64 internal_syscall - // Helper function used to avoid cobbler errno. bool internal_iserror(uptr retval, int *rverrno) { if (retval >= (uptr)-4095) { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc index a3fdb9e60d5d..b4fd0962a9fd 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc @@ -127,9 +127,6 @@ static uptr __internal_syscall(u32 nr, u32 arg1, long arg2, long arg3, #define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__) -#define internal_syscall_ptr internal_syscall -#define internal_syscall64 internal_syscall - // Helper function used to avoid cobbler errno. bool internal_iserror(uptr retval, int *rverrno) { if (retval >= (uptr)-4095) { diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc index 327aaa80a674..9853a6a675d3 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc @@ -20,9 +20,6 @@ static uptr internal_syscall(u64 nr) { return retval; } -#define internal_syscall_ptr internal_syscall -#define internal_syscall64 internal_syscall - template <typename T1> static uptr internal_syscall(u64 nr, T1 arg1) { u64 retval; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscalls_netbsd.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscalls_netbsd.inc index 4f766100813d..75aea2760d2c 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscalls_netbsd.inc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscalls_netbsd.inc @@ -43,8 +43,8 @@ // DO NOT EDIT! THIS FILE HAS BEEN GENERATED! // // Generated with: generate_netbsd_syscalls.awk -// Generated date: 2018-03-03 -// Generated from: syscalls.master,v 1.291 2018/01/06 16:41:23 kamil Exp +// Generated date: 2018-10-30 +// Generated from: syscalls.master,v 1.293 2018/07/31 13:00:13 rjs Exp // //===----------------------------------------------------------------------===// @@ -1454,7 +1454,15 @@ PRE_SYSCALL(fpathconf)(long long fd_, long long name_) { /* Nothing to do */ } POST_SYSCALL(fpathconf)(long long res, long long fd_, long long name_) { /* Nothing to do */ } -/* syscall 193 has been skipped */ +PRE_SYSCALL(getsockopt2) +(long long s_, long long level_, long long name_, void *val_, void *avalsize_) { + /* TODO */ +} +POST_SYSCALL(getsockopt2) +(long long res, long long s_, long long level_, long long name_, void *val_, + void *avalsize_) { + /* TODO */ +} PRE_SYSCALL(getrlimit)(long long which_, void *rlp_) { PRE_WRITE(rlp_, struct_rlimit_sz); } @@ -2341,20 +2349,8 @@ POST_SYSCALL(__sigaction_sigtramp) PRE_READ(nsa_, sizeof(__sanitizer_sigaction)); } } -PRE_SYSCALL(pmc_get_info)(long long ctr_, long long op_, void *args_) { - /* TODO */ -} -POST_SYSCALL(pmc_get_info) -(long long res, long long ctr_, long long op_, void *args_) { - /* TODO */ -} -PRE_SYSCALL(pmc_control)(long long ctr_, long long op_, void *args_) { - /* TODO */ -} -POST_SYSCALL(pmc_control) -(long long res, long long ctr_, long long op_, void *args_) { - /* TODO */ -} +/* syscall 341 has been skipped */ +/* syscall 342 has been skipped */ PRE_SYSCALL(rasctl)(void *addr_, long long len_, long long op_) { /* Nothing to do */ } @@ -3695,18 +3691,18 @@ POST_SYSCALL(recvmmsg) PRE_SYSCALL(sendmmsg) (long long s_, void *mmsg_, long long vlen_, long long flags_) { struct __sanitizer_mmsghdr *mmsg = (struct __sanitizer_mmsghdr *)mmsg_; - unsigned int vlen = (vlen_ > 1024 ? 1024 : vlen_); if (mmsg) { - PRE_READ(mmsg, sizeof(struct __sanitizer_mmsghdr) * vlen); + PRE_READ(mmsg, sizeof(struct __sanitizer_mmsghdr) * + (vlen_ > 1024 ? 1024 : vlen_)); } } POST_SYSCALL(sendmmsg) (long long res, long long s_, void *mmsg_, long long vlen_, long long flags_) { struct __sanitizer_mmsghdr *mmsg = (struct __sanitizer_mmsghdr *)mmsg_; - unsigned int vlen = (vlen_ > 1024 ? 1024 : vlen_); if (res >= 0) { if (mmsg) { - POST_READ(mmsg, sizeof(struct __sanitizer_mmsghdr) * vlen); + POST_READ(mmsg, sizeof(struct __sanitizer_mmsghdr) * + (vlen_ > 1024 ? 1024 : vlen_)); } } } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_termination.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_termination.cc index 8243fc05d02e..35e4403adfb4 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_termination.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_termination.cc @@ -84,3 +84,12 @@ void NORETURN CheckFailed(const char *file, int line, const char *cond, } } // namespace __sanitizer + +using namespace __sanitizer; // NOLINT + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_set_death_callback(void (*callback)(void)) { + SetUserDieCallback(callback); +} +} // extern "C" diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cc index d9fd6549ba48..eb35cb6c5830 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cc @@ -338,4 +338,15 @@ ThreadContextBase *ThreadRegistry::QuarantinePop() { return tctx; } +void ThreadRegistry::SetThreadUserId(u32 tid, uptr user_id) { + BlockingMutexLock l(&mtx_); + CHECK_LT(tid, n_contexts_); + ThreadContextBase *tctx = threads_[tid]; + CHECK_NE(tctx, 0); + CHECK_NE(tctx->status, ThreadStatusInvalid); + CHECK_NE(tctx->status, ThreadStatusDead); + CHECK_EQ(tctx->user_id, 0); + tctx->user_id = user_id; +} + } // namespace __sanitizer diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h index b203be2f4dbc..30dc603febd4 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h @@ -122,6 +122,7 @@ class ThreadRegistry { void JoinThread(u32 tid, void *arg); void FinishThread(u32 tid); void StartThread(u32 tid, tid_t os_id, bool workerthread, void *arg); + void SetThreadUserId(u32 tid, uptr user_id); private: const ThreadContextFactory context_factory_; diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_type_traits.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_type_traits.cc new file mode 100644 index 000000000000..27fec6e1f494 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_type_traits.cc @@ -0,0 +1,21 @@ +//===-- sanitizer_type_traits.cc --------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Implements a subset of C++ type traits. This is so we can avoid depending +// on system C++ headers. +// +//===----------------------------------------------------------------------===// +#include "sanitizer_type_traits.h" + +namespace __sanitizer { + +const bool true_type::value; +const bool false_type::value; + +} // namespace __sanitizer diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_type_traits.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_type_traits.h new file mode 100644 index 000000000000..4495f2c34b71 --- /dev/null +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_type_traits.h @@ -0,0 +1,44 @@ +//===-- sanitizer_type_traits.h ---------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Implements a subset of C++ type traits. This is so we can avoid depending +// on system C++ headers. +// +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_TYPE_TRAITS_H +#define SANITIZER_TYPE_TRAITS_H + +namespace __sanitizer { + +struct true_type { + static const bool value = true; +}; + +struct false_type { + static const bool value = false; +}; + +// is_same<T, U> +// +// Type trait to compare if types are the same. +// E.g. +// +// ``` +// is_same<int,int>::value - True +// is_same<int,char>::value - False +// ``` +template <typename T, typename U> +struct is_same : public false_type {}; + +template <typename T> +struct is_same<T, T> : public true_type {}; + +} // namespace __sanitizer + +#endif diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc index 9e12c417c71d..c7a5ec86fc7c 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc @@ -97,7 +97,7 @@ uptr Unwind_GetIP(struct _Unwind_Context *ctx) { // Clear the Thumb bit. return val & ~(uptr)1; #else - return _Unwind_GetIP(ctx); + return (uptr)_Unwind_GetIP(ctx); #endif } diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc index 38e567d9a732..9a574dd23de6 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc @@ -29,6 +29,10 @@ #include "sanitizer_placement_new.h" #include "sanitizer_win_defs.h" +#if defined(PSAPI_VERSION) && PSAPI_VERSION == 1 +#pragma comment(lib, "psapi") +#endif + // A macro to tell the compiler that this part of the code cannot be reached, // if the compiler supports this feature. Since we're using this in // code that is called when terminating the process, the expansion of the @@ -733,10 +737,6 @@ bool WriteToFile(fd_t fd, const void *buff, uptr buff_size, uptr *bytes_written, } } -bool RenameFile(const char *oldpath, const char *newpath, error_t *error_p) { - UNIMPLEMENTED(); -} - uptr internal_sched_yield() { Sleep(0); return 0; @@ -1004,6 +1004,10 @@ void CheckVMASize() { // Do nothing. } +void InitializePlatformEarly() { + // Do nothing. +} + void MaybeReexec() { // No need to re-exec on Windows. } @@ -1012,11 +1016,20 @@ void CheckASLR() { // Do nothing } +void CheckMPROTECT() { + // Do nothing +} + char **GetArgv() { // FIXME: Actually implement this function. return 0; } +char **GetEnviron() { + // FIXME: Actually implement this function. + return 0; +} + pid_t StartSubprocess(const char *program, const char *const argv[], fd_t stdin_fd, fd_t stdout_fd, fd_t stderr_fd) { // FIXME: implement on this platform diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win_defs.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win_defs.h index 077ff9ccc8df..10fc2d021ee0 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win_defs.h +++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win_defs.h @@ -17,17 +17,27 @@ #if SANITIZER_WINDOWS #ifndef WINAPI -#ifdef _M_IX86 +#if defined(_M_IX86) || defined(__i386__) #define WINAPI __stdcall #else #define WINAPI #endif #endif -#if defined(_WIN64) +#if defined(_M_IX86) || defined(__i386__) +#define WIN_SYM_PREFIX "_" +#else #define WIN_SYM_PREFIX +#endif + +// For MinGW, the /export: directives contain undecorated symbols, contrary to +// link/lld-link. The GNU linker doesn't support /alternatename and /include +// though, thus lld-link in MinGW mode interprets them in the same way as +// in the default mode. +#ifdef __MINGW32__ +#define WIN_EXPORT_PREFIX #else -#define WIN_SYM_PREFIX "_" +#define WIN_EXPORT_PREFIX WIN_SYM_PREFIX #endif // Intermediate macro to ensure the parameter is expanded before stringified. @@ -62,8 +72,8 @@ __pragma(comment(linker, "/include:" WIN_SYM_PREFIX STRINGIFY(Name))) #define WIN_EXPORT(ExportedName, Name) \ - __pragma(comment(linker, "/export:" WIN_SYM_PREFIX STRINGIFY(ExportedName) \ - "=" WIN_SYM_PREFIX STRINGIFY(Name))) + __pragma(comment(linker, "/export:" WIN_EXPORT_PREFIX STRINGIFY(ExportedName)\ + "=" WIN_EXPORT_PREFIX STRINGIFY(Name))) // We cannot define weak functions on Windows, but we can use WIN_WEAK_ALIAS() // which defines an alias to a default implementation, and only works when diff --git a/contrib/compiler-rt/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt b/contrib/compiler-rt/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt index f77648d7818a..beee0acf484e 100644 --- a/contrib/compiler-rt/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt +++ b/contrib/compiler-rt/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt @@ -49,6 +49,7 @@ dup U dup2 U environ U execv U +execve U exit U fclose U fflush U @@ -65,6 +66,7 @@ getcwd U getenv U getpagesize U getpid U +getrlimit U gettimeofday U ioctl U isalpha U @@ -101,6 +103,7 @@ rand U readlink U realloc U remove U +setrlimit U setvbuf U sigfillset U sigprocmask U diff --git a/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp b/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp index 4a11bf5fcc21..fb04fb281c4b 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp +++ b/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp @@ -129,16 +129,9 @@ namespace Chunk { computeChecksum(Ptr, &NewUnpackedHeader)); } - // Nulls out a chunk header. When returning the chunk to the backend, there - // is no need to store a valid ChunkAvailable header, as this would be - // computationally expensive. Zeroing out serves the same purpose by making - // the header invalid. In the extremely rare event where 0 would be a valid - // checksum for the chunk, the state of the chunk is ChunkAvailable anyway. + // Ensure that ChunkAvailable is 0, so that if a 0 checksum is ever valid + // for a fully nulled out header, its state will be available anyway. COMPILER_CHECK(ChunkAvailable == 0); - static INLINE void eraseHeader(void *Ptr) { - const PackedHeader NullPackedHeader = 0; - atomic_store_relaxed(getAtomicHeader(Ptr), NullPackedHeader); - } // Loads and unpacks the header, verifying the checksum in the process. static INLINE @@ -185,7 +178,9 @@ struct QuarantineCallback { Chunk::loadHeader(Ptr, &Header); if (UNLIKELY(Header.State != ChunkQuarantine)) dieWithMessage("invalid chunk state when recycling address %p\n", Ptr); - Chunk::eraseHeader(Ptr); + UnpackedHeader NewHeader = Header; + NewHeader.State = ChunkAvailable; + Chunk::compareExchangeHeader(Ptr, &NewHeader, &Header); void *BackendPtr = Chunk::getBackendPtr(Ptr, &Header); if (Header.ClassId) getBackend().deallocatePrimary(Cache_, BackendPtr, Header.ClassId); @@ -264,7 +259,8 @@ struct Allocator { Quarantine.Init( static_cast<uptr>(getFlags()->QuarantineSizeKb) << 10, static_cast<uptr>(getFlags()->ThreadLocalQuarantineSizeKb) << 10); - QuarantineChunksUpToSize = getFlags()->QuarantineChunksUpToSize; + QuarantineChunksUpToSize = (Quarantine.GetCacheSize() == 0) ? 0 : + getFlags()->QuarantineChunksUpToSize; DeallocationTypeMismatch = getFlags()->DeallocationTypeMismatch; DeleteSizeMismatch = getFlags()->DeleteSizeMismatch; ZeroContents = getFlags()->ZeroContents; @@ -389,10 +385,11 @@ struct Allocator { // quarantine chunk size threshold. void quarantineOrDeallocateChunk(void *Ptr, UnpackedHeader *Header, uptr Size) { - const bool BypassQuarantine = (Quarantine.GetCacheSize() == 0) || - (Size > QuarantineChunksUpToSize); + const bool BypassQuarantine = !Size || (Size > QuarantineChunksUpToSize); if (BypassQuarantine) { - Chunk::eraseHeader(Ptr); + UnpackedHeader NewHeader = *Header; + NewHeader.State = ChunkAvailable; + Chunk::compareExchangeHeader(Ptr, &NewHeader, Header); void *BackendPtr = Chunk::getBackendPtr(Ptr, Header); if (Header->ClassId) { bool UnlockRequired; @@ -675,7 +672,7 @@ void *scudoValloc(uptr Size) { } void *scudoPvalloc(uptr Size) { - uptr PageSize = GetPageSizeCached(); + const uptr PageSize = GetPageSizeCached(); if (UNLIKELY(CheckForPvallocOverflow(Size, PageSize))) { errno = ENOMEM; if (Instance.canReturnNull()) diff --git a/contrib/compiler-rt/lib/scudo/scudo_allocator.h b/contrib/compiler-rt/lib/scudo/scudo_allocator.h index 0002b4a44b78..814bb08ab013 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_allocator.h +++ b/contrib/compiler-rt/lib/scudo/scudo_allocator.h @@ -81,6 +81,7 @@ struct AP64 { typedef NoOpMapUnmapCallback MapUnmapCallback; static const uptr kFlags = SizeClassAllocator64FlagMasks::kRandomShuffleChunks; + using AddressSpaceView = LocalAddressSpaceView; }; typedef SizeClassAllocator64<AP64> PrimaryT; #else @@ -96,7 +97,8 @@ struct AP32 { static const uptr kMetadataSize = 0; typedef __scudo::SizeClassMap SizeClassMap; static const uptr kRegionSizeLog = RegionSizeLog; - typedef __scudo::ByteMap ByteMap; + using AddressSpaceView = LocalAddressSpaceView; + using ByteMap = __scudo::ByteMap; typedef NoOpMapUnmapCallback MapUnmapCallback; static const uptr kFlags = SizeClassAllocator32FlagMasks::kRandomShuffleChunks | diff --git a/contrib/compiler-rt/lib/scudo/scudo_malloc.cpp b/contrib/compiler-rt/lib/scudo/scudo_malloc.cpp index 91a77b365823..eef776809bd2 100644 --- a/contrib/compiler-rt/lib/scudo/scudo_malloc.cpp +++ b/contrib/compiler-rt/lib/scudo/scudo_malloc.cpp @@ -79,7 +79,7 @@ INTERCEPTOR_ATTRIBUTE size_t malloc_usable_size(void *ptr) { #if SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO INTERCEPTOR_ATTRIBUTE int mallopt(int cmd, int value) { - return -1; + return 0; } #endif } // extern "C" diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_debugging.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_debugging.cc index 9e5465d37557..067aeef97c0e 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_debugging.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_debugging.cc @@ -20,31 +20,37 @@ using namespace __tsan; static const char *ReportTypeDescription(ReportType typ) { - if (typ == ReportTypeRace) return "data-race"; - if (typ == ReportTypeVptrRace) return "data-race-vptr"; - if (typ == ReportTypeUseAfterFree) return "heap-use-after-free"; - if (typ == ReportTypeVptrUseAfterFree) return "heap-use-after-free-vptr"; - if (typ == ReportTypeExternalRace) return "external-race"; - if (typ == ReportTypeThreadLeak) return "thread-leak"; - if (typ == ReportTypeMutexDestroyLocked) return "locked-mutex-destroy"; - if (typ == ReportTypeMutexDoubleLock) return "mutex-double-lock"; - if (typ == ReportTypeMutexInvalidAccess) return "mutex-invalid-access"; - if (typ == ReportTypeMutexBadUnlock) return "mutex-bad-unlock"; - if (typ == ReportTypeMutexBadReadLock) return "mutex-bad-read-lock"; - if (typ == ReportTypeMutexBadReadUnlock) return "mutex-bad-read-unlock"; - if (typ == ReportTypeSignalUnsafe) return "signal-unsafe-call"; - if (typ == ReportTypeErrnoInSignal) return "errno-in-signal-handler"; - if (typ == ReportTypeDeadlock) return "lock-order-inversion"; - return ""; + switch (typ) { + case ReportTypeRace: return "data-race"; + case ReportTypeVptrRace: return "data-race-vptr"; + case ReportTypeUseAfterFree: return "heap-use-after-free"; + case ReportTypeVptrUseAfterFree: return "heap-use-after-free-vptr"; + case ReportTypeExternalRace: return "external-race"; + case ReportTypeThreadLeak: return "thread-leak"; + case ReportTypeMutexDestroyLocked: return "locked-mutex-destroy"; + case ReportTypeMutexDoubleLock: return "mutex-double-lock"; + case ReportTypeMutexInvalidAccess: return "mutex-invalid-access"; + case ReportTypeMutexBadUnlock: return "mutex-bad-unlock"; + case ReportTypeMutexBadReadLock: return "mutex-bad-read-lock"; + case ReportTypeMutexBadReadUnlock: return "mutex-bad-read-unlock"; + case ReportTypeSignalUnsafe: return "signal-unsafe-call"; + case ReportTypeErrnoInSignal: return "errno-in-signal-handler"; + case ReportTypeDeadlock: return "lock-order-inversion"; + // No default case so compiler warns us if we miss one + } + UNREACHABLE("missing case"); } static const char *ReportLocationTypeDescription(ReportLocationType typ) { - if (typ == ReportLocationGlobal) return "global"; - if (typ == ReportLocationHeap) return "heap"; - if (typ == ReportLocationStack) return "stack"; - if (typ == ReportLocationTLS) return "tls"; - if (typ == ReportLocationFD) return "fd"; - return ""; + switch (typ) { + case ReportLocationGlobal: return "global"; + case ReportLocationHeap: return "heap"; + case ReportLocationStack: return "stack"; + case ReportLocationTLS: return "tls"; + case ReportLocationFD: return "fd"; + // No default case so compiler warns us if we miss one + } + UNREACHABLE("missing case"); } static void CopyTrace(SymbolizedStack *first_frame, void **trace, diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.cc index 89e22a132786..877fc8b819fd 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.cc @@ -61,8 +61,7 @@ void InitializeFlags(Flags *f, const char *env) { // Does not work as expected for Go: runtime handles SIGABRT and crashes. cf.abort_on_error = false; // Go does not have mutexes. - } else { - cf.detect_deadlocks = true; + cf.detect_deadlocks = false; } cf.print_suppressions = false; cf.stack_trace_format = " #%n %f %S %M"; diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.inc b/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.inc index e9b3e35f07e5..d3b678fdd151 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.inc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.inc @@ -77,8 +77,6 @@ TSAN_FLAG(int, io_sync, 1, TSAN_FLAG(bool, die_after_fork, true, "Die after multi-threaded fork if the child creates new threads.") TSAN_FLAG(const char *, suppressions, "", "Suppressions file name.") -TSAN_FLAG(bool, ignore_interceptors_accesses, false, - "Ignore reads and writes from all interceptors.") TSAN_FLAG(bool, ignore_noninstrumented_modules, SANITIZER_MAC ? true : false, "Interceptors should only detect races when called from instrumented " "modules.") diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc index 901997b3e85d..9e49dfe5d582 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc @@ -42,17 +42,15 @@ using namespace __tsan; // NOLINT #if SANITIZER_NETBSD #define dirfd(dirp) (*(int *)(dirp)) -#define fileno_unlocked fileno +#define fileno_unlocked(fp) \ + (((__sanitizer_FILE*)fp)->_file == -1 ? -1 : \ + (int)(unsigned short)(((__sanitizer_FILE*)fp)->_file)) // NOLINT -#if _LP64 -#define __sF_size 152 -#else -#define __sF_size 88 -#endif - -#define stdout ((char*)&__sF + (__sF_size * 1)) -#define stderr ((char*)&__sF + (__sF_size * 2)) +#define stdout ((__sanitizer_FILE*)&__sF[1]) +#define stderr ((__sanitizer_FILE*)&__sF[2]) +#define nanosleep __nanosleep50 +#define vfork __vfork14 #endif #if SANITIZER_ANDROID @@ -94,8 +92,8 @@ DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size) DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) extern "C" void *pthread_self(); extern "C" void _exit(int status); -extern "C" int fileno_unlocked(void *stream); #if !SANITIZER_NETBSD +extern "C" int fileno_unlocked(void *stream); extern "C" int dirfd(void *dirp); #endif #if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_NETBSD @@ -226,6 +224,16 @@ void InitializeLibIgnore() { libignore()->OnLibraryLoaded(0); } +// The following two hooks can be used by for cooperative scheduling when +// locking. +#ifdef TSAN_EXTERNAL_HOOKS +void OnPotentiallyBlockingRegionBegin(); +void OnPotentiallyBlockingRegionEnd(); +#else +SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionBegin() {} +SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionEnd() {} +#endif + } // namespace __tsan static ThreadSignalContext *SigCtx(ThreadState *thr) { @@ -246,8 +254,7 @@ ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname, if (!thr_->ignore_interceptors) FuncEntry(thr, pc); DPrintf("#%d: intercept %s()\n", thr_->tid, fname); ignoring_ = - !thr_->in_ignored_lib && (flags()->ignore_interceptors_accesses || - libignore()->IsIgnored(pc, &in_ignored_lib_)); + !thr_->in_ignored_lib && libignore()->IsIgnored(pc, &in_ignored_lib_); EnableIgnores(); } @@ -508,7 +515,8 @@ static void LongJmp(ThreadState *thr, uptr *env) { uptr mangled_sp = env[6]; #elif SANITIZER_MAC # ifdef __aarch64__ - uptr mangled_sp = env[13]; + uptr mangled_sp = + (GetMacosVersion() >= MACOS_VERSION_MOJAVE) ? env[12] : env[13]; # else uptr mangled_sp = env[2]; # endif @@ -863,6 +871,8 @@ TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) { // Used in thread-safe function static initialization. STDCXX_INTERCEPTOR(int, __cxa_guard_acquire, atomic_uint32_t *g) { SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g); + OnPotentiallyBlockingRegionBegin(); + auto on_exit = at_scope_exit(&OnPotentiallyBlockingRegionEnd); for (;;) { u32 cmp = atomic_load(g, memory_order_acquire); if (cmp == 0) { @@ -1041,6 +1051,35 @@ TSAN_INTERCEPTOR(int, pthread_detach, void *th) { return res; } +#if SANITIZER_LINUX +TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void *th, void **ret) { + SCOPED_TSAN_INTERCEPTOR(pthread_tryjoin_np, th, ret); + int tid = ThreadTid(thr, pc, (uptr)th); + ThreadIgnoreBegin(thr, pc); + int res = REAL(pthread_tryjoin_np)(th, ret); + ThreadIgnoreEnd(thr, pc); + if (res == 0) + ThreadJoin(thr, pc, tid); + else + ThreadNotJoined(thr, pc, tid, (uptr)th); + return res; +} + +TSAN_INTERCEPTOR(int, pthread_timedjoin_np, void *th, void **ret, + const struct timespec *abstime) { + SCOPED_TSAN_INTERCEPTOR(pthread_timedjoin_np, th, ret, abstime); + int tid = ThreadTid(thr, pc, (uptr)th); + ThreadIgnoreBegin(thr, pc); + int res = BLOCK_REAL(pthread_timedjoin_np)(th, ret, abstime); + ThreadIgnoreEnd(thr, pc); + if (res == 0) + ThreadJoin(thr, pc, tid); + else + ThreadNotJoined(thr, pc, tid, (uptr)th); + return res; +} +#endif + // Problem: // NPTL implementation of pthread_cond has 2 versions (2.2.5 and 2.3.2). // pthread_cond_t has different size in the different versions. @@ -2210,7 +2249,8 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, (void) ctx; #define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \ - Acquire(thr, pc, File2addr(path)); \ + if (path) \ + Acquire(thr, pc, File2addr(path)); \ if (file) { \ int fd = fileno_unlocked(file); \ if (fd >= 0) FdFileCreate(thr, pc, fd); \ @@ -2559,6 +2599,8 @@ TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_wrlock, void *m) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_trywrlock, void *m) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_unlock, void *m) TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(int, once, void *o, void (*f)()) +TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(int, sigsetmask, sigmask, int a, void *b, + void *c) namespace __tsan { @@ -2635,6 +2677,10 @@ void InitializeInterceptors() { TSAN_INTERCEPT(pthread_create); TSAN_INTERCEPT(pthread_join); TSAN_INTERCEPT(pthread_detach); + #if SANITIZER_LINUX + TSAN_INTERCEPT(pthread_tryjoin_np); + TSAN_INTERCEPT(pthread_timedjoin_np); + #endif TSAN_INTERCEPT_VER(pthread_cond_init, PTHREAD_ABI_BASE); TSAN_INTERCEPT_VER(pthread_cond_signal, PTHREAD_ABI_BASE); @@ -2768,6 +2814,7 @@ void InitializeInterceptors() { TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_trywrlock); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_unlock); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(once); + TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(sigsetmask); FdInit(); } diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h index 959a39465e33..763b46b88c8f 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h @@ -56,9 +56,13 @@ LibIgnore *libignore(); # define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...) \ TSAN_INTERCEPTOR(ret, __libc_thr_##func, __VA_ARGS__) \ ALIAS(WRAPPER_NAME(pthread_##func)); +# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(ret, func, func2, ...) \ + TSAN_INTERCEPTOR(ret, __libc_thr_##func, __VA_ARGS__) \ + ALIAS(WRAPPER_NAME(pthread_##func2)); #else # define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...) # define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...) +# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(ret, func, func2, ...) #endif #endif // TSAN_INTERCEPTORS_H diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc index b58e6b7071cf..579c4d0c0b81 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc @@ -19,8 +19,10 @@ #include "tsan_interceptors.h" #include "tsan_interface.h" #include "tsan_interface_ann.h" +#include "sanitizer_common/sanitizer_addrhashmap.h" #include <libkern/OSAtomic.h> +#include <objc/objc-sync.h> #if defined(__has_include) && __has_include(<xpc/xpc.h>) #include <xpc/xpc.h> @@ -294,41 +296,62 @@ TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) { #endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>) -// Is the Obj-C object a tagged pointer (i.e. isn't really a valid pointer and -// contains data in the pointers bits instead)? -static bool IsTaggedObjCPointer(void *obj) { +// Determines whether the Obj-C object pointer is a tagged pointer. Tagged +// pointers encode the object data directly in their pointer bits and do not +// have an associated memory allocation. The Obj-C runtime uses tagged pointers +// to transparently optimize small objects. +static bool IsTaggedObjCPointer(id obj) { const uptr kPossibleTaggedBits = 0x8000000000000001ull; return ((uptr)obj & kPossibleTaggedBits) != 0; } -// Return an address on which we can synchronize (Acquire and Release) for a -// Obj-C tagged pointer (which is not a valid pointer). Ideally should be a -// derived address from 'obj', but for now just return the same global address. -// TODO(kubamracek): Return different address for different pointers. -static uptr SyncAddressForTaggedPointer(void *obj) { - (void)obj; - static u64 addr; - return (uptr)&addr; +// Returns an address which can be used to inform TSan about synchronization +// points (MutexLock/Unlock). The TSan infrastructure expects this to be a valid +// address in the process space. We do a small allocation here to obtain a +// stable address (the array backing the hash map can change). The memory is +// never free'd (leaked) and allocation and locking are slow, but this code only +// runs for @synchronized with tagged pointers, which is very rare. +static uptr GetOrCreateSyncAddress(uptr addr, ThreadState *thr, uptr pc) { + typedef AddrHashMap<uptr, 5> Map; + static Map Addresses; + Map::Handle h(&Addresses, addr); + if (h.created()) { + ThreadIgnoreBegin(thr, pc); + *h = (uptr) user_alloc(thr, pc, /*size=*/1); + ThreadIgnoreEnd(thr, pc); + } + return *h; } -// Address on which we can synchronize for an Objective-C object. Supports -// tagged pointers. -static uptr SyncAddressForObjCObject(void *obj) { - if (IsTaggedObjCPointer(obj)) return SyncAddressForTaggedPointer(obj); +// Returns an address on which we can synchronize given an Obj-C object pointer. +// For normal object pointers, this is just the address of the object in memory. +// Tagged pointers are not backed by an actual memory allocation, so we need to +// synthesize a valid address. +static uptr SyncAddressForObjCObject(id obj, ThreadState *thr, uptr pc) { + if (IsTaggedObjCPointer(obj)) + return GetOrCreateSyncAddress((uptr)obj, thr, pc); return (uptr)obj; } -TSAN_INTERCEPTOR(int, objc_sync_enter, void *obj) { +TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) { SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj); + if (!obj) return REAL(objc_sync_enter)(obj); + uptr addr = SyncAddressForObjCObject(obj, thr, pc); + MutexPreLock(thr, pc, addr, MutexFlagWriteReentrant); int result = REAL(objc_sync_enter)(obj); - if (obj) Acquire(thr, pc, SyncAddressForObjCObject(obj)); + CHECK_EQ(result, OBJC_SYNC_SUCCESS); + MutexPostLock(thr, pc, addr, MutexFlagWriteReentrant); return result; } -TSAN_INTERCEPTOR(int, objc_sync_exit, void *obj) { - SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj); - if (obj) Release(thr, pc, SyncAddressForObjCObject(obj)); - return REAL(objc_sync_exit)(obj); +TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) { + SCOPED_TSAN_INTERCEPTOR(objc_sync_exit, obj); + if (!obj) return REAL(objc_sync_exit)(obj); + uptr addr = SyncAddressForObjCObject(obj, thr, pc); + MutexUnlock(thr, pc, addr); + int result = REAL(objc_sync_exit)(obj); + if (result != OBJC_SYNC_SUCCESS) MutexInvalidAccess(thr, pc, addr); + return result; } // On macOS, libc++ is always linked dynamically, so intercepting works the diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc index d6c1ca6622f4..df22888b3f1c 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc @@ -185,11 +185,8 @@ static void invoke_and_release_block(void *param) { TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \ DISPATCH_NOESCAPE dispatch_block_t block) { \ SCOPED_TSAN_INTERCEPTOR(name, q, block); \ - SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ - dispatch_block_t heap_block = Block_copy(block); \ - SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ tsan_block_context_t new_context = { \ - q, heap_block, &invoke_and_release_block, false, true, barrier, 0}; \ + q, block, &invoke_block, false, true, barrier, 0}; \ Release(thr, pc, (uptr)&new_context); \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ REAL(name##_f)(q, &new_context, dispatch_callback_wrap); \ diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h index 6b3e6bac27cb..8303c241809b 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h @@ -458,6 +458,32 @@ struct Mapping47 { static const uptr kAppMemEnd = 0x00e000000000ull; }; +#elif SANITIZER_GO && defined(__aarch64__) + +/* Go on linux/aarch64 (48-bit VMA) +0000 0000 1000 - 0000 1000 0000: executable +0000 1000 0000 - 00c0 0000 0000: - +00c0 0000 0000 - 00e0 0000 0000: heap +00e0 0000 0000 - 2000 0000 0000: - +2000 0000 0000 - 3000 0000 0000: shadow +3000 0000 0000 - 3000 0000 0000: - +3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects) +4000 0000 0000 - 6000 0000 0000: - +6000 0000 0000 - 6200 0000 0000: traces +6200 0000 0000 - 8000 0000 0000: - +*/ + +struct Mapping { + static const uptr kMetaShadowBeg = 0x300000000000ull; + static const uptr kMetaShadowEnd = 0x400000000000ull; + static const uptr kTraceMemBeg = 0x600000000000ull; + static const uptr kTraceMemEnd = 0x620000000000ull; + static const uptr kShadowBeg = 0x200000000000ull; + static const uptr kShadowEnd = 0x300000000000ull; + static const uptr kAppMemBeg = 0x000000001000ull; + static const uptr kAppMemEnd = 0x00e000000000ull; +}; + // Indicates the runtime will define the memory regions at runtime. #define TSAN_RUNTIME_VMA 1 @@ -523,7 +549,7 @@ uptr MappingImpl(void) { template<int Type> uptr MappingArchImpl(void) { -#if defined(__aarch64__) && !defined(__APPLE__) +#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO switch (vmaSize) { case 39: return MappingImpl<Mapping39, Type>(); case 42: return MappingImpl<Mapping42, Type>(); @@ -680,7 +706,7 @@ bool IsAppMemImpl(uptr mem) { ALWAYS_INLINE bool IsAppMem(uptr mem) { -#if defined(__aarch64__) && !defined(__APPLE__) +#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO switch (vmaSize) { case 39: return IsAppMemImpl<Mapping39>(mem); case 42: return IsAppMemImpl<Mapping42>(mem); @@ -711,7 +737,7 @@ bool IsShadowMemImpl(uptr mem) { ALWAYS_INLINE bool IsShadowMem(uptr mem) { -#if defined(__aarch64__) && !defined(__APPLE__) +#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO switch (vmaSize) { case 39: return IsShadowMemImpl<Mapping39>(mem); case 42: return IsShadowMemImpl<Mapping42>(mem); @@ -742,7 +768,7 @@ bool IsMetaMemImpl(uptr mem) { ALWAYS_INLINE bool IsMetaMem(uptr mem) { -#if defined(__aarch64__) && !defined(__APPLE__) +#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO switch (vmaSize) { case 39: return IsMetaMemImpl<Mapping39>(mem); case 42: return IsMetaMemImpl<Mapping42>(mem); @@ -783,7 +809,7 @@ uptr MemToShadowImpl(uptr x) { ALWAYS_INLINE uptr MemToShadow(uptr x) { -#if defined(__aarch64__) && !defined(__APPLE__) +#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO switch (vmaSize) { case 39: return MemToShadowImpl<Mapping39>(x); case 42: return MemToShadowImpl<Mapping42>(x); @@ -826,7 +852,7 @@ u32 *MemToMetaImpl(uptr x) { ALWAYS_INLINE u32 *MemToMeta(uptr x) { -#if defined(__aarch64__) && !defined(__APPLE__) +#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO switch (vmaSize) { case 39: return MemToMetaImpl<Mapping39>(x); case 42: return MemToMetaImpl<Mapping42>(x); @@ -882,7 +908,7 @@ uptr ShadowToMemImpl(uptr s) { ALWAYS_INLINE uptr ShadowToMem(uptr s) { -#if defined(__aarch64__) && !defined(__APPLE__) +#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO switch (vmaSize) { case 39: return ShadowToMemImpl<Mapping39>(s); case 42: return ShadowToMemImpl<Mapping42>(s); @@ -921,7 +947,7 @@ uptr GetThreadTraceImpl(int tid) { ALWAYS_INLINE uptr GetThreadTrace(int tid) { -#if defined(__aarch64__) && !defined(__APPLE__) +#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO switch (vmaSize) { case 39: return GetThreadTraceImpl<Mapping39>(tid); case 42: return GetThreadTraceImpl<Mapping42>(tid); @@ -955,7 +981,7 @@ uptr GetThreadTraceHeaderImpl(int tid) { ALWAYS_INLINE uptr GetThreadTraceHeader(int tid) { -#if defined(__aarch64__) && !defined(__APPLE__) +#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO switch (vmaSize) { case 39: return GetThreadTraceHeaderImpl<Mapping39>(tid); case 42: return GetThreadTraceHeaderImpl<Mapping42>(tid); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc index de989b780883..d2ce60709067 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc @@ -212,11 +212,19 @@ void InitializePlatformEarly() { vmaSize = (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1); #if defined(__aarch64__) +# if !SANITIZER_GO if (vmaSize != 39 && vmaSize != 42 && vmaSize != 48) { Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); Printf("FATAL: Found %zd - Supported 39, 42 and 48\n", vmaSize); Die(); } +#else + if (vmaSize != 48) { + Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); + Printf("FATAL: Found %zd - Supported 48\n", vmaSize); + Die(); + } +#endif #elif defined(__powerpc64__) # if !SANITIZER_GO if (vmaSize != 44 && vmaSize != 46 && vmaSize != 47) { diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc index f8d7324b763c..7e3a47387b6d 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc @@ -240,6 +240,9 @@ void InitializePlatformEarly() { #endif } +static const uptr kPthreadSetjmpXorKeySlot = 0x7; +extern "C" uptr __tsan_darwin_setjmp_xor_key = 0; + void InitializePlatform() { DisableCoreDumperIfNecessary(); #if !SANITIZER_GO @@ -251,6 +254,11 @@ void InitializePlatform() { prev_pthread_introspection_hook = pthread_introspection_hook_install(&my_pthread_introspection_hook); #endif + + if (GetMacosVersion() >= MACOS_VERSION_MOJAVE) { + __tsan_darwin_setjmp_xor_key = + (uptr)pthread_getspecific(kPthreadSetjmpXorKeySlot); + } } #if !SANITIZER_GO diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc index af47076968d3..629c3e9337ab 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc @@ -77,39 +77,42 @@ const char *thread_name(char *buf, int tid) { } static const char *ReportTypeString(ReportType typ, uptr tag) { - if (typ == ReportTypeRace) - return "data race"; - if (typ == ReportTypeVptrRace) - return "data race on vptr (ctor/dtor vs virtual call)"; - if (typ == ReportTypeUseAfterFree) - return "heap-use-after-free"; - if (typ == ReportTypeVptrUseAfterFree) - return "heap-use-after-free (virtual call vs free)"; - if (typ == ReportTypeExternalRace) { - const char *str = GetReportHeaderFromTag(tag); - return str ? str : "race on external object"; + switch (typ) { + case ReportTypeRace: + return "data race"; + case ReportTypeVptrRace: + return "data race on vptr (ctor/dtor vs virtual call)"; + case ReportTypeUseAfterFree: + return "heap-use-after-free"; + case ReportTypeVptrUseAfterFree: + return "heap-use-after-free (virtual call vs free)"; + case ReportTypeExternalRace: { + const char *str = GetReportHeaderFromTag(tag); + return str ? str : "race on external object"; + } + case ReportTypeThreadLeak: + return "thread leak"; + case ReportTypeMutexDestroyLocked: + return "destroy of a locked mutex"; + case ReportTypeMutexDoubleLock: + return "double lock of a mutex"; + case ReportTypeMutexInvalidAccess: + return "use of an invalid mutex (e.g. uninitialized or destroyed)"; + case ReportTypeMutexBadUnlock: + return "unlock of an unlocked mutex (or by a wrong thread)"; + case ReportTypeMutexBadReadLock: + return "read lock of a write locked mutex"; + case ReportTypeMutexBadReadUnlock: + return "read unlock of a write locked mutex"; + case ReportTypeSignalUnsafe: + return "signal-unsafe call inside of a signal"; + case ReportTypeErrnoInSignal: + return "signal handler spoils errno"; + case ReportTypeDeadlock: + return "lock-order-inversion (potential deadlock)"; + // No default case so compiler warns us if we miss one } - if (typ == ReportTypeThreadLeak) - return "thread leak"; - if (typ == ReportTypeMutexDestroyLocked) - return "destroy of a locked mutex"; - if (typ == ReportTypeMutexDoubleLock) - return "double lock of a mutex"; - if (typ == ReportTypeMutexInvalidAccess) - return "use of an invalid mutex (e.g. uninitialized or destroyed)"; - if (typ == ReportTypeMutexBadUnlock) - return "unlock of an unlocked mutex (or by a wrong thread)"; - if (typ == ReportTypeMutexBadReadLock) - return "read lock of a write locked mutex"; - if (typ == ReportTypeMutexBadReadUnlock) - return "read unlock of a write locked mutex"; - if (typ == ReportTypeSignalUnsafe) - return "signal-unsafe call inside of a signal"; - if (typ == ReportTypeErrnoInSignal) - return "signal handler spoils errno"; - if (typ == ReportTypeDeadlock) - return "lock-order-inversion (potential deadlock)"; - return ""; + UNREACHABLE("missing case"); } #if SANITIZER_MAC diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc index 5841222927b3..f038e9682d89 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc @@ -359,7 +359,9 @@ void Initialize(ThreadState *thr) { CheckASLR(); InitializeFlags(&ctx->flags, options); AvoidCVE_2016_2143(); - InitializePlatformEarly(); + __sanitizer::InitializePlatformEarly(); + __tsan::InitializePlatformEarly(); + #if !SANITIZER_GO // Re-exec ourselves if we need to set additional env or command line args. MaybeReexec(); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h index 523b69aaa6bc..3410be294502 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h @@ -59,15 +59,16 @@ struct MapUnmapCallback; static const uptr kAllocatorRegionSizeLog = 20; static const uptr kAllocatorNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kAllocatorRegionSizeLog; -typedef TwoLevelByteMap<(kAllocatorNumRegions >> 12), 1 << 12, - MapUnmapCallback> ByteMap; +using ByteMap = TwoLevelByteMap<(kAllocatorNumRegions >> 12), 1 << 12, + LocalAddressSpaceView, MapUnmapCallback>; struct AP32 { static const uptr kSpaceBeg = 0; static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; static const uptr kMetadataSize = 0; typedef __sanitizer::CompactSizeClassMap SizeClassMap; static const uptr kRegionSizeLog = kAllocatorRegionSizeLog; - typedef __tsan::ByteMap ByteMap; + using AddressSpaceView = LocalAddressSpaceView; + using ByteMap = __tsan::ByteMap; typedef __tsan::MapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; }; @@ -80,6 +81,7 @@ struct AP64 { // Allocator64 parameters. Deliberately using a short name. typedef DefaultSizeClassMap SizeClassMap; typedef __tsan::MapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; + using AddressSpaceView = LocalAddressSpaceView; }; typedef SizeClassAllocator64<AP64> PrimaryAllocator; #endif @@ -772,6 +774,7 @@ void ThreadFinalize(ThreadState *thr); void ThreadSetName(ThreadState *thr, const char *name); int ThreadCount(ThreadState *thr); void ProcessPendingSignals(ThreadState *thr); +void ThreadNotJoined(ThreadState *thr, uptr pc, int tid, uptr uid); Processor *ProcCreate(); void ProcDestroy(Processor *proc); diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S index f1e4dbb50fbb..31999cf346a9 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S @@ -120,8 +120,10 @@ ASM_SYMBOL_INTERCEPTOR(setjmp): add x0, x29, 32 eor x1, x2, x0 #else + adrp x2, ___tsan_darwin_setjmp_xor_key@page + ldr x2, [x2, ___tsan_darwin_setjmp_xor_key@pageoff] add x0, x29, 32 - mov x1, x0 + eor x1, x2, x0 #endif // call tsan interceptor @@ -178,8 +180,10 @@ ASM_SYMBOL_INTERCEPTOR(_setjmp): add x0, x29, 32 eor x1, x2, x0 #else + adrp x2, ___tsan_darwin_setjmp_xor_key@page + ldr x2, [x2, ___tsan_darwin_setjmp_xor_key@pageoff] add x0, x29, 32 - mov x1, x0 + eor x1, x2, x0 #endif // call tsan interceptor @@ -238,8 +242,10 @@ ASM_SYMBOL_INTERCEPTOR(sigsetjmp): add x0, x29, 32 eor x1, x2, x0 #else + adrp x2, ___tsan_darwin_setjmp_xor_key@page + ldr x2, [x2, ___tsan_darwin_setjmp_xor_key@pageoff] add x0, x29, 32 - mov x1, x0 + eor x1, x2, x0 #endif // call tsan interceptor diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S index 8af61bf0e892..34ef51c2a725 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S @@ -196,6 +196,7 @@ ASM_SYMBOL_INTERCEPTOR(setjmp): #elif defined(__APPLE__) lea 16(%rsp), %rdi mov %rdi, %rsi + xorq ___tsan_darwin_setjmp_xor_key(%rip), %rsi #elif defined(__linux__) lea 16(%rsp), %rdi mov %rdi, %rsi @@ -244,6 +245,7 @@ ASM_SYMBOL_INTERCEPTOR(_setjmp): #elif defined(__APPLE__) lea 16(%rsp), %rdi mov %rdi, %rsi + xorq ___tsan_darwin_setjmp_xor_key(%rip), %rsi #elif defined(__linux__) lea 16(%rsp), %rdi mov %rdi, %rsi @@ -299,6 +301,7 @@ ASM_SYMBOL_INTERCEPTOR(sigsetjmp): #elif defined(__APPLE__) lea 32(%rsp), %rdi mov %rdi, %rsi + xorq ___tsan_darwin_setjmp_xor_key(%rip), %rsi #elif defined(__linux__) lea 32(%rsp), %rdi mov %rdi, %rsi diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc index e4d65b9a909b..766a0f5a505f 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc @@ -312,6 +312,12 @@ void ThreadDetach(ThreadState *thr, uptr pc, int tid) { ctx->thread_registry->DetachThread(tid, thr); } +void ThreadNotJoined(ThreadState *thr, uptr pc, int tid, uptr uid) { + CHECK_GT(tid, 0); + CHECK_LT(tid, kMaxTid); + ctx->thread_registry->SetThreadUserId(tid, uid); +} + void ThreadSetName(ThreadState *thr, const char *name) { ctx->thread_registry->SetThreadName(thr->tid, name); } diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_suppressions.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_suppressions.cc index be38f331ad96..6df074118a10 100644 --- a/contrib/compiler-rt/lib/tsan/rtl/tsan_suppressions.cc +++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_suppressions.cc @@ -66,38 +66,30 @@ SuppressionContext *Suppressions() { } static const char *conv(ReportType typ) { - if (typ == ReportTypeRace) - return kSuppressionRace; - else if (typ == ReportTypeVptrRace) - return kSuppressionRace; - else if (typ == ReportTypeUseAfterFree) - return kSuppressionRace; - else if (typ == ReportTypeVptrUseAfterFree) - return kSuppressionRace; - else if (typ == ReportTypeExternalRace) - return kSuppressionRace; - else if (typ == ReportTypeThreadLeak) - return kSuppressionThread; - else if (typ == ReportTypeMutexDestroyLocked) - return kSuppressionMutex; - else if (typ == ReportTypeMutexDoubleLock) - return kSuppressionMutex; - else if (typ == ReportTypeMutexInvalidAccess) - return kSuppressionMutex; - else if (typ == ReportTypeMutexBadUnlock) - return kSuppressionMutex; - else if (typ == ReportTypeMutexBadReadLock) - return kSuppressionMutex; - else if (typ == ReportTypeMutexBadReadUnlock) - return kSuppressionMutex; - else if (typ == ReportTypeSignalUnsafe) - return kSuppressionSignal; - else if (typ == ReportTypeErrnoInSignal) - return kSuppressionNone; - else if (typ == ReportTypeDeadlock) - return kSuppressionDeadlock; - Printf("ThreadSanitizer: unknown report type %d\n", typ); - Die(); + switch (typ) { + case ReportTypeRace: + case ReportTypeVptrRace: + case ReportTypeUseAfterFree: + case ReportTypeVptrUseAfterFree: + case ReportTypeExternalRace: + return kSuppressionRace; + case ReportTypeThreadLeak: + return kSuppressionThread; + case ReportTypeMutexDestroyLocked: + case ReportTypeMutexDoubleLock: + case ReportTypeMutexInvalidAccess: + case ReportTypeMutexBadUnlock: + case ReportTypeMutexBadReadLock: + case ReportTypeMutexBadReadUnlock: + return kSuppressionMutex; + case ReportTypeSignalUnsafe: + case ReportTypeErrnoInSignal: + return kSuppressionSignal; + case ReportTypeDeadlock: + return kSuppressionDeadlock; + // No default case so compiler warns us if we miss one + } + UNREACHABLE("missing case"); } static uptr IsSuppressed(const char *stype, const AddressInfo &info, diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_checks.inc b/contrib/compiler-rt/lib/ubsan/ubsan_checks.inc index 5a7bdec2df12..ea82f89e129a 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_checks.inc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_checks.inc @@ -21,6 +21,7 @@ UBSAN_CHECK(GenericUB, "undefined-behavior", "undefined") UBSAN_CHECK(NullPointerUse, "null-pointer-use", "null") UBSAN_CHECK(PointerOverflow, "pointer-overflow", "pointer-overflow") UBSAN_CHECK(MisalignedPointerUse, "misaligned-pointer-use", "alignment") +UBSAN_CHECK(AlignmentAssumption, "alignment-assumption", "alignment") UBSAN_CHECK(InsufficientObjectSize, "insufficient-object-size", "object-size") UBSAN_CHECK(SignedIntegerOverflow, "signed-integer-overflow", "signed-integer-overflow") @@ -30,8 +31,18 @@ UBSAN_CHECK(IntegerDivideByZero, "integer-divide-by-zero", "integer-divide-by-zero") UBSAN_CHECK(FloatDivideByZero, "float-divide-by-zero", "float-divide-by-zero") UBSAN_CHECK(InvalidBuiltin, "invalid-builtin-use", "invalid-builtin-use") -UBSAN_CHECK(ImplicitIntegerTruncation, "implicit-integer-truncation", - "implicit-integer-truncation") +UBSAN_CHECK(ImplicitUnsignedIntegerTruncation, + "implicit-unsigned-integer-truncation", + "implicit-unsigned-integer-truncation") +UBSAN_CHECK(ImplicitSignedIntegerTruncation, + "implicit-signed-integer-truncation", + "implicit-signed-integer-truncation") +UBSAN_CHECK(ImplicitIntegerSignChange, + "implicit-integer-sign-change", + "implicit-integer-sign-change") +UBSAN_CHECK(ImplicitSignedIntegerTruncationOrSignChange, + "implicit-signed-integer-truncation-or-sign-change", + "implicit-signed-integer-truncation,implicit-integer-sign-change") UBSAN_CHECK(InvalidShiftBase, "invalid-shift-base", "shift-base") UBSAN_CHECK(InvalidShiftExponent, "invalid-shift-exponent", "shift-exponent") UBSAN_CHECK(OutOfBoundsIndex, "out-of-bounds-index", "bounds") diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_flags.inc b/contrib/compiler-rt/lib/ubsan/ubsan_flags.inc index 1638a054e8f0..e75a4c44e62c 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_flags.inc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_flags.inc @@ -25,5 +25,5 @@ UBSAN_FLAG(const char *, suppressions, "", "Suppressions file name.") UBSAN_FLAG(bool, report_error_type, false, "Print specific error type instead of 'undefined-behavior' in summary.") UBSAN_FLAG(bool, silence_unsigned_overflow, false, - "Do not print error reports for unsigned integer overflow. " - "Used to provide fuzzing signal without blowing up logs.") + "Do not print non-fatal error reports for unsigned integer overflow. " + "Used to provide fuzzing signal without blowing up logs.") diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc b/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc index e72a32cf32c2..11e09b0ff0f5 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc @@ -106,6 +106,62 @@ void __ubsan::__ubsan_handle_type_mismatch_v1_abort(TypeMismatchData *Data, Die(); } +static void handleAlignmentAssumptionImpl(AlignmentAssumptionData *Data, + ValueHandle Pointer, + ValueHandle Alignment, + ValueHandle Offset, + ReportOptions Opts) { + Location Loc = Data->Loc.acquire(); + SourceLocation AssumptionLoc = Data->AssumptionLoc.acquire(); + + ErrorType ET = ErrorType::AlignmentAssumption; + + if (ignoreReport(Loc.getSourceLocation(), Opts, ET)) + return; + + ScopedReport R(Opts, Loc, ET); + + uptr RealPointer = Pointer - Offset; + uptr LSB = LeastSignificantSetBitIndex(RealPointer); + uptr ActualAlignment = uptr(1) << LSB; + + uptr Mask = Alignment - 1; + uptr MisAlignmentOffset = RealPointer & Mask; + + if (!Offset) { + Diag(Loc, DL_Error, ET, + "assumption of %0 byte alignment for pointer of type %1 failed") + << Alignment << Data->Type; + } else { + Diag(Loc, DL_Error, ET, + "assumption of %0 byte alignment (with offset of %1 byte) for pointer " + "of type %2 failed") + << Alignment << Offset << Data->Type; + } + + if (!AssumptionLoc.isInvalid()) + Diag(AssumptionLoc, DL_Note, ET, "alignment assumption was specified here"); + + Diag(RealPointer, DL_Note, ET, + "%0address is %1 aligned, misalignment offset is %2 bytes") + << (Offset ? "offset " : "") << ActualAlignment << MisAlignmentOffset; +} + +void __ubsan::__ubsan_handle_alignment_assumption(AlignmentAssumptionData *Data, + ValueHandle Pointer, + ValueHandle Alignment, + ValueHandle Offset) { + GET_REPORT_OPTIONS(false); + handleAlignmentAssumptionImpl(Data, Pointer, Alignment, Offset, Opts); +} +void __ubsan::__ubsan_handle_alignment_assumption_abort( + AlignmentAssumptionData *Data, ValueHandle Pointer, ValueHandle Alignment, + ValueHandle Offset) { + GET_REPORT_OPTIONS(true); + handleAlignmentAssumptionImpl(Data, Pointer, Alignment, Offset, Opts); + Die(); +} + /// \brief Common diagnostic emission for various forms of integer overflow. template <typename T> static void handleIntegerOverflowImpl(OverflowData *Data, ValueHandle LHS, @@ -119,7 +175,9 @@ static void handleIntegerOverflowImpl(OverflowData *Data, ValueHandle LHS, if (ignoreReport(Loc, Opts, ET)) return; - if (!IsSigned && flags()->silence_unsigned_overflow) + // If this is an unsigned overflow in non-fatal mode, potentially ignore it. + if (!IsSigned && !Opts.FromUnrecoverableHandler && + flags()->silence_unsigned_overflow) return; ScopedReport R(Opts, Loc, ET); @@ -457,18 +515,41 @@ static void handleImplicitConversion(ImplicitConversionData *Data, SourceLocation Loc = Data->Loc.acquire(); ErrorType ET = ErrorType::GenericUB; + const TypeDescriptor &SrcTy = Data->FromType; + const TypeDescriptor &DstTy = Data->ToType; + + bool SrcSigned = SrcTy.isSignedIntegerTy(); + bool DstSigned = DstTy.isSignedIntegerTy(); + switch (Data->Kind) { - case ICCK_IntegerTruncation: - ET = ErrorType::ImplicitIntegerTruncation; + case ICCK_IntegerTruncation: { // Legacy, no longer used. + // Let's figure out what it should be as per the new types, and upgrade. + // If both types are unsigned, then it's an unsigned truncation. + // Else, it is a signed truncation. + if (!SrcSigned && !DstSigned) { + ET = ErrorType::ImplicitUnsignedIntegerTruncation; + } else { + ET = ErrorType::ImplicitSignedIntegerTruncation; + } + break; + } + case ICCK_UnsignedIntegerTruncation: + ET = ErrorType::ImplicitUnsignedIntegerTruncation; + break; + case ICCK_SignedIntegerTruncation: + ET = ErrorType::ImplicitSignedIntegerTruncation; + break; + case ICCK_IntegerSignChange: + ET = ErrorType::ImplicitIntegerSignChange; + break; + case ICCK_SignedIntegerTruncationOrSignChange: + ET = ErrorType::ImplicitSignedIntegerTruncationOrSignChange; break; } if (ignoreReport(Loc, Opts, ET)) return; - const TypeDescriptor &SrcTy = Data->FromType; - const TypeDescriptor &DstTy = Data->ToType; - ScopedReport R(Opts, Loc, ET); // FIXME: is it possible to dump the values as hex with fixed width? @@ -477,8 +558,8 @@ static void handleImplicitConversion(ImplicitConversionData *Data, "implicit conversion from type %0 of value %1 (%2-bit, %3signed) to " "type %4 changed the value to %5 (%6-bit, %7signed)") << SrcTy << Value(SrcTy, Src) << SrcTy.getIntegerBitWidth() - << (SrcTy.isSignedIntegerTy() ? "" : "un") << DstTy << Value(DstTy, Dst) - << DstTy.getIntegerBitWidth() << (DstTy.isSignedIntegerTy() ? "" : "un"); + << (SrcSigned ? "" : "un") << DstTy << Value(DstTy, Dst) + << DstTy.getIntegerBitWidth() << (DstSigned ? "" : "un"); } void __ubsan::__ubsan_handle_implicit_conversion(ImplicitConversionData *Data, diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_handlers.h b/contrib/compiler-rt/lib/ubsan/ubsan_handlers.h index 07644b7ea15d..2bf9ff4320e8 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_handlers.h +++ b/contrib/compiler-rt/lib/ubsan/ubsan_handlers.h @@ -39,6 +39,17 @@ struct TypeMismatchData { /// type. RECOVERABLE(type_mismatch_v1, TypeMismatchData *Data, ValueHandle Pointer) +struct AlignmentAssumptionData { + SourceLocation Loc; + SourceLocation AssumptionLoc; + const TypeDescriptor &Type; +}; + +/// \brief Handle a runtime alignment assumption check failure, +/// caused by a misaligned pointer. +RECOVERABLE(alignment_assumption, AlignmentAssumptionData *Data, + ValueHandle Pointer, ValueHandle Alignment, ValueHandle Offset) + struct OverflowData { SourceLocation Loc; const TypeDescriptor &Type; @@ -125,7 +136,11 @@ RECOVERABLE(load_invalid_value, InvalidValueData *Data, ValueHandle Val) /// Known implicit conversion check kinds. /// Keep in sync with the enum of the same name in CGExprScalar.cpp enum ImplicitConversionCheckKind : unsigned char { - ICCK_IntegerTruncation = 0, + ICCK_IntegerTruncation = 0, // Legacy, was only used by clang 7. + ICCK_UnsignedIntegerTruncation = 1, + ICCK_SignedIntegerTruncation = 2, + ICCK_IntegerSignChange = 3, + ICCK_SignedIntegerTruncationOrSignChange = 4, }; struct ImplicitConversionData { diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_interface.inc b/contrib/compiler-rt/lib/ubsan/ubsan_interface.inc index 0be6010ad27b..81e06345dcff 100644 --- a/contrib/compiler-rt/lib/ubsan/ubsan_interface.inc +++ b/contrib/compiler-rt/lib/ubsan/ubsan_interface.inc @@ -10,6 +10,8 @@ //===----------------------------------------------------------------------===// INTERFACE_FUNCTION(__ubsan_handle_add_overflow) INTERFACE_FUNCTION(__ubsan_handle_add_overflow_abort) +INTERFACE_FUNCTION(__ubsan_handle_alignment_assumption) +INTERFACE_FUNCTION(__ubsan_handle_alignment_assumption_abort) INTERFACE_FUNCTION(__ubsan_handle_builtin_unreachable) INTERFACE_FUNCTION(__ubsan_handle_cfi_bad_type) INTERFACE_FUNCTION(__ubsan_handle_cfi_check_fail) diff --git a/contrib/compiler-rt/lib/ubsan_minimal/ubsan_minimal_handlers.cc b/contrib/compiler-rt/lib/ubsan_minimal/ubsan_minimal_handlers.cc index e8fc3a8499b5..ed62ddd0fa34 100644 --- a/contrib/compiler-rt/lib/ubsan_minimal/ubsan_minimal_handlers.cc +++ b/contrib/compiler-rt/lib/ubsan_minimal/ubsan_minimal_handlers.cc @@ -95,6 +95,7 @@ void NORETURN CheckFailed(const char *file, int, const char *cond, u64, u64) { HANDLER_NORECOVER(name, msg) HANDLER(type_mismatch, "type-mismatch") +HANDLER(alignment_assumption, "alignment-assumption") HANDLER(add_overflow, "add-overflow") HANDLER(sub_overflow, "sub-overflow") HANDLER(mul_overflow, "mul-overflow") diff --git a/contrib/compiler-rt/lib/xray/xray_allocator.h b/contrib/compiler-rt/lib/xray/xray_allocator.h index 8244815284a8..907c54542a56 100644 --- a/contrib/compiler-rt/lib/xray/xray_allocator.h +++ b/contrib/compiler-rt/lib/xray/xray_allocator.h @@ -19,18 +19,131 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_internal_defs.h" #include "sanitizer_common/sanitizer_mutex.h" +#if SANITIZER_FUCHSIA +#include <zircon/process.h> +#include <zircon/status.h> +#include <zircon/syscalls.h> +#else #include "sanitizer_common/sanitizer_posix.h" +#endif +#include "xray_defs.h" #include "xray_utils.h" -#include <sys/mman.h> #include <cstddef> #include <cstdint> +#include <sys/mman.h> + +namespace __xray { + +// We implement our own memory allocation routine which will bypass the +// internal allocator. This allows us to manage the memory directly, using +// mmap'ed memory to back the allocators. +template <class T> T *allocate() XRAY_NEVER_INSTRUMENT { + uptr RoundedSize = RoundUpTo(sizeof(T), GetPageSizeCached()); +#if SANITIZER_FUCHSIA + zx_handle_t Vmo; + zx_status_t Status = _zx_vmo_create(RoundedSize, 0, &Vmo); + if (Status != ZX_OK) { + if (Verbosity()) + Report("XRay Profiling: Failed to create VMO of size %zu: %s\n", + sizeof(T), _zx_status_get_string(Status)); + return nullptr; + } + uintptr_t B; + Status = + _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0, + Vmo, 0, sizeof(T), &B); + _zx_handle_close(Vmo); + if (Status != ZX_OK) { + if (Verbosity()) + Report("XRay Profiling: Failed to map VMAR of size %zu: %s\n", sizeof(T), + _zx_status_get_string(Status)); + return nullptr; + } + return reinterpret_cast<T *>(B); +#else + uptr B = internal_mmap(NULL, RoundedSize, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int ErrNo = 0; + if (UNLIKELY(internal_iserror(B, &ErrNo))) { + if (Verbosity()) + Report( + "XRay Profiling: Failed to allocate memory of size %d; Error = %d.\n", + RoundedSize, B); + return nullptr; + } +#endif + return reinterpret_cast<T *>(B); +} -#ifndef MAP_NORESERVE -// no-op on NetBSD (at least), unsupported flag on FreeBSD basically because unneeded -#define MAP_NORESERVE 0 +template <class T> void deallocate(T *B) XRAY_NEVER_INSTRUMENT { + if (B == nullptr) + return; + uptr RoundedSize = RoundUpTo(sizeof(T), GetPageSizeCached()); +#if SANITIZER_FUCHSIA + _zx_vmar_unmap(_zx_vmar_root_self(), reinterpret_cast<uintptr_t>(B), + RoundedSize); +#else + internal_munmap(B, RoundedSize); #endif +} -namespace __xray { +template <class T = unsigned char> +T *allocateBuffer(size_t S) XRAY_NEVER_INSTRUMENT { + uptr RoundedSize = RoundUpTo(S * sizeof(T), GetPageSizeCached()); +#if SANITIZER_FUCHSIA + zx_handle_t Vmo; + zx_status_t Status = _zx_vmo_create(RoundedSize, 0, &Vmo); + if (Status != ZX_OK) { + if (Verbosity()) + Report("XRay Profiling: Failed to create VMO of size %zu: %s\n", S, + _zx_status_get_string(Status)); + return nullptr; + } + uintptr_t B; + Status = _zx_vmar_map(_zx_vmar_root_self(), + ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0, Vmo, 0, S, &B); + _zx_handle_close(Vmo); + if (Status != ZX_OK) { + if (Verbosity()) + Report("XRay Profiling: Failed to map VMAR of size %zu: %s\n", S, + _zx_status_get_string(Status)); + return nullptr; + } +#else + uptr B = internal_mmap(NULL, RoundedSize, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int ErrNo = 0; + if (UNLIKELY(internal_iserror(B, &ErrNo))) { + if (Verbosity()) + Report( + "XRay Profiling: Failed to allocate memory of size %d; Error = %d.\n", + RoundedSize, B); + return nullptr; + } +#endif + return reinterpret_cast<T *>(B); +} + +template <class T> void deallocateBuffer(T *B, size_t S) XRAY_NEVER_INSTRUMENT { + if (B == nullptr) + return; + uptr RoundedSize = RoundUpTo(S * sizeof(T), GetPageSizeCached()); +#if SANITIZER_FUCHSIA + _zx_vmar_unmap(_zx_vmar_root_self(), reinterpret_cast<uintptr_t>(B), + RoundedSize); +#else + internal_munmap(B, RoundedSize); +#endif +} + +template <class T, class... U> +T *initArray(size_t N, U &&... Us) XRAY_NEVER_INSTRUMENT { + auto A = allocateBuffer<T>(N); + if (A != nullptr) + while (N > 0) + new (A + (--N)) T(std::forward<U>(Us)...); + return A; +} /// The Allocator type hands out fixed-sized chunks of memory that are /// cache-line aligned and sized. This is useful for placement of @@ -58,20 +171,18 @@ template <size_t N> struct Allocator { }; private: - const size_t MaxMemory{0}; - void *BackingStore = nullptr; - void *AlignedNextBlock = nullptr; + size_t MaxMemory{0}; + unsigned char *BackingStore = nullptr; + unsigned char *AlignedNextBlock = nullptr; size_t AllocatedBlocks = 0; + bool Owned; SpinMutex Mutex{}; - void *Alloc() { + void *Alloc() XRAY_NEVER_INSTRUMENT { SpinMutexLock Lock(&Mutex); if (UNLIKELY(BackingStore == nullptr)) { - BackingStore = reinterpret_cast<void *>( - internal_mmap(NULL, MaxMemory, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, 0, 0)); - if (BackingStore == MAP_FAILED) { - BackingStore = nullptr; + BackingStore = allocateBuffer(MaxMemory); + if (BackingStore == nullptr) { if (Verbosity()) Report("XRay Profiling: Failed to allocate memory for allocator.\n"); return nullptr; @@ -84,7 +195,7 @@ private: auto AlignedNextBlockNum = nearest_boundary( reinterpret_cast<uintptr_t>(AlignedNextBlock), kCacheLineSize); if (diff(AlignedNextBlockNum, BackingStoreNum) > ptrdiff_t(MaxMemory)) { - munmap(BackingStore, MaxMemory); + deallocateBuffer(BackingStore, MaxMemory); AlignedNextBlock = BackingStore = nullptr; if (Verbosity()) Report("XRay Profiling: Cannot obtain enough memory from " @@ -92,34 +203,83 @@ private: return nullptr; } - AlignedNextBlock = reinterpret_cast<void *>(AlignedNextBlockNum); + AlignedNextBlock = reinterpret_cast<unsigned char *>(AlignedNextBlockNum); // Assert that AlignedNextBlock is cache-line aligned. DCHECK_EQ(reinterpret_cast<uintptr_t>(AlignedNextBlock) % kCacheLineSize, 0); } - if ((AllocatedBlocks * Block::Size) >= MaxMemory) + if (((AllocatedBlocks + 1) * Block::Size) > MaxMemory) return nullptr; // Align the pointer we'd like to return to an appropriate alignment, then // advance the pointer from where to start allocations. void *Result = AlignedNextBlock; - AlignedNextBlock = reinterpret_cast<void *>( - reinterpret_cast<char *>(AlignedNextBlock) + N); + AlignedNextBlock = + reinterpret_cast<unsigned char *>(AlignedNextBlock) + Block::Size; ++AllocatedBlocks; return Result; } public: - explicit Allocator(size_t M) - : MaxMemory(nearest_boundary(M, kCacheLineSize)) {} + explicit Allocator(size_t M) XRAY_NEVER_INSTRUMENT + : MaxMemory(RoundUpTo(M, kCacheLineSize)), + BackingStore(nullptr), + AlignedNextBlock(nullptr), + AllocatedBlocks(0), + Owned(true), + Mutex() {} + + explicit Allocator(void *P, size_t M) XRAY_NEVER_INSTRUMENT + : MaxMemory(M), + BackingStore(reinterpret_cast<unsigned char *>(P)), + AlignedNextBlock(reinterpret_cast<unsigned char *>(P)), + AllocatedBlocks(0), + Owned(false), + Mutex() {} + + Allocator(const Allocator &) = delete; + Allocator &operator=(const Allocator &) = delete; + + Allocator(Allocator &&O) XRAY_NEVER_INSTRUMENT { + SpinMutexLock L0(&Mutex); + SpinMutexLock L1(&O.Mutex); + MaxMemory = O.MaxMemory; + O.MaxMemory = 0; + BackingStore = O.BackingStore; + O.BackingStore = nullptr; + AlignedNextBlock = O.AlignedNextBlock; + O.AlignedNextBlock = nullptr; + AllocatedBlocks = O.AllocatedBlocks; + O.AllocatedBlocks = 0; + Owned = O.Owned; + O.Owned = false; + } + + Allocator &operator=(Allocator &&O) XRAY_NEVER_INSTRUMENT { + SpinMutexLock L0(&Mutex); + SpinMutexLock L1(&O.Mutex); + MaxMemory = O.MaxMemory; + O.MaxMemory = 0; + if (BackingStore != nullptr) + deallocateBuffer(BackingStore, MaxMemory); + BackingStore = O.BackingStore; + O.BackingStore = nullptr; + AlignedNextBlock = O.AlignedNextBlock; + O.AlignedNextBlock = nullptr; + AllocatedBlocks = O.AllocatedBlocks; + O.AllocatedBlocks = 0; + Owned = O.Owned; + O.Owned = false; + return *this; + } - Block Allocate() { return {Alloc()}; } + Block Allocate() XRAY_NEVER_INSTRUMENT { return {Alloc()}; } - ~Allocator() NOEXCEPT { - if (BackingStore != nullptr) { - internal_munmap(BackingStore, MaxMemory); + ~Allocator() NOEXCEPT XRAY_NEVER_INSTRUMENT { + if (Owned && BackingStore != nullptr) { + deallocateBuffer(BackingStore, MaxMemory); } } }; diff --git a/contrib/compiler-rt/lib/xray/xray_basic_logging.cc b/contrib/compiler-rt/lib/xray/xray_basic_logging.cc index 585ca641cd0c..ae1cc0ba79dd 100644 --- a/contrib/compiler-rt/lib/xray/xray_basic_logging.cc +++ b/contrib/compiler-rt/lib/xray/xray_basic_logging.cc @@ -19,7 +19,9 @@ #include <fcntl.h> #include <pthread.h> #include <sys/stat.h> +#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD || SANITIZER_MAC #include <sys/syscall.h> +#endif #include <sys/types.h> #include <time.h> #include <unistd.h> @@ -38,8 +40,9 @@ namespace __xray { -SpinMutex LogMutex; +static SpinMutex LogMutex; +namespace { // We use elements of this type to record the entry TSC of every function ID we // see as we're tracing a particular thread's execution. struct alignas(16) StackEntry { @@ -52,21 +55,28 @@ struct alignas(16) StackEntry { static_assert(sizeof(StackEntry) == 16, "Wrong size for StackEntry"); -struct alignas(64) ThreadLocalData { +struct XRAY_TLS_ALIGNAS(64) ThreadLocalData { void *InMemoryBuffer = nullptr; size_t BufferSize = 0; size_t BufferOffset = 0; void *ShadowStack = nullptr; size_t StackSize = 0; size_t StackEntries = 0; - int Fd = -1; + __xray::LogWriter *LogWriter = nullptr; }; +struct BasicLoggingOptions { + int DurationFilterMicros = 0; + size_t MaxStackDepth = 0; + size_t ThreadBufferSize = 0; +}; +} // namespace + static pthread_key_t PThreadKey; static atomic_uint8_t BasicInitialized{0}; -BasicLoggingOptions GlobalOptions; +struct BasicLoggingOptions GlobalOptions; thread_local atomic_uint8_t Guard{0}; @@ -75,10 +85,10 @@ static atomic_uint64_t ThresholdTicks{0}; static atomic_uint64_t TicksPerSec{0}; static atomic_uint64_t CycleFrequency{NanosecondsPerSecond}; -static int openLogFile() XRAY_NEVER_INSTRUMENT { - int F = getLogFD(); - if (F == -1) - return -1; +static LogWriter *getLog() XRAY_NEVER_INSTRUMENT { + LogWriter* LW = LogWriter::Open(); + if (LW == nullptr) + return LW; static pthread_once_t DetectOnce = PTHREAD_ONCE_INIT; pthread_once(&DetectOnce, +[] { @@ -100,16 +110,16 @@ static int openLogFile() XRAY_NEVER_INSTRUMENT { // before setting the values in the header. Header.ConstantTSC = 1; Header.NonstopTSC = 1; - retryingWriteAll(F, reinterpret_cast<char *>(&Header), - reinterpret_cast<char *>(&Header) + sizeof(Header)); - return F; + LW->WriteAll(reinterpret_cast<char *>(&Header), + reinterpret_cast<char *>(&Header) + sizeof(Header)); + return LW; } -static int getGlobalFd() XRAY_NEVER_INSTRUMENT { +static LogWriter *getGlobalLog() XRAY_NEVER_INSTRUMENT { static pthread_once_t OnceInit = PTHREAD_ONCE_INIT; - static int Fd = 0; - pthread_once(&OnceInit, +[] { Fd = openLogFile(); }); - return Fd; + static LogWriter *LW = nullptr; + pthread_once(&OnceInit, +[] { LW = getLog(); }); + return LW; } static ThreadLocalData &getThreadLocalData() XRAY_NEVER_INSTRUMENT { @@ -121,7 +131,7 @@ static ThreadLocalData &getThreadLocalData() XRAY_NEVER_INSTRUMENT { return false; } pthread_setspecific(PThreadKey, &TLD); - TLD.Fd = getGlobalFd(); + TLD.LogWriter = getGlobalLog(); TLD.InMemoryBuffer = reinterpret_cast<XRayRecord *>( InternalAlloc(sizeof(XRayRecord) * GlobalOptions.ThreadBufferSize, nullptr, alignof(XRayRecord))); @@ -149,8 +159,8 @@ template <class RDTSC> void InMemoryRawLog(int32_t FuncId, XRayEntryType Type, RDTSC ReadTSC) XRAY_NEVER_INSTRUMENT { auto &TLD = getThreadLocalData(); - int Fd = getGlobalFd(); - if (Fd == -1) + LogWriter *LW = getGlobalLog(); + if (LW == nullptr) return; // Use a simple recursion guard, to handle cases where we're already logging @@ -234,9 +244,9 @@ void InMemoryRawLog(int32_t FuncId, XRayEntryType Type, auto FirstEntry = reinterpret_cast<XRayRecord *>(TLD.InMemoryBuffer); internal_memcpy(FirstEntry + TLD.BufferOffset, &R, sizeof(R)); if (++TLD.BufferOffset == TLD.BufferSize) { - SpinMutexLock L(&LogMutex); - retryingWriteAll(Fd, reinterpret_cast<char *>(FirstEntry), - reinterpret_cast<char *>(FirstEntry + TLD.BufferOffset)); + SpinMutexLock Lock(&LogMutex); + LW->WriteAll(reinterpret_cast<char *>(FirstEntry), + reinterpret_cast<char *>(FirstEntry + TLD.BufferOffset)); TLD.BufferOffset = 0; TLD.StackEntries = 0; } @@ -249,17 +259,17 @@ void InMemoryRawLogWithArg(int32_t FuncId, XRayEntryType Type, uint64_t Arg1, auto FirstEntry = reinterpret_cast<XRayArgPayload *>(TLD.InMemoryBuffer); const auto &BuffLen = TLD.BufferSize; - int Fd = getGlobalFd(); - if (Fd == -1) + LogWriter *LW = getGlobalLog(); + if (LW == nullptr) return; // First we check whether there's enough space to write the data consecutively // in the thread-local buffer. If not, we first flush the buffer before // attempting to write the two records that must be consecutive. if (TLD.BufferOffset + 2 > BuffLen) { - SpinMutexLock L(&LogMutex); - retryingWriteAll(Fd, reinterpret_cast<char *>(FirstEntry), - reinterpret_cast<char *>(FirstEntry + TLD.BufferOffset)); + SpinMutexLock Lock(&LogMutex); + LW->WriteAll(reinterpret_cast<char *>(FirstEntry), + reinterpret_cast<char *>(FirstEntry + TLD.BufferOffset)); TLD.BufferOffset = 0; TLD.StackEntries = 0; } @@ -280,9 +290,9 @@ void InMemoryRawLogWithArg(int32_t FuncId, XRayEntryType Type, uint64_t Arg1, R.Arg = Arg1; internal_memcpy(FirstEntry + TLD.BufferOffset, &R, sizeof(R)); if (++TLD.BufferOffset == BuffLen) { - SpinMutexLock L(&LogMutex); - retryingWriteAll(Fd, reinterpret_cast<char *>(FirstEntry), - reinterpret_cast<char *>(FirstEntry + TLD.BufferOffset)); + SpinMutexLock Lock(&LogMutex); + LW->WriteAll(reinterpret_cast<char *>(FirstEntry), + reinterpret_cast<char *>(FirstEntry + TLD.BufferOffset)); TLD.BufferOffset = 0; TLD.StackEntries = 0; } @@ -339,29 +349,29 @@ static void TLDDestructor(void *P) XRAY_NEVER_INSTRUMENT { Report("Cleaned up log for TID: %d\n", GetTid()); }); - if (TLD.Fd == -1 || TLD.BufferOffset == 0) { + if (TLD.LogWriter == nullptr || TLD.BufferOffset == 0) { if (Verbosity()) - Report("Skipping buffer for TID: %d; Fd = %d; Offset = %llu\n", GetTid(), - TLD.Fd, TLD.BufferOffset); + Report("Skipping buffer for TID: %d; Offset = %llu\n", GetTid(), + TLD.BufferOffset); return; } { SpinMutexLock L(&LogMutex); - retryingWriteAll(TLD.Fd, reinterpret_cast<char *>(TLD.InMemoryBuffer), - reinterpret_cast<char *>(TLD.InMemoryBuffer) + - (sizeof(XRayRecord) * TLD.BufferOffset)); + TLD.LogWriter->WriteAll(reinterpret_cast<char *>(TLD.InMemoryBuffer), + reinterpret_cast<char *>(TLD.InMemoryBuffer) + + (sizeof(XRayRecord) * TLD.BufferOffset)); } // Because this thread's exit could be the last one trying to write to // the file and that we're not able to close out the file properly, we // sync instead and hope that the pending writes are flushed as the // thread exits. - fsync(TLD.Fd); + TLD.LogWriter->Flush(); } -XRayLogInitStatus basicLoggingInit(size_t BufferSize, size_t BufferMax, - void *Options, +XRayLogInitStatus basicLoggingInit(UNUSED size_t BufferSize, + UNUSED size_t BufferMax, void *Options, size_t OptionsSize) XRAY_NEVER_INSTRUMENT { uint8_t Expected = 0; if (!atomic_compare_exchange_strong(&BasicInitialized, &Expected, 1, @@ -385,43 +395,32 @@ XRayLogInitStatus basicLoggingInit(size_t BufferSize, size_t BufferMax, "using emulation instead.\n"); }); - if (BufferSize == 0 && BufferMax == 0 && Options != nullptr) { - FlagParser P; - BasicFlags F; - F.setDefaults(); - registerXRayBasicFlags(&P, &F); - P.ParseString(useCompilerDefinedBasicFlags()); - auto *EnvOpts = GetEnv("XRAY_BASIC_OPTIONS"); - if (EnvOpts == nullptr) - EnvOpts = ""; - - P.ParseString(EnvOpts); - - // If XRAY_BASIC_OPTIONS was not defined, then we use the deprecated options - // set through XRAY_OPTIONS instead. - if (internal_strlen(EnvOpts) == 0) { - F.func_duration_threshold_us = - flags()->xray_naive_log_func_duration_threshold_us; - F.max_stack_depth = flags()->xray_naive_log_max_stack_depth; - F.thread_buffer_size = flags()->xray_naive_log_thread_buffer_size; - } - - P.ParseString(static_cast<const char *>(Options)); - GlobalOptions.ThreadBufferSize = F.thread_buffer_size; - GlobalOptions.DurationFilterMicros = F.func_duration_threshold_us; - GlobalOptions.MaxStackDepth = F.max_stack_depth; - *basicFlags() = F; - } else if (OptionsSize != sizeof(BasicLoggingOptions)) { - Report("Invalid options size, potential ABI mismatch; expected %d got %d", - sizeof(BasicLoggingOptions), OptionsSize); - return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; - } else { - if (Verbosity()) - Report("XRay Basic: struct-based init is deprecated, please use " - "string-based configuration instead.\n"); - GlobalOptions = *reinterpret_cast<BasicLoggingOptions *>(Options); + FlagParser P; + BasicFlags F; + F.setDefaults(); + registerXRayBasicFlags(&P, &F); + P.ParseString(useCompilerDefinedBasicFlags()); + auto *EnvOpts = GetEnv("XRAY_BASIC_OPTIONS"); + if (EnvOpts == nullptr) + EnvOpts = ""; + + P.ParseString(EnvOpts); + + // If XRAY_BASIC_OPTIONS was not defined, then we use the deprecated options + // set through XRAY_OPTIONS instead. + if (internal_strlen(EnvOpts) == 0) { + F.func_duration_threshold_us = + flags()->xray_naive_log_func_duration_threshold_us; + F.max_stack_depth = flags()->xray_naive_log_max_stack_depth; + F.thread_buffer_size = flags()->xray_naive_log_thread_buffer_size; } + P.ParseString(static_cast<const char *>(Options)); + GlobalOptions.ThreadBufferSize = F.thread_buffer_size; + GlobalOptions.DurationFilterMicros = F.func_duration_threshold_us; + GlobalOptions.MaxStackDepth = F.max_stack_depth; + *basicFlags() = F; + atomic_store(&ThresholdTicks, atomic_load(&TicksPerSec, memory_order_acquire) * GlobalOptions.DurationFilterMicros / 1000000, diff --git a/contrib/compiler-rt/lib/xray/xray_buffer_queue.cc b/contrib/compiler-rt/lib/xray/xray_buffer_queue.cc index 3ce728900787..7d0e5a1f323c 100644 --- a/contrib/compiler-rt/lib/xray/xray_buffer_queue.cc +++ b/contrib/compiler-rt/lib/xray/xray_buffer_queue.cc @@ -13,141 +13,206 @@ // //===----------------------------------------------------------------------===// #include "xray_buffer_queue.h" +#include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_libc.h" +#if !SANITIZER_FUCHSIA #include "sanitizer_common/sanitizer_posix.h" +#endif +#include "xray_allocator.h" +#include "xray_defs.h" #include <memory> #include <sys/mman.h> -#ifndef MAP_NORESERVE -// no-op on NetBSD (at least), unsupported flag on FreeBSD -#define MAP_NORESERVE 0 -#endif - using namespace __xray; -using namespace __sanitizer; - -template <class T> static T *allocRaw(size_t N) { - // TODO: Report errors? - // We use MAP_NORESERVE on platforms where it's supported to ensure that the - // pages we're allocating for XRay never end up in pages that can be swapped - // in/out. We're doing this because for FDR mode, we want to ensure that - // writes to the buffers stay resident in memory to prevent XRay itself from - // causing swapping/thrashing. - // - // In the case when XRay pages cannot be swapped in/out or there's not enough - // RAM to back these pages, we're willing to cause a segmentation fault - // instead of introducing latency in the measurement. We assume here that - // there are enough pages that are swappable in/out outside of the buffers - // being used by FDR mode (which are bounded and configurable anyway) to allow - // us to keep using always-resident memory. - // - // TODO: Make this configurable? - void *A = reinterpret_cast<void *>( - internal_mmap(NULL, N * sizeof(T), PROT_WRITE | PROT_READ, - MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, 0)); - return (A == MAP_FAILED) ? nullptr : reinterpret_cast<T *>(A); -} -template <class T> static void deallocRaw(T *ptr, size_t N) { - // TODO: Report errors? - if (ptr != nullptr) - internal_munmap(ptr, N); +namespace { + +BufferQueue::ControlBlock *allocControlBlock(size_t Size, size_t Count) { + auto B = + allocateBuffer((sizeof(BufferQueue::ControlBlock) - 1) + (Size * Count)); + return B == nullptr ? nullptr + : reinterpret_cast<BufferQueue::ControlBlock *>(B); } -template <class T> static T *initArray(size_t N) { - auto A = allocRaw<T>(N); - if (A != nullptr) - while (N > 0) - new (A + (--N)) T(); - return A; +void deallocControlBlock(BufferQueue::ControlBlock *C, size_t Size, + size_t Count) { + deallocateBuffer(reinterpret_cast<unsigned char *>(C), + (sizeof(BufferQueue::ControlBlock) - 1) + (Size * Count)); } -BufferQueue::BufferQueue(size_t B, size_t N, bool &Success) - : BufferSize(B), Buffers(initArray<BufferQueue::BufferRep>(N)), - BufferCount(N), Finalizing{0}, OwnedBuffers(initArray<void *>(N)), - Next(Buffers), First(Buffers), LiveBuffers(0) { - if (Buffers == nullptr) { - Success = false; +void decRefCount(BufferQueue::ControlBlock *C, size_t Size, size_t Count) { + if (C == nullptr) return; - } - if (OwnedBuffers == nullptr) { - // Clean up the buffers we've already allocated. - for (auto B = Buffers, E = Buffers + BufferCount; B != E; ++B) - B->~BufferRep(); - deallocRaw(Buffers, N); - Success = false; + if (atomic_fetch_sub(&C->RefCount, 1, memory_order_acq_rel) == 1) + deallocControlBlock(C, Size, Count); +} + +void incRefCount(BufferQueue::ControlBlock *C) { + if (C == nullptr) return; + atomic_fetch_add(&C->RefCount, 1, memory_order_acq_rel); +} + +// We use a struct to ensure that we are allocating one atomic_uint64_t per +// cache line. This allows us to not worry about false-sharing among atomic +// objects being updated (constantly) by different threads. +struct ExtentsPadded { + union { + atomic_uint64_t Extents; + unsigned char Storage[kCacheLineSize]; }; +}; - for (size_t i = 0; i < N; ++i) { - auto &T = Buffers[i]; - void *Tmp = allocRaw<char>(BufferSize); - if (Tmp == nullptr) { - Success = false; +constexpr size_t kExtentsSize = sizeof(ExtentsPadded); + +} // namespace + +BufferQueue::ErrorCode BufferQueue::init(size_t BS, size_t BC) { + SpinMutexLock Guard(&Mutex); + + if (!finalizing()) + return BufferQueue::ErrorCode::AlreadyInitialized; + + cleanupBuffers(); + + bool Success = false; + BufferSize = BS; + BufferCount = BC; + + BackingStore = allocControlBlock(BufferSize, BufferCount); + if (BackingStore == nullptr) + return BufferQueue::ErrorCode::NotEnoughMemory; + + auto CleanupBackingStore = at_scope_exit([&, this] { + if (Success) return; - } - auto *Extents = allocRaw<BufferExtents>(1); - if (Extents == nullptr) { - Success = false; + deallocControlBlock(BackingStore, BufferSize, BufferCount); + BackingStore = nullptr; + }); + + // Initialize enough atomic_uint64_t instances, each + ExtentsBackingStore = allocControlBlock(kExtentsSize, BufferCount); + if (ExtentsBackingStore == nullptr) + return BufferQueue::ErrorCode::NotEnoughMemory; + + auto CleanupExtentsBackingStore = at_scope_exit([&, this] { + if (Success) return; - } + deallocControlBlock(ExtentsBackingStore, kExtentsSize, BufferCount); + ExtentsBackingStore = nullptr; + }); + + Buffers = initArray<BufferRep>(BufferCount); + if (Buffers == nullptr) + return BufferQueue::ErrorCode::NotEnoughMemory; + + // At this point we increment the generation number to associate the buffers + // to the new generation. + atomic_fetch_add(&Generation, 1, memory_order_acq_rel); + + // First, we initialize the refcount in the ControlBlock, which we treat as + // being at the start of the BackingStore pointer. + atomic_store(&BackingStore->RefCount, 1, memory_order_release); + atomic_store(&ExtentsBackingStore->RefCount, 1, memory_order_release); + + // Then we initialise the individual buffers that sub-divide the whole backing + // store. Each buffer will start at the `Data` member of the ControlBlock, and + // will be offsets from these locations. + for (size_t i = 0; i < BufferCount; ++i) { + auto &T = Buffers[i]; auto &Buf = T.Buff; - Buf.Data = Tmp; - Buf.Size = B; - Buf.Extents = Extents; - OwnedBuffers[i] = Tmp; + auto *E = reinterpret_cast<ExtentsPadded *>(&ExtentsBackingStore->Data + + (kExtentsSize * i)); + Buf.Extents = &E->Extents; + atomic_store(Buf.Extents, 0, memory_order_release); + Buf.Generation = generation(); + Buf.Data = &BackingStore->Data + (BufferSize * i); + Buf.Size = BufferSize; + Buf.BackingStore = BackingStore; + Buf.ExtentsBackingStore = ExtentsBackingStore; + Buf.Count = BufferCount; + T.Used = false; } + + Next = Buffers; + First = Buffers; + LiveBuffers = 0; + atomic_store(&Finalizing, 0, memory_order_release); Success = true; + return BufferQueue::ErrorCode::Ok; +} + +BufferQueue::BufferQueue(size_t B, size_t N, + bool &Success) XRAY_NEVER_INSTRUMENT + : BufferSize(B), + BufferCount(N), + Mutex(), + Finalizing{1}, + BackingStore(nullptr), + ExtentsBackingStore(nullptr), + Buffers(nullptr), + Next(Buffers), + First(Buffers), + LiveBuffers(0), + Generation{0} { + Success = init(B, N) == BufferQueue::ErrorCode::Ok; } BufferQueue::ErrorCode BufferQueue::getBuffer(Buffer &Buf) { if (atomic_load(&Finalizing, memory_order_acquire)) return ErrorCode::QueueFinalizing; - SpinMutexLock Guard(&Mutex); - if (LiveBuffers == BufferCount) - return ErrorCode::NotEnoughMemory; - auto &T = *Next; - auto &B = T.Buff; - Buf = B; - T.Used = true; - ++LiveBuffers; - - if (++Next == (Buffers + BufferCount)) - Next = Buffers; + BufferRep *B = nullptr; + { + SpinMutexLock Guard(&Mutex); + if (LiveBuffers == BufferCount) + return ErrorCode::NotEnoughMemory; + B = Next++; + if (Next == (Buffers + BufferCount)) + Next = Buffers; + ++LiveBuffers; + } + incRefCount(BackingStore); + incRefCount(ExtentsBackingStore); + Buf = B->Buff; + Buf.Generation = generation(); + B->Used = true; return ErrorCode::Ok; } BufferQueue::ErrorCode BufferQueue::releaseBuffer(Buffer &Buf) { - // Blitz through the buffers array to find the buffer. - bool Found = false; - for (auto I = OwnedBuffers, E = OwnedBuffers + BufferCount; I != E; ++I) { - if (*I == Buf.Data) { - Found = true; - break; + // Check whether the buffer being referred to is within the bounds of the + // backing store's range. + BufferRep *B = nullptr; + { + SpinMutexLock Guard(&Mutex); + if (Buf.Generation != generation() || LiveBuffers == 0) { + Buf = {}; + decRefCount(Buf.BackingStore, Buf.Size, Buf.Count); + decRefCount(Buf.ExtentsBackingStore, kExtentsSize, Buf.Count); + return BufferQueue::ErrorCode::Ok; } - } - if (!Found) - return ErrorCode::UnrecognizedBuffer; - SpinMutexLock Guard(&Mutex); + if (Buf.Data < &BackingStore->Data || + Buf.Data > &BackingStore->Data + (BufferCount * BufferSize)) + return BufferQueue::ErrorCode::UnrecognizedBuffer; - // This points to a semantic bug, we really ought to not be releasing more - // buffers than we actually get. - if (LiveBuffers == 0) - return ErrorCode::NotEnoughMemory; + --LiveBuffers; + B = First++; + if (First == (Buffers + BufferCount)) + First = Buffers; + } // Now that the buffer has been released, we mark it as "used". - First->Buff = Buf; - First->Used = true; - Buf.Data = nullptr; - Buf.Size = 0; - --LiveBuffers; - if (++First == (Buffers + BufferCount)) - First = Buffers; - + B->Buff = Buf; + B->Used = true; + decRefCount(Buf.BackingStore, Buf.Size, Buf.Count); + decRefCount(Buf.ExtentsBackingStore, kExtentsSize, Buf.Count); + atomic_store(B->Buff.Extents, atomic_load(Buf.Extents, memory_order_acquire), + memory_order_release); + Buf = {}; return ErrorCode::Ok; } @@ -157,15 +222,17 @@ BufferQueue::ErrorCode BufferQueue::finalize() { return ErrorCode::Ok; } -BufferQueue::~BufferQueue() { - for (auto I = Buffers, E = Buffers + BufferCount; I != E; ++I) { - auto &T = *I; - auto &Buf = T.Buff; - deallocRaw(Buf.Data, Buf.Size); - deallocRaw(Buf.Extents, 1); - } +void BufferQueue::cleanupBuffers() { for (auto B = Buffers, E = Buffers + BufferCount; B != E; ++B) B->~BufferRep(); - deallocRaw(Buffers, BufferCount); - deallocRaw(OwnedBuffers, BufferCount); + deallocateBuffer(Buffers, BufferCount); + decRefCount(BackingStore, BufferSize, BufferCount); + decRefCount(ExtentsBackingStore, kExtentsSize, BufferCount); + BackingStore = nullptr; + ExtentsBackingStore = nullptr; + Buffers = nullptr; + BufferCount = 0; + BufferSize = 0; } + +BufferQueue::~BufferQueue() { cleanupBuffers(); } diff --git a/contrib/compiler-rt/lib/xray/xray_buffer_queue.h b/contrib/compiler-rt/lib/xray/xray_buffer_queue.h index e76fa7983c90..ef2b433f9a3f 100644 --- a/contrib/compiler-rt/lib/xray/xray_buffer_queue.h +++ b/contrib/compiler-rt/lib/xray/xray_buffer_queue.h @@ -18,25 +18,51 @@ #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_mutex.h" +#include "xray_defs.h" #include <cstddef> +#include <cstdint> namespace __xray { /// BufferQueue implements a circular queue of fixed sized buffers (much like a -/// freelist) but is concerned mostly with making it really quick to initialise, -/// finalise, and get/return buffers to the queue. This is one key component of -/// the "flight data recorder" (FDR) mode to support ongoing XRay function call +/// freelist) but is concerned with making it quick to initialise, finalise, and +/// get from or return buffers to the queue. This is one key component of the +/// "flight data recorder" (FDR) mode to support ongoing XRay function call /// trace collection. class BufferQueue { public: - struct alignas(64) BufferExtents { - atomic_uint64_t Size; + /// ControlBlock represents the memory layout of how we interpret the backing + /// store for all buffers and extents managed by a BufferQueue instance. The + /// ControlBlock has the reference count as the first member, sized according + /// to platform-specific cache-line size. We never use the Buffer member of + /// the union, which is only there for compiler-supported alignment and + /// sizing. + /// + /// This ensures that the `Data` member will be placed at least kCacheLineSize + /// bytes from the beginning of the structure. + struct ControlBlock { + union { + atomic_uint64_t RefCount; + char Buffer[kCacheLineSize]; + }; + + /// We need to make this size 1, to conform to the C++ rules for array data + /// members. Typically, we want to subtract this 1 byte for sizing + /// information. + char Data[1]; }; struct Buffer { + atomic_uint64_t *Extents = nullptr; + uint64_t Generation{0}; void *Data = nullptr; size_t Size = 0; - BufferExtents *Extents; + + private: + friend class BufferQueue; + ControlBlock *BackingStore = nullptr; + ControlBlock *ExtentsBackingStore = nullptr; + size_t Count = 0; }; struct BufferRep { @@ -76,8 +102,10 @@ private: T *operator->() const { return &(Buffers[Offset].Buff); } - Iterator(BufferRep *Root, size_t O, size_t M) - : Buffers(Root), Offset(O), Max(M) { + Iterator(BufferRep *Root, size_t O, size_t M) XRAY_NEVER_INSTRUMENT + : Buffers(Root), + Offset(O), + Max(M) { // We want to advance to the first Offset where the 'Used' property is // true, or to the end of the list/queue. while (!Buffers[Offset].Used && Offset != Max) { @@ -107,16 +135,20 @@ private: // Size of each individual Buffer. size_t BufferSize; - BufferRep *Buffers; - // Amount of pre-allocated buffers. size_t BufferCount; SpinMutex Mutex; atomic_uint8_t Finalizing; - // Pointers to buffers managed/owned by the BufferQueue. - void **OwnedBuffers; + // The collocated ControlBlock and buffer storage. + ControlBlock *BackingStore; + + // The collocated ControlBlock and extents storage. + ControlBlock *ExtentsBackingStore; + + // A dynamically allocated array of BufferRep instances. + BufferRep *Buffers; // Pointer to the next buffer to be handed out. BufferRep *Next; @@ -128,6 +160,13 @@ private: // Count of buffers that have been handed out through 'getBuffer'. size_t LiveBuffers; + // We use a generation number to identify buffers and which generation they're + // associated with. + atomic_uint64_t Generation; + + /// Releases references to the buffers backed by the current buffer queue. + void cleanupBuffers(); + public: enum class ErrorCode : unsigned { Ok, @@ -135,6 +174,7 @@ public: QueueFinalizing, UnrecognizedBuffer, AlreadyFinalized, + AlreadyInitialized, }; static const char *getErrorString(ErrorCode E) { @@ -149,6 +189,8 @@ public: return "buffer being returned not owned by buffer queue"; case ErrorCode::AlreadyFinalized: return "queue already finalized"; + case ErrorCode::AlreadyInitialized: + return "queue already initialized"; } return "unknown error"; } @@ -179,10 +221,23 @@ public: /// the buffer being released. ErrorCode releaseBuffer(Buffer &Buf); + /// Initializes the buffer queue, starting a new generation. We can re-set the + /// size of buffers with |BS| along with the buffer count with |BC|. + /// + /// Returns: + /// - ErrorCode::Ok when we successfully initialize the buffer. This + /// requires that the buffer queue is previously finalized. + /// - ErrorCode::AlreadyInitialized when the buffer queue is not finalized. + ErrorCode init(size_t BS, size_t BC); + bool finalizing() const { return atomic_load(&Finalizing, memory_order_acquire); } + uint64_t generation() const { + return atomic_load(&Generation, memory_order_acquire); + } + /// Returns the configured size of the buffers in the buffer queue. size_t ConfiguredBufferSize() const { return BufferSize; } @@ -198,7 +253,7 @@ public: /// Applies the provided function F to each Buffer in the queue, only if the /// Buffer is marked 'used' (i.e. has been the result of getBuffer(...) and a /// releaseBuffer(...) operation). - template <class F> void apply(F Fn) { + template <class F> void apply(F Fn) XRAY_NEVER_INSTRUMENT { SpinMutexLock G(&Mutex); for (auto I = begin(), E = end(); I != E; ++I) Fn(*I); diff --git a/contrib/compiler-rt/lib/xray/xray_defs.h b/contrib/compiler-rt/lib/xray/xray_defs.h index e5c37c0665db..c009bcc879f1 100644 --- a/contrib/compiler-rt/lib/xray/xray_defs.h +++ b/contrib/compiler-rt/lib/xray/xray_defs.h @@ -19,4 +19,14 @@ #define XRAY_NEVER_INSTRUMENT #endif +#if SANITIZER_NETBSD +// NetBSD: thread_local is not aligned properly, and the code relying +// on it segfaults +#define XRAY_TLS_ALIGNAS(x) +#define XRAY_HAS_TLS_ALIGNAS 0 +#else +#define XRAY_TLS_ALIGNAS(x) alignas(x) +#define XRAY_HAS_TLS_ALIGNAS 1 +#endif + #endif // XRAY_XRAY_DEFS_H diff --git a/contrib/compiler-rt/lib/xray/xray_fdr_controller.h b/contrib/compiler-rt/lib/xray/xray_fdr_controller.h new file mode 100644 index 000000000000..d44d0309b373 --- /dev/null +++ b/contrib/compiler-rt/lib/xray/xray_fdr_controller.h @@ -0,0 +1,373 @@ +//===-- xray_fdr_controller.h ---------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of XRay, a function call tracing system. +// +//===----------------------------------------------------------------------===// +#ifndef COMPILER_RT_LIB_XRAY_XRAY_FDR_CONTROLLER_H_ +#define COMPILER_RT_LIB_XRAY_XRAY_FDR_CONTROLLER_H_ + +#include <limits> +#include <time.h> + +#include "xray/xray_interface.h" +#include "xray/xray_records.h" +#include "xray_buffer_queue.h" +#include "xray_fdr_log_writer.h" + +namespace __xray { + +template <size_t Version = 5> class FDRController { + BufferQueue *BQ; + BufferQueue::Buffer &B; + FDRLogWriter &W; + int (*WallClockReader)(clockid_t, struct timespec *) = 0; + uint64_t CycleThreshold = 0; + + uint64_t LastFunctionEntryTSC = 0; + uint64_t LatestTSC = 0; + uint16_t LatestCPU = 0; + tid_t TId = 0; + pid_t PId = 0; + bool First = true; + + uint32_t UndoableFunctionEnters = 0; + uint32_t UndoableTailExits = 0; + + bool finalized() const XRAY_NEVER_INSTRUMENT { + return BQ == nullptr || BQ->finalizing(); + } + + bool hasSpace(size_t S) XRAY_NEVER_INSTRUMENT { + return B.Data != nullptr && B.Generation == BQ->generation() && + W.getNextRecord() + S <= reinterpret_cast<char *>(B.Data) + B.Size; + } + + constexpr int32_t mask(int32_t FuncId) const XRAY_NEVER_INSTRUMENT { + return FuncId & ((1 << 29) - 1); + } + + bool getNewBuffer() XRAY_NEVER_INSTRUMENT { + if (BQ->getBuffer(B) != BufferQueue::ErrorCode::Ok) + return false; + + W.resetRecord(); + DCHECK_EQ(W.getNextRecord(), B.Data); + LatestTSC = 0; + LatestCPU = 0; + First = true; + UndoableFunctionEnters = 0; + UndoableTailExits = 0; + atomic_store(B.Extents, 0, memory_order_release); + return true; + } + + bool setupNewBuffer() XRAY_NEVER_INSTRUMENT { + if (finalized()) + return false; + + DCHECK(hasSpace(sizeof(MetadataRecord) * 3)); + TId = GetTid(); + PId = internal_getpid(); + struct timespec TS { + 0, 0 + }; + WallClockReader(CLOCK_MONOTONIC, &TS); + + MetadataRecord Metadata[] = { + // Write out a MetadataRecord to signify that this is the start of a new + // buffer, associated with a particular thread, with a new CPU. For the + // data, we have 15 bytes to squeeze as much information as we can. At + // this point we only write down the following bytes: + // - Thread ID (tid_t, cast to 4 bytes type due to Darwin being 8 + // bytes) + createMetadataRecord<MetadataRecord::RecordKinds::NewBuffer>( + static_cast<int32_t>(TId)), + + // Also write the WalltimeMarker record. We only really need microsecond + // precision here, and enforce across platforms that we need 64-bit + // seconds and 32-bit microseconds encoded in the Metadata record. + createMetadataRecord<MetadataRecord::RecordKinds::WalltimeMarker>( + static_cast<int64_t>(TS.tv_sec), + static_cast<int32_t>(TS.tv_nsec / 1000)), + + // Also write the Pid record. + createMetadataRecord<MetadataRecord::RecordKinds::Pid>( + static_cast<int32_t>(PId)), + }; + + if (finalized()) + return false; + return W.writeMetadataRecords(Metadata); + } + + bool prepareBuffer(size_t S) XRAY_NEVER_INSTRUMENT { + if (finalized()) + return returnBuffer(); + + if (UNLIKELY(!hasSpace(S))) { + if (!returnBuffer()) + return false; + if (!getNewBuffer()) + return false; + if (!setupNewBuffer()) + return false; + } + + if (First) { + First = false; + W.resetRecord(); + atomic_store(B.Extents, 0, memory_order_release); + return setupNewBuffer(); + } + + return true; + } + + bool returnBuffer() XRAY_NEVER_INSTRUMENT { + if (BQ == nullptr) + return false; + + First = true; + if (finalized()) { + BQ->releaseBuffer(B); // ignore result. + return false; + } + + return BQ->releaseBuffer(B) == BufferQueue::ErrorCode::Ok; + } + + enum class PreambleResult { NoChange, WroteMetadata, InvalidBuffer }; + PreambleResult recordPreamble(uint64_t TSC, + uint16_t CPU) XRAY_NEVER_INSTRUMENT { + if (UNLIKELY(LatestCPU != CPU || LatestTSC == 0)) { + // We update our internal tracking state for the Latest TSC and CPU we've + // seen, then write out the appropriate metadata and function records. + LatestTSC = TSC; + LatestCPU = CPU; + + if (B.Generation != BQ->generation()) + return PreambleResult::InvalidBuffer; + + W.writeMetadata<MetadataRecord::RecordKinds::NewCPUId>(CPU, TSC); + return PreambleResult::WroteMetadata; + } + + DCHECK_EQ(LatestCPU, CPU); + + if (UNLIKELY(LatestTSC > TSC || + TSC - LatestTSC > + uint64_t{std::numeric_limits<int32_t>::max()})) { + // Either the TSC has wrapped around from the last TSC we've seen or the + // delta is too large to fit in a 32-bit signed integer, so we write a + // wrap-around record. + LatestTSC = TSC; + + if (B.Generation != BQ->generation()) + return PreambleResult::InvalidBuffer; + + W.writeMetadata<MetadataRecord::RecordKinds::TSCWrap>(TSC); + return PreambleResult::WroteMetadata; + } + + return PreambleResult::NoChange; + } + + bool rewindRecords(int32_t FuncId, uint64_t TSC, + uint16_t CPU) XRAY_NEVER_INSTRUMENT { + // Undo one enter record, because at this point we are either at the state + // of: + // - We are exiting a function that we recently entered. + // - We are exiting a function that was the result of a sequence of tail + // exits, and we can check whether the tail exits can be re-wound. + // + FunctionRecord F; + W.undoWrites(sizeof(FunctionRecord)); + if (B.Generation != BQ->generation()) + return false; + internal_memcpy(&F, W.getNextRecord(), sizeof(FunctionRecord)); + + DCHECK(F.RecordKind == + uint8_t(FunctionRecord::RecordKinds::FunctionEnter) && + "Expected to find function entry recording when rewinding."); + DCHECK_EQ(F.FuncId, FuncId & ~(0x0F << 28)); + + LatestTSC -= F.TSCDelta; + if (--UndoableFunctionEnters != 0) { + LastFunctionEntryTSC -= F.TSCDelta; + return true; + } + + LastFunctionEntryTSC = 0; + auto RewindingTSC = LatestTSC; + auto RewindingRecordPtr = W.getNextRecord() - sizeof(FunctionRecord); + while (UndoableTailExits) { + if (B.Generation != BQ->generation()) + return false; + internal_memcpy(&F, RewindingRecordPtr, sizeof(FunctionRecord)); + DCHECK_EQ(F.RecordKind, + uint8_t(FunctionRecord::RecordKinds::FunctionTailExit)); + RewindingTSC -= F.TSCDelta; + RewindingRecordPtr -= sizeof(FunctionRecord); + if (B.Generation != BQ->generation()) + return false; + internal_memcpy(&F, RewindingRecordPtr, sizeof(FunctionRecord)); + + // This tail call exceeded the threshold duration. It will not be erased. + if ((TSC - RewindingTSC) >= CycleThreshold) { + UndoableTailExits = 0; + return true; + } + + --UndoableTailExits; + W.undoWrites(sizeof(FunctionRecord) * 2); + LatestTSC = RewindingTSC; + } + return true; + } + +public: + template <class WallClockFunc> + FDRController(BufferQueue *BQ, BufferQueue::Buffer &B, FDRLogWriter &W, + WallClockFunc R, uint64_t C) XRAY_NEVER_INSTRUMENT + : BQ(BQ), + B(B), + W(W), + WallClockReader(R), + CycleThreshold(C) {} + + bool functionEnter(int32_t FuncId, uint64_t TSC, + uint16_t CPU) XRAY_NEVER_INSTRUMENT { + if (finalized() || + !prepareBuffer(sizeof(MetadataRecord) + sizeof(FunctionRecord))) + return returnBuffer(); + + auto PreambleStatus = recordPreamble(TSC, CPU); + if (PreambleStatus == PreambleResult::InvalidBuffer) + return returnBuffer(); + + if (PreambleStatus == PreambleResult::WroteMetadata) { + UndoableFunctionEnters = 1; + UndoableTailExits = 0; + } else { + ++UndoableFunctionEnters; + } + + auto Delta = TSC - LatestTSC; + LastFunctionEntryTSC = TSC; + LatestTSC = TSC; + return W.writeFunction(FDRLogWriter::FunctionRecordKind::Enter, + mask(FuncId), Delta); + } + + bool functionTailExit(int32_t FuncId, uint64_t TSC, + uint16_t CPU) XRAY_NEVER_INSTRUMENT { + if (finalized()) + return returnBuffer(); + + if (!prepareBuffer(sizeof(MetadataRecord) + sizeof(FunctionRecord))) + return returnBuffer(); + + auto PreambleStatus = recordPreamble(TSC, CPU); + if (PreambleStatus == PreambleResult::InvalidBuffer) + return returnBuffer(); + + if (PreambleStatus == PreambleResult::NoChange && + UndoableFunctionEnters != 0 && + TSC - LastFunctionEntryTSC < CycleThreshold) + return rewindRecords(FuncId, TSC, CPU); + + UndoableTailExits = UndoableFunctionEnters ? UndoableTailExits + 1 : 0; + UndoableFunctionEnters = 0; + auto Delta = TSC - LatestTSC; + LatestTSC = TSC; + return W.writeFunction(FDRLogWriter::FunctionRecordKind::TailExit, + mask(FuncId), Delta); + } + + bool functionEnterArg(int32_t FuncId, uint64_t TSC, uint16_t CPU, + uint64_t Arg) XRAY_NEVER_INSTRUMENT { + if (finalized() || + !prepareBuffer((2 * sizeof(MetadataRecord)) + sizeof(FunctionRecord)) || + recordPreamble(TSC, CPU) == PreambleResult::InvalidBuffer) + return returnBuffer(); + + auto Delta = TSC - LatestTSC; + LatestTSC = TSC; + LastFunctionEntryTSC = 0; + UndoableFunctionEnters = 0; + UndoableTailExits = 0; + + return W.writeFunctionWithArg(FDRLogWriter::FunctionRecordKind::EnterArg, + mask(FuncId), Delta, Arg); + } + + bool functionExit(int32_t FuncId, uint64_t TSC, + uint16_t CPU) XRAY_NEVER_INSTRUMENT { + if (finalized() || + !prepareBuffer(sizeof(MetadataRecord) + sizeof(FunctionRecord))) + return returnBuffer(); + + auto PreambleStatus = recordPreamble(TSC, CPU); + if (PreambleStatus == PreambleResult::InvalidBuffer) + return returnBuffer(); + + if (PreambleStatus == PreambleResult::NoChange && + UndoableFunctionEnters != 0 && + TSC - LastFunctionEntryTSC < CycleThreshold) + return rewindRecords(FuncId, TSC, CPU); + + auto Delta = TSC - LatestTSC; + LatestTSC = TSC; + UndoableFunctionEnters = 0; + UndoableTailExits = 0; + return W.writeFunction(FDRLogWriter::FunctionRecordKind::Exit, mask(FuncId), + Delta); + } + + bool customEvent(uint64_t TSC, uint16_t CPU, const void *Event, + int32_t EventSize) XRAY_NEVER_INSTRUMENT { + if (finalized() || + !prepareBuffer((2 * sizeof(MetadataRecord)) + EventSize) || + recordPreamble(TSC, CPU) == PreambleResult::InvalidBuffer) + return returnBuffer(); + + auto Delta = TSC - LatestTSC; + LatestTSC = TSC; + UndoableFunctionEnters = 0; + UndoableTailExits = 0; + return W.writeCustomEvent(Delta, Event, EventSize); + } + + bool typedEvent(uint64_t TSC, uint16_t CPU, uint16_t EventType, + const void *Event, int32_t EventSize) XRAY_NEVER_INSTRUMENT { + if (finalized() || + !prepareBuffer((2 * sizeof(MetadataRecord)) + EventSize) || + recordPreamble(TSC, CPU) == PreambleResult::InvalidBuffer) + return returnBuffer(); + + auto Delta = TSC - LatestTSC; + LatestTSC = TSC; + UndoableFunctionEnters = 0; + UndoableTailExits = 0; + return W.writeTypedEvent(Delta, EventType, Event, EventSize); + } + + bool flush() XRAY_NEVER_INSTRUMENT { + if (finalized()) { + returnBuffer(); // ignore result. + return true; + } + return returnBuffer(); + } +}; + +} // namespace __xray + +#endif // COMPILER-RT_LIB_XRAY_XRAY_FDR_CONTROLLER_H_ diff --git a/contrib/compiler-rt/lib/xray/xray_fdr_log_records.h b/contrib/compiler-rt/lib/xray/xray_fdr_log_records.h index 87096d4fc29e..e7b1ee562e1b 100644 --- a/contrib/compiler-rt/lib/xray/xray_fdr_log_records.h +++ b/contrib/compiler-rt/lib/xray/xray_fdr_log_records.h @@ -12,6 +12,9 @@ //===----------------------------------------------------------------------===// #ifndef XRAY_XRAY_FDR_LOG_RECORDS_H #define XRAY_XRAY_FDR_LOG_RECORDS_H +#include <cstdint> + +namespace __xray { enum class RecordType : uint8_t { Function, Metadata }; @@ -68,4 +71,6 @@ struct alignas(8) FunctionRecord { static_assert(sizeof(FunctionRecord) == 8, "Wrong size for FunctionRecord."); +} // namespace __xray + #endif // XRAY_XRAY_FDR_LOG_RECORDS_H diff --git a/contrib/compiler-rt/lib/xray/xray_fdr_log_writer.h b/contrib/compiler-rt/lib/xray/xray_fdr_log_writer.h new file mode 100644 index 000000000000..7712e1377763 --- /dev/null +++ b/contrib/compiler-rt/lib/xray/xray_fdr_log_writer.h @@ -0,0 +1,232 @@ +//===-- xray_fdr_log_writer.h ---------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of XRay, a function call tracing system. +// +//===----------------------------------------------------------------------===// +#ifndef COMPILER_RT_LIB_XRAY_XRAY_FDR_LOG_WRITER_H_ +#define COMPILER_RT_LIB_XRAY_XRAY_FDR_LOG_WRITER_H_ + +#include "xray_buffer_queue.h" +#include "xray_fdr_log_records.h" +#include <functional> +#include <tuple> +#include <type_traits> +#include <utility> + +namespace __xray { + +template <size_t Index> struct SerializerImpl { + template <class Tuple, + typename std::enable_if< + Index<std::tuple_size< + typename std::remove_reference<Tuple>::type>::value, + int>::type = 0> static void serializeTo(char *Buffer, + Tuple &&T) { + auto P = reinterpret_cast<const char *>(&std::get<Index>(T)); + constexpr auto Size = sizeof(std::get<Index>(T)); + internal_memcpy(Buffer, P, Size); + SerializerImpl<Index + 1>::serializeTo(Buffer + Size, + std::forward<Tuple>(T)); + } + + template <class Tuple, + typename std::enable_if< + Index >= std::tuple_size<typename std::remove_reference< + Tuple>::type>::value, + int>::type = 0> + static void serializeTo(char *, Tuple &&) {} +}; + +using Serializer = SerializerImpl<0>; + +template <class Tuple, size_t Index> struct AggregateSizesImpl { + static constexpr size_t value = + sizeof(typename std::tuple_element<Index, Tuple>::type) + + AggregateSizesImpl<Tuple, Index - 1>::value; +}; + +template <class Tuple> struct AggregateSizesImpl<Tuple, 0> { + static constexpr size_t value = + sizeof(typename std::tuple_element<0, Tuple>::type); +}; + +template <class Tuple> struct AggregateSizes { + static constexpr size_t value = + AggregateSizesImpl<Tuple, std::tuple_size<Tuple>::value - 1>::value; +}; + +template <MetadataRecord::RecordKinds Kind, class... DataTypes> +MetadataRecord createMetadataRecord(DataTypes &&... Ds) { + static_assert(AggregateSizes<std::tuple<DataTypes...>>::value <= + sizeof(MetadataRecord) - 1, + "Metadata payload longer than metadata buffer!"); + MetadataRecord R; + R.Type = 1; + R.RecordKind = static_cast<uint8_t>(Kind); + Serializer::serializeTo(R.Data, + std::make_tuple(std::forward<DataTypes>(Ds)...)); + return R; +} + +class FDRLogWriter { + BufferQueue::Buffer &Buffer; + char *NextRecord = nullptr; + + template <class T> void writeRecord(const T &R) { + internal_memcpy(NextRecord, reinterpret_cast<const char *>(&R), sizeof(T)); + NextRecord += sizeof(T); + // We need this atomic fence here to ensure that other threads attempting to + // read the bytes in the buffer will see the writes committed before the + // extents are updated. + atomic_thread_fence(memory_order_release); + atomic_fetch_add(Buffer.Extents, sizeof(T), memory_order_acq_rel); + } + +public: + explicit FDRLogWriter(BufferQueue::Buffer &B, char *P) + : Buffer(B), NextRecord(P) { + DCHECK_NE(Buffer.Data, nullptr); + DCHECK_NE(NextRecord, nullptr); + } + + explicit FDRLogWriter(BufferQueue::Buffer &B) + : FDRLogWriter(B, static_cast<char *>(B.Data)) {} + + template <MetadataRecord::RecordKinds Kind, class... Data> + bool writeMetadata(Data &&... Ds) { + // TODO: Check boundary conditions: + // 1) Buffer is full, and cannot handle one metadata record. + // 2) Buffer queue is finalising. + writeRecord(createMetadataRecord<Kind>(std::forward<Data>(Ds)...)); + return true; + } + + template <size_t N> size_t writeMetadataRecords(MetadataRecord (&Recs)[N]) { + constexpr auto Size = sizeof(MetadataRecord) * N; + internal_memcpy(NextRecord, reinterpret_cast<const char *>(Recs), Size); + NextRecord += Size; + // We need this atomic fence here to ensure that other threads attempting to + // read the bytes in the buffer will see the writes committed before the + // extents are updated. + atomic_thread_fence(memory_order_release); + atomic_fetch_add(Buffer.Extents, Size, memory_order_acq_rel); + return Size; + } + + enum class FunctionRecordKind : uint8_t { + Enter = 0x00, + Exit = 0x01, + TailExit = 0x02, + EnterArg = 0x03, + }; + + bool writeFunction(FunctionRecordKind Kind, int32_t FuncId, int32_t Delta) { + FunctionRecord R; + R.Type = 0; + R.RecordKind = uint8_t(Kind); + R.FuncId = FuncId; + R.TSCDelta = Delta; + writeRecord(R); + return true; + } + + bool writeFunctionWithArg(FunctionRecordKind Kind, int32_t FuncId, + int32_t Delta, uint64_t Arg) { + // We need to write the function with arg into the buffer, and then + // atomically update the buffer extents. This ensures that any reads + // synchronised on the buffer extents record will always see the writes + // that happen before the atomic update. + FunctionRecord R; + R.Type = 0; + R.RecordKind = uint8_t(Kind); + R.FuncId = FuncId; + R.TSCDelta = Delta; + MetadataRecord A = + createMetadataRecord<MetadataRecord::RecordKinds::CallArgument>(Arg); + NextRecord = reinterpret_cast<char *>(internal_memcpy( + NextRecord, reinterpret_cast<char *>(&R), sizeof(R))) + + sizeof(R); + NextRecord = reinterpret_cast<char *>(internal_memcpy( + NextRecord, reinterpret_cast<char *>(&A), sizeof(A))) + + sizeof(A); + // We need this atomic fence here to ensure that other threads attempting to + // read the bytes in the buffer will see the writes committed before the + // extents are updated. + atomic_thread_fence(memory_order_release); + atomic_fetch_add(Buffer.Extents, sizeof(R) + sizeof(A), + memory_order_acq_rel); + return true; + } + + bool writeCustomEvent(int32_t Delta, const void *Event, int32_t EventSize) { + // We write the metadata record and the custom event data into the buffer + // first, before we atomically update the extents for the buffer. This + // allows us to ensure that any threads reading the extents of the buffer + // will only ever see the full metadata and custom event payload accounted + // (no partial writes accounted). + MetadataRecord R = + createMetadataRecord<MetadataRecord::RecordKinds::CustomEventMarker>( + EventSize, Delta); + NextRecord = reinterpret_cast<char *>(internal_memcpy( + NextRecord, reinterpret_cast<char *>(&R), sizeof(R))) + + sizeof(R); + NextRecord = reinterpret_cast<char *>( + internal_memcpy(NextRecord, Event, EventSize)) + + EventSize; + + // We need this atomic fence here to ensure that other threads attempting to + // read the bytes in the buffer will see the writes committed before the + // extents are updated. + atomic_thread_fence(memory_order_release); + atomic_fetch_add(Buffer.Extents, sizeof(R) + EventSize, + memory_order_acq_rel); + return true; + } + + bool writeTypedEvent(int32_t Delta, uint16_t EventType, const void *Event, + int32_t EventSize) { + // We do something similar when writing out typed events, see + // writeCustomEvent(...) above for details. + MetadataRecord R = + createMetadataRecord<MetadataRecord::RecordKinds::TypedEventMarker>( + EventSize, Delta, EventType); + NextRecord = reinterpret_cast<char *>(internal_memcpy( + NextRecord, reinterpret_cast<char *>(&R), sizeof(R))) + + sizeof(R); + NextRecord = reinterpret_cast<char *>( + internal_memcpy(NextRecord, Event, EventSize)) + + EventSize; + + // We need this atomic fence here to ensure that other threads attempting to + // read the bytes in the buffer will see the writes committed before the + // extents are updated. + atomic_thread_fence(memory_order_release); + atomic_fetch_add(Buffer.Extents, EventSize, memory_order_acq_rel); + return true; + } + + char *getNextRecord() const { return NextRecord; } + + void resetRecord() { + NextRecord = reinterpret_cast<char *>(Buffer.Data); + atomic_store(Buffer.Extents, 0, memory_order_release); + } + + void undoWrites(size_t B) { + DCHECK_GE(NextRecord - B, reinterpret_cast<char *>(Buffer.Data)); + NextRecord -= B; + atomic_fetch_sub(Buffer.Extents, B, memory_order_acq_rel); + } + +}; // namespace __xray + +} // namespace __xray + +#endif // COMPILER-RT_LIB_XRAY_XRAY_FDR_LOG_WRITER_H_ diff --git a/contrib/compiler-rt/lib/xray/xray_fdr_logging.cc b/contrib/compiler-rt/lib/xray/xray_fdr_logging.cc index 6cb2dfa0c658..1eda26df7a85 100644 --- a/contrib/compiler-rt/lib/xray/xray_fdr_logging.cc +++ b/contrib/compiler-rt/lib/xray/xray_fdr_logging.cc @@ -20,7 +20,6 @@ #include <limits> #include <memory> #include <pthread.h> -#include <sys/syscall.h> #include <sys/time.h> #include <time.h> #include <unistd.h> @@ -30,9 +29,12 @@ #include "sanitizer_common/sanitizer_common.h" #include "xray/xray_interface.h" #include "xray/xray_records.h" +#include "xray_allocator.h" #include "xray_buffer_queue.h" #include "xray_defs.h" +#include "xray_fdr_controller.h" #include "xray_fdr_flags.h" +#include "xray_fdr_log_writer.h" #include "xray_flags.h" #include "xray_recursion_guard.h" #include "xray_tsc.h" @@ -40,55 +42,53 @@ namespace __xray { -atomic_sint32_t LoggingStatus = {XRayLogInitStatus::XRAY_LOG_UNINITIALIZED}; +static atomic_sint32_t LoggingStatus = { + XRayLogInitStatus::XRAY_LOG_UNINITIALIZED}; + +namespace { // Group together thread-local-data in a struct, then hide it behind a function // call so that it can be initialized on first use instead of as a global. We // force the alignment to 64-bytes for x86 cache line alignment, as this // structure is used in the hot path of implementation. -struct alignas(64) ThreadLocalData { - BufferQueue::Buffer Buffer; - char *RecordPtr = nullptr; - // The number of FunctionEntry records immediately preceding RecordPtr. - uint8_t NumConsecutiveFnEnters = 0; - - // The number of adjacent, consecutive pairs of FunctionEntry, Tail Exit - // records preceding RecordPtr. - uint8_t NumTailCalls = 0; - - // We use a thread_local variable to keep track of which CPUs we've already - // run, and the TSC times for these CPUs. This allows us to stop repeating the - // CPU field in the function records. - // - // We assume that we'll support only 65536 CPUs for x86_64. - uint16_t CurrentCPU = std::numeric_limits<uint16_t>::max(); - uint64_t LastTSC = 0; - uint64_t LastFunctionEntryTSC = 0; - - // Make sure a thread that's ever called handleArg0 has a thread-local - // live reference to the buffer queue for this particular instance of - // FDRLogging, and that we're going to clean it up when the thread exits. +struct XRAY_TLS_ALIGNAS(64) ThreadLocalData { + BufferQueue::Buffer Buffer{}; BufferQueue *BQ = nullptr; + + using LogWriterStorage = + typename std::aligned_storage<sizeof(FDRLogWriter), + alignof(FDRLogWriter)>::type; + + LogWriterStorage LWStorage; + FDRLogWriter *Writer = nullptr; + + using ControllerStorage = + typename std::aligned_storage<sizeof(FDRController<>), + alignof(FDRController<>)>::type; + ControllerStorage CStorage; + FDRController<> *Controller = nullptr; }; +} // namespace + static_assert(std::is_trivially_destructible<ThreadLocalData>::value, "ThreadLocalData must be trivially destructible"); -static constexpr auto MetadataRecSize = sizeof(MetadataRecord); -static constexpr auto FunctionRecSize = sizeof(FunctionRecord); - // Use a global pthread key to identify thread-local data for logging. static pthread_key_t Key; // Global BufferQueue. +static std::aligned_storage<sizeof(BufferQueue)>::type BufferQueueStorage; static BufferQueue *BQ = nullptr; -static atomic_sint32_t LogFlushStatus = { - XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING}; +// Global thresholds for function durations. +static atomic_uint64_t ThresholdTicks{0}; -static FDRLoggingOptions FDROptions; +// Global for ticks per second. +static atomic_uint64_t TicksPerSec{0}; -static SpinMutex FDROptionsMutex; +static atomic_sint32_t LogFlushStatus = { + XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING}; // This function will initialize the thread-local data structure used by the FDR // logging implementation and return a reference to it. The implementation @@ -124,8 +124,10 @@ static SpinMutex FDROptionsMutex; // critical section, calling a function that might be XRay instrumented (and // thus in turn calling into malloc by virtue of registration of the // thread_local's destructor). +#if XRAY_HAS_TLS_ALIGNAS static_assert(alignof(ThreadLocalData) >= 64, "ThreadLocalData must be cache line aligned."); +#endif static ThreadLocalData &getThreadLocalData() { thread_local typename std::aligned_storage< sizeof(ThreadLocalData), alignof(ThreadLocalData)>::type TLDStorage{}; @@ -138,559 +140,36 @@ static ThreadLocalData &getThreadLocalData() { return *reinterpret_cast<ThreadLocalData *>(&TLDStorage); } -static void writeNewBufferPreamble(tid_t Tid, timespec TS, - pid_t Pid) XRAY_NEVER_INSTRUMENT { - static constexpr int InitRecordsCount = 3; - auto &TLD = getThreadLocalData(); - MetadataRecord Metadata[InitRecordsCount]; - { - // Write out a MetadataRecord to signify that this is the start of a new - // buffer, associated with a particular thread, with a new CPU. For the - // data, we have 15 bytes to squeeze as much information as we can. At this - // point we only write down the following bytes: - // - Thread ID (tid_t, cast to 4 bytes type due to Darwin being 8 bytes) - auto &NewBuffer = Metadata[0]; - NewBuffer.Type = uint8_t(RecordType::Metadata); - NewBuffer.RecordKind = uint8_t(MetadataRecord::RecordKinds::NewBuffer); - int32_t tid = static_cast<int32_t>(Tid); - internal_memcpy(&NewBuffer.Data, &tid, sizeof(tid)); - } - - // Also write the WalltimeMarker record. - { - static_assert(sizeof(time_t) <= 8, "time_t needs to be at most 8 bytes"); - auto &WalltimeMarker = Metadata[1]; - WalltimeMarker.Type = uint8_t(RecordType::Metadata); - WalltimeMarker.RecordKind = - uint8_t(MetadataRecord::RecordKinds::WalltimeMarker); - - // We only really need microsecond precision here, and enforce across - // platforms that we need 64-bit seconds and 32-bit microseconds encoded in - // the Metadata record. - int32_t Micros = TS.tv_nsec / 1000; - int64_t Seconds = TS.tv_sec; - internal_memcpy(WalltimeMarker.Data, &Seconds, sizeof(Seconds)); - internal_memcpy(WalltimeMarker.Data + sizeof(Seconds), &Micros, - sizeof(Micros)); - } - - // Also write the Pid record. - { - // Write out a MetadataRecord that contains the current pid - auto &PidMetadata = Metadata[2]; - PidMetadata.Type = uint8_t(RecordType::Metadata); - PidMetadata.RecordKind = uint8_t(MetadataRecord::RecordKinds::Pid); - int32_t pid = static_cast<int32_t>(Pid); - internal_memcpy(&PidMetadata.Data, &pid, sizeof(pid)); - } - - TLD.NumConsecutiveFnEnters = 0; - TLD.NumTailCalls = 0; - if (TLD.BQ == nullptr || TLD.BQ->finalizing()) - return; - internal_memcpy(TLD.RecordPtr, Metadata, sizeof(Metadata)); - TLD.RecordPtr += sizeof(Metadata); - // Since we write out the extents as the first metadata record of the - // buffer, we need to write out the extents including the extents record. - atomic_store(&TLD.Buffer.Extents->Size, sizeof(Metadata), - memory_order_release); -} - -static void setupNewBuffer(int (*wall_clock_reader)( - clockid_t, struct timespec *)) XRAY_NEVER_INSTRUMENT { - auto &TLD = getThreadLocalData(); - auto &B = TLD.Buffer; - TLD.RecordPtr = static_cast<char *>(B.Data); - tid_t Tid = GetTid(); - timespec TS{0, 0}; - pid_t Pid = internal_getpid(); - // This is typically clock_gettime, but callers have injection ability. - wall_clock_reader(CLOCK_MONOTONIC, &TS); - writeNewBufferPreamble(Tid, TS, Pid); - TLD.NumConsecutiveFnEnters = 0; - TLD.NumTailCalls = 0; -} - -static void incrementExtents(size_t Add) { - auto &TLD = getThreadLocalData(); - atomic_fetch_add(&TLD.Buffer.Extents->Size, Add, memory_order_acq_rel); -} - -static void decrementExtents(size_t Subtract) { - auto &TLD = getThreadLocalData(); - atomic_fetch_sub(&TLD.Buffer.Extents->Size, Subtract, memory_order_acq_rel); -} - -static void writeNewCPUIdMetadata(uint16_t CPU, - uint64_t TSC) XRAY_NEVER_INSTRUMENT { - auto &TLD = getThreadLocalData(); - MetadataRecord NewCPUId; - NewCPUId.Type = uint8_t(RecordType::Metadata); - NewCPUId.RecordKind = uint8_t(MetadataRecord::RecordKinds::NewCPUId); - - // The data for the New CPU will contain the following bytes: - // - CPU ID (uint16_t, 2 bytes) - // - Full TSC (uint64_t, 8 bytes) - // Total = 10 bytes. - internal_memcpy(&NewCPUId.Data, &CPU, sizeof(CPU)); - internal_memcpy(&NewCPUId.Data[sizeof(CPU)], &TSC, sizeof(TSC)); - internal_memcpy(TLD.RecordPtr, &NewCPUId, sizeof(MetadataRecord)); - TLD.RecordPtr += sizeof(MetadataRecord); - TLD.NumConsecutiveFnEnters = 0; - TLD.NumTailCalls = 0; - incrementExtents(sizeof(MetadataRecord)); -} - -static void writeTSCWrapMetadata(uint64_t TSC) XRAY_NEVER_INSTRUMENT { - auto &TLD = getThreadLocalData(); - MetadataRecord TSCWrap; - TSCWrap.Type = uint8_t(RecordType::Metadata); - TSCWrap.RecordKind = uint8_t(MetadataRecord::RecordKinds::TSCWrap); - - // The data for the TSCWrap record contains the following bytes: - // - Full TSC (uint64_t, 8 bytes) - // Total = 8 bytes. - internal_memcpy(&TSCWrap.Data, &TSC, sizeof(TSC)); - internal_memcpy(TLD.RecordPtr, &TSCWrap, sizeof(MetadataRecord)); - TLD.RecordPtr += sizeof(MetadataRecord); - TLD.NumConsecutiveFnEnters = 0; - TLD.NumTailCalls = 0; - incrementExtents(sizeof(MetadataRecord)); -} - -// Call Argument metadata records store the arguments to a function in the -// order of their appearance; holes are not supported by the buffer format. -static void writeCallArgumentMetadata(uint64_t A) XRAY_NEVER_INSTRUMENT { - auto &TLD = getThreadLocalData(); - MetadataRecord CallArg; - CallArg.Type = uint8_t(RecordType::Metadata); - CallArg.RecordKind = uint8_t(MetadataRecord::RecordKinds::CallArgument); - - internal_memcpy(CallArg.Data, &A, sizeof(A)); - internal_memcpy(TLD.RecordPtr, &CallArg, sizeof(MetadataRecord)); - TLD.RecordPtr += sizeof(MetadataRecord); - incrementExtents(sizeof(MetadataRecord)); -} - -static void writeFunctionRecord(int FuncId, uint32_t TSCDelta, - XRayEntryType EntryType) XRAY_NEVER_INSTRUMENT { - FunctionRecord FuncRecord; - FuncRecord.Type = uint8_t(RecordType::Function); - // Only take 28 bits of the function id. - FuncRecord.FuncId = FuncId & ~(0x0F << 28); - FuncRecord.TSCDelta = TSCDelta; - - auto &TLD = getThreadLocalData(); - switch (EntryType) { - case XRayEntryType::ENTRY: - ++TLD.NumConsecutiveFnEnters; - FuncRecord.RecordKind = uint8_t(FunctionRecord::RecordKinds::FunctionEnter); - break; - case XRayEntryType::LOG_ARGS_ENTRY: - // We should not rewind functions with logged args. - TLD.NumConsecutiveFnEnters = 0; - TLD.NumTailCalls = 0; - FuncRecord.RecordKind = uint8_t(FunctionRecord::RecordKinds::FunctionEnter); - break; - case XRayEntryType::EXIT: - // If we've decided to log the function exit, we will never erase the log - // before it. - TLD.NumConsecutiveFnEnters = 0; - TLD.NumTailCalls = 0; - FuncRecord.RecordKind = uint8_t(FunctionRecord::RecordKinds::FunctionExit); - break; - case XRayEntryType::TAIL: - // If we just entered the function we're tail exiting from or erased every - // invocation since then, this function entry tail pair is a candidate to - // be erased when the child function exits. - if (TLD.NumConsecutiveFnEnters > 0) { - ++TLD.NumTailCalls; - TLD.NumConsecutiveFnEnters = 0; - } else { - // We will never be able to erase this tail call since we have logged - // something in between the function entry and tail exit. - TLD.NumTailCalls = 0; - TLD.NumConsecutiveFnEnters = 0; - } - FuncRecord.RecordKind = - uint8_t(FunctionRecord::RecordKinds::FunctionTailExit); - break; - case XRayEntryType::CUSTOM_EVENT: { - // This is a bug in patching, so we'll report it once and move on. - static atomic_uint8_t ErrorLatch{0}; - if (!atomic_exchange(&ErrorLatch, 1, memory_order_acq_rel)) - Report("Internal error: patched an XRay custom event call as a function; " - "func id = %d\n", - FuncId); - return; - } - case XRayEntryType::TYPED_EVENT: { - static atomic_uint8_t ErrorLatch{0}; - if (!atomic_exchange(&ErrorLatch, 1, memory_order_acq_rel)) - Report("Internal error: patched an XRay typed event call as a function; " - "func id = %d\n", - FuncId); - return; - } - } - - internal_memcpy(TLD.RecordPtr, &FuncRecord, sizeof(FunctionRecord)); - TLD.RecordPtr += sizeof(FunctionRecord); - incrementExtents(sizeof(FunctionRecord)); -} - -static atomic_uint64_t TicksPerSec{0}; -static atomic_uint64_t ThresholdTicks{0}; - -// Re-point the thread local pointer into this thread's Buffer before the recent -// "Function Entry" record and any "Tail Call Exit" records after that. -static void rewindRecentCall(uint64_t TSC, uint64_t &LastTSC, - uint64_t &LastFunctionEntryTSC, int32_t FuncId) { - auto &TLD = getThreadLocalData(); - TLD.RecordPtr -= FunctionRecSize; - decrementExtents(FunctionRecSize); - FunctionRecord FuncRecord; - internal_memcpy(&FuncRecord, TLD.RecordPtr, FunctionRecSize); - DCHECK(FuncRecord.RecordKind == - uint8_t(FunctionRecord::RecordKinds::FunctionEnter) && - "Expected to find function entry recording when rewinding."); - DCHECK(FuncRecord.FuncId == (FuncId & ~(0x0F << 28)) && - "Expected matching function id when rewinding Exit"); - --TLD.NumConsecutiveFnEnters; - LastTSC -= FuncRecord.TSCDelta; - - // We unwound one call. Update the state and return without writing a log. - if (TLD.NumConsecutiveFnEnters != 0) { - LastFunctionEntryTSC -= FuncRecord.TSCDelta; - return; - } - - // Otherwise we've rewound the stack of all function entries, we might be - // able to rewind further by erasing tail call functions that are being - // exited from via this exit. - LastFunctionEntryTSC = 0; - auto RewindingTSC = LastTSC; - auto RewindingRecordPtr = TLD.RecordPtr - FunctionRecSize; - while (TLD.NumTailCalls > 0) { - // Rewind the TSC back over the TAIL EXIT record. - FunctionRecord ExpectedTailExit; - internal_memcpy(&ExpectedTailExit, RewindingRecordPtr, FunctionRecSize); - - DCHECK(ExpectedTailExit.RecordKind == - uint8_t(FunctionRecord::RecordKinds::FunctionTailExit) && - "Expected to find tail exit when rewinding."); - RewindingRecordPtr -= FunctionRecSize; - RewindingTSC -= ExpectedTailExit.TSCDelta; - FunctionRecord ExpectedFunctionEntry; - internal_memcpy(&ExpectedFunctionEntry, RewindingRecordPtr, - FunctionRecSize); - DCHECK(ExpectedFunctionEntry.RecordKind == - uint8_t(FunctionRecord::RecordKinds::FunctionEnter) && - "Expected to find function entry when rewinding tail call."); - DCHECK(ExpectedFunctionEntry.FuncId == ExpectedTailExit.FuncId && - "Expected funcids to match when rewinding tail call."); - - // This tail call exceeded the threshold duration. It will not be erased. - if ((TSC - RewindingTSC) >= atomic_load_relaxed(&ThresholdTicks)) { - TLD.NumTailCalls = 0; - return; - } - - // We can erase a tail exit pair that we're exiting through since - // its duration is under threshold. - --TLD.NumTailCalls; - RewindingRecordPtr -= FunctionRecSize; - RewindingTSC -= ExpectedFunctionEntry.TSCDelta; - TLD.RecordPtr -= 2 * FunctionRecSize; - LastTSC = RewindingTSC; - decrementExtents(2 * FunctionRecSize); - } -} - -static bool releaseThreadLocalBuffer(BufferQueue &BQArg) { - auto &TLD = getThreadLocalData(); - auto EC = BQArg.releaseBuffer(TLD.Buffer); - if (EC != BufferQueue::ErrorCode::Ok) { - Report("Failed to release buffer at %p; error=%s\n", TLD.Buffer.Data, - BufferQueue::getErrorString(EC)); - return false; - } - return true; -} - -static bool prepareBuffer(uint64_t TSC, unsigned char CPU, - int (*wall_clock_reader)(clockid_t, - struct timespec *), - size_t MaxSize) XRAY_NEVER_INSTRUMENT { - auto &TLD = getThreadLocalData(); - char *BufferStart = static_cast<char *>(TLD.Buffer.Data); - if ((TLD.RecordPtr + MaxSize) > (BufferStart + TLD.Buffer.Size)) { - if (!releaseThreadLocalBuffer(*TLD.BQ)) - return false; - auto EC = TLD.BQ->getBuffer(TLD.Buffer); - if (EC != BufferQueue::ErrorCode::Ok) { - Report("Failed to prepare a buffer; error = '%s'\n", - BufferQueue::getErrorString(EC)); - return false; - } - setupNewBuffer(wall_clock_reader); - - // Always write the CPU metadata as the first record in the buffer. - writeNewCPUIdMetadata(CPU, TSC); - } - return true; -} - -static bool -isLogInitializedAndReady(BufferQueue *LBQ, uint64_t TSC, unsigned char CPU, - int (*wall_clock_reader)(clockid_t, struct timespec *)) - XRAY_NEVER_INSTRUMENT { - // Bail out right away if logging is not initialized yet. - // We should take the opportunity to release the buffer though. - auto Status = atomic_load(&LoggingStatus, memory_order_acquire); - auto &TLD = getThreadLocalData(); - if (Status != XRayLogInitStatus::XRAY_LOG_INITIALIZED) { - if (TLD.RecordPtr != nullptr && - (Status == XRayLogInitStatus::XRAY_LOG_FINALIZING || - Status == XRayLogInitStatus::XRAY_LOG_FINALIZED)) { - if (!releaseThreadLocalBuffer(*LBQ)) - return false; - TLD.RecordPtr = nullptr; - return false; - } - return false; - } - - if (atomic_load(&LoggingStatus, memory_order_acquire) != - XRayLogInitStatus::XRAY_LOG_INITIALIZED || - LBQ->finalizing()) { - if (!releaseThreadLocalBuffer(*LBQ)) - return false; - TLD.RecordPtr = nullptr; - } - - if (TLD.Buffer.Data == nullptr) { - auto EC = LBQ->getBuffer(TLD.Buffer); - if (EC != BufferQueue::ErrorCode::Ok) { - auto LS = atomic_load(&LoggingStatus, memory_order_acquire); - if (LS != XRayLogInitStatus::XRAY_LOG_FINALIZING && - LS != XRayLogInitStatus::XRAY_LOG_FINALIZED) - Report("Failed to acquire a buffer; error = '%s'\n", - BufferQueue::getErrorString(EC)); - return false; - } - - setupNewBuffer(wall_clock_reader); - - // Always write the CPU metadata as the first record in the buffer. - writeNewCPUIdMetadata(CPU, TSC); - } - - if (TLD.CurrentCPU == std::numeric_limits<uint16_t>::max()) { - // This means this is the first CPU this thread has ever run on. We set - // the current CPU and record this as the first TSC we've seen. - TLD.CurrentCPU = CPU; - writeNewCPUIdMetadata(CPU, TSC); - } - - return true; -} - -// Compute the TSC difference between the time of measurement and the previous -// event. There are a few interesting situations we need to account for: -// -// - The thread has migrated to a different CPU. If this is the case, then -// we write down the following records: -// -// 1. A 'NewCPUId' Metadata record. -// 2. A FunctionRecord with a 0 for the TSCDelta field. -// -// - The TSC delta is greater than the 32 bits we can store in a -// FunctionRecord. In this case we write down the following records: -// -// 1. A 'TSCWrap' Metadata record. -// 2. A FunctionRecord with a 0 for the TSCDelta field. -// -// - The TSC delta is representable within the 32 bits we can store in a -// FunctionRecord. In this case we write down just a FunctionRecord with -// the correct TSC delta. -static uint32_t writeCurrentCPUTSC(ThreadLocalData &TLD, uint64_t TSC, - uint8_t CPU) { - if (CPU != TLD.CurrentCPU) { - // We've moved to a new CPU. - writeNewCPUIdMetadata(CPU, TSC); - return 0; - } - // If the delta is greater than the range for a uint32_t, then we write out - // the TSC wrap metadata entry with the full TSC, and the TSC for the - // function record be 0. - uint64_t Delta = TSC - TLD.LastTSC; - if (Delta <= std::numeric_limits<uint32_t>::max()) - return Delta; - - writeTSCWrapMetadata(TSC); - return 0; -} - -static void endBufferIfFull() XRAY_NEVER_INSTRUMENT { - auto &TLD = getThreadLocalData(); - auto BufferStart = static_cast<char *>(TLD.Buffer.Data); - if ((TLD.RecordPtr + MetadataRecSize) - BufferStart <= - ptrdiff_t{MetadataRecSize}) { - if (!releaseThreadLocalBuffer(*TLD.BQ)) - return; - TLD.RecordPtr = nullptr; - } -} - -thread_local atomic_uint8_t Running{0}; - -/// Here's where the meat of the processing happens. The writer captures -/// function entry, exit and tail exit points with a time and will create -/// TSCWrap, NewCPUId and Function records as necessary. The writer might -/// walk backward through its buffer and erase trivial functions to avoid -/// polluting the log and may use the buffer queue to obtain or release a -/// buffer. -static void processFunctionHook(int32_t FuncId, XRayEntryType Entry, - uint64_t TSC, unsigned char CPU, uint64_t Arg1, - int (*wall_clock_reader)(clockid_t, - struct timespec *)) - XRAY_NEVER_INSTRUMENT { - __asm volatile("# LLVM-MCA-BEGIN processFunctionHook"); - // Prevent signal handler recursion, so in case we're already in a log writing - // mode and the signal handler comes in (and is also instrumented) then we - // don't want to be clobbering potentially partial writes already happening in - // the thread. We use a simple thread_local latch to only allow one on-going - // handleArg0 to happen at any given time. - RecursionGuard Guard{Running}; - if (!Guard) { - DCHECK(atomic_load_relaxed(&Running) && "RecursionGuard is buggy!"); - return; - } - - auto &TLD = getThreadLocalData(); - - if (TLD.BQ == nullptr) - TLD.BQ = BQ; - - if (!isLogInitializedAndReady(TLD.BQ, TSC, CPU, wall_clock_reader)) - return; - - // Before we go setting up writing new function entries, we need to be really - // careful about the pointer math we're doing. This means we need to ensure - // that the record we are about to write is going to fit into the buffer, - // without overflowing the buffer. - // - // To do this properly, we use the following assumptions: - // - // - The least number of bytes we will ever write is 8 - // (sizeof(FunctionRecord)) only if the delta between the previous entry - // and this entry is within 32 bits. - // - The most number of bytes we will ever write is 8 + 16 + 16 = 40. - // This is computed by: - // - // MaxSize = sizeof(FunctionRecord) + 2 * sizeof(MetadataRecord) - // - // These arise in the following cases: - // - // 1. When the delta between the TSC we get and the previous TSC for the - // same CPU is outside of the uint32_t range, we end up having to - // write a MetadataRecord to indicate a "tsc wrap" before the actual - // FunctionRecord. - // 2. When we learn that we've moved CPUs, we need to write a - // MetadataRecord to indicate a "cpu change", and thus write out the - // current TSC for that CPU before writing out the actual - // FunctionRecord. - // 3. When we learn about a new CPU ID, we need to write down a "new cpu - // id" MetadataRecord before writing out the actual FunctionRecord. - // 4. The second MetadataRecord is the optional function call argument. - // - // So the math we need to do is to determine whether writing 40 bytes past the - // current pointer exceeds the buffer's maximum size. If we don't have enough - // space to write 40 bytes in the buffer, we need get a new Buffer, set it up - // properly before doing any further writing. - size_t MaxSize = FunctionRecSize + 2 * MetadataRecSize; - if (!prepareBuffer(TSC, CPU, wall_clock_reader, MaxSize)) { - TLD.BQ = nullptr; - return; - } - - // By this point, we are now ready to write up to 40 bytes (explained above). - DCHECK((TLD.RecordPtr + MaxSize) - static_cast<char *>(TLD.Buffer.Data) >= - static_cast<ptrdiff_t>(MetadataRecSize) && - "Misconfigured BufferQueue provided; Buffer size not large enough."); - - auto RecordTSCDelta = writeCurrentCPUTSC(TLD, TSC, CPU); - TLD.LastTSC = TSC; - TLD.CurrentCPU = CPU; - switch (Entry) { - case XRayEntryType::ENTRY: - case XRayEntryType::LOG_ARGS_ENTRY: - // Update the thread local state for the next invocation. - TLD.LastFunctionEntryTSC = TSC; - break; - case XRayEntryType::TAIL: - case XRayEntryType::EXIT: - // Break out and write the exit record if we can't erase any functions. - if (TLD.NumConsecutiveFnEnters == 0 || - (TSC - TLD.LastFunctionEntryTSC) >= - atomic_load_relaxed(&ThresholdTicks)) - break; - rewindRecentCall(TSC, TLD.LastTSC, TLD.LastFunctionEntryTSC, FuncId); - return; // without writing log. - case XRayEntryType::CUSTOM_EVENT: { - // This is a bug in patching, so we'll report it once and move on. - static atomic_uint8_t ErrorLatch{0}; - if (!atomic_exchange(&ErrorLatch, 1, memory_order_acq_rel)) - Report("Internal error: patched an XRay custom event call as a function; " - "func id = %d\n", - FuncId); - return; - } - case XRayEntryType::TYPED_EVENT: { - static atomic_uint8_t ErrorLatch{0}; - if (!atomic_exchange(&ErrorLatch, 1, memory_order_acq_rel)) - Report("Internal error: patched an XRay typed event call as a function; " - "func id = %d\n", - FuncId); - return; - } - } - - writeFunctionRecord(FuncId, RecordTSCDelta, Entry); - if (Entry == XRayEntryType::LOG_ARGS_ENTRY) - writeCallArgumentMetadata(Arg1); - - // If we've exhausted the buffer by this time, we then release the buffer to - // make sure that other threads may start using this buffer. - endBufferIfFull(); - __asm volatile("# LLVM-MCA-END"); -} - static XRayFileHeader &fdrCommonHeaderInfo() { static std::aligned_storage<sizeof(XRayFileHeader)>::type HStorage; static pthread_once_t OnceInit = PTHREAD_ONCE_INIT; static bool TSCSupported = true; static uint64_t CycleFrequency = NanosecondsPerSecond; - pthread_once(&OnceInit, +[] { - XRayFileHeader &H = reinterpret_cast<XRayFileHeader &>(HStorage); - // Version 2 of the log writes the extents of the buffer, instead of - // relying on an end-of-buffer record. - // Version 3 includes PID metadata record - H.Version = 3; - H.Type = FileTypes::FDR_LOG; - - // Test for required CPU features and cache the cycle frequency - TSCSupported = probeRequiredCPUFeatures(); - if (TSCSupported) - CycleFrequency = getTSCFrequency(); - H.CycleFrequency = CycleFrequency; - - // FIXME: Actually check whether we have 'constant_tsc' and - // 'nonstop_tsc' before setting the values in the header. - H.ConstantTSC = 1; - H.NonstopTSC = 1; - }); + pthread_once( + &OnceInit, +[] { + XRayFileHeader &H = reinterpret_cast<XRayFileHeader &>(HStorage); + // Version 2 of the log writes the extents of the buffer, instead of + // relying on an end-of-buffer record. + // Version 3 includes PID metadata record. + // Version 4 includes CPU data in the custom event records. + // Version 5 uses relative deltas for custom and typed event records, + // and removes the CPU data in custom event records (similar to how + // function records use deltas instead of full TSCs and rely on other + // metadata records for TSC wraparound and CPU migration). + H.Version = 5; + H.Type = FileTypes::FDR_LOG; + + // Test for required CPU features and cache the cycle frequency + TSCSupported = probeRequiredCPUFeatures(); + if (TSCSupported) + CycleFrequency = getTSCFrequency(); + H.CycleFrequency = CycleFrequency; + + // FIXME: Actually check whether we have 'constant_tsc' and + // 'nonstop_tsc' before setting the values in the header. + H.ConstantTSC = 1; + H.NonstopTSC = 1; + }); return reinterpret_cast<XRayFileHeader &>(HStorage); } @@ -728,9 +207,11 @@ XRayBuffer fdrIterator(const XRayBuffer B) { // buffers to expect). static std::aligned_storage<sizeof(XRayFileHeader)>::type HeaderStorage; static pthread_once_t HeaderOnce = PTHREAD_ONCE_INIT; - pthread_once(&HeaderOnce, +[] { - reinterpret_cast<XRayFileHeader &>(HeaderStorage) = fdrCommonHeaderInfo(); - }); + pthread_once( + &HeaderOnce, +[] { + reinterpret_cast<XRayFileHeader &>(HeaderStorage) = + fdrCommonHeaderInfo(); + }); // We use a convenience alias for code referring to Header from here on out. auto &Header = reinterpret_cast<XRayFileHeader &>(HeaderStorage); @@ -741,7 +222,8 @@ XRayBuffer fdrIterator(const XRayBuffer B) { static BufferQueue::const_iterator It{}; static BufferQueue::const_iterator End{}; - static void *CurrentBuffer{nullptr}; + static uint8_t *CurrentBuffer{nullptr}; + static size_t SerializedBufferSize = 0; if (B.Data == static_cast<void *>(&Header) && B.Size == sizeof(Header)) { // From this point on, we provide raw access to the raw buffer we're getting // from the BufferQueue. We're relying on the iterators from the current @@ -751,7 +233,7 @@ XRayBuffer fdrIterator(const XRayBuffer B) { } if (CurrentBuffer != nullptr) { - InternalFree(CurrentBuffer); + deallocateBuffer(CurrentBuffer, SerializedBufferSize); CurrentBuffer = nullptr; } @@ -762,9 +244,16 @@ XRayBuffer fdrIterator(const XRayBuffer B) { // out to disk. The difference here would be that we still write "empty" // buffers, or at least go through the iterators faithfully to let the // handlers see the empty buffers in the queue. - auto BufferSize = atomic_load(&It->Extents->Size, memory_order_acquire); - auto SerializedBufferSize = BufferSize + sizeof(MetadataRecord); - CurrentBuffer = InternalAlloc(SerializedBufferSize); + // + // We need this atomic fence here to ensure that writes happening to the + // buffer have been committed before we load the extents atomically. Because + // the buffer is not explicitly synchronised across threads, we rely on the + // fence ordering to ensure that writes we expect to have been completed + // before the fence are fully committed before we read the extents. + atomic_thread_fence(memory_order_acquire); + auto BufferSize = atomic_load(It->Extents, memory_order_acquire); + SerializedBufferSize = BufferSize + sizeof(MetadataRecord); + CurrentBuffer = allocateBuffer(SerializedBufferSize); if (CurrentBuffer == nullptr) return {nullptr, 0}; @@ -827,14 +316,9 @@ XRayLogFlushStatus fdrLoggingFlush() XRAY_NEVER_INSTRUMENT { }); auto CleanupBuffers = at_scope_exit([] { - if (BQ != nullptr) { - auto &TLD = getThreadLocalData(); - if (TLD.RecordPtr != nullptr && TLD.BQ != nullptr) - releaseThreadLocalBuffer(*TLD.BQ); - BQ->~BufferQueue(); - InternalFree(BQ); - BQ = nullptr; - } + auto &TLD = getThreadLocalData(); + if (TLD.Controller != nullptr) + TLD.Controller->flush(); }); if (fdrFlags()->no_file_flush) { @@ -855,16 +339,8 @@ XRayLogFlushStatus fdrLoggingFlush() XRAY_NEVER_INSTRUMENT { // (fixed-sized) and let the tools reading the buffers deal with the data // afterwards. // - int Fd = -1; - { - // FIXME: Remove this section of the code, when we remove the struct-based - // configuration API. - SpinMutexLock Guard(&FDROptionsMutex); - Fd = FDROptions.Fd; - } - if (Fd == -1) - Fd = getLogFD(); - if (Fd == -1) { + LogWriter *LW = LogWriter::Open(); + if (LW == nullptr) { auto Result = XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; atomic_store(&LogFlushStatus, Result, memory_order_release); return Result; @@ -872,8 +348,15 @@ XRayLogFlushStatus fdrLoggingFlush() XRAY_NEVER_INSTRUMENT { XRayFileHeader Header = fdrCommonHeaderInfo(); Header.FdrData = FdrAdditionalHeaderData{BQ->ConfiguredBufferSize()}; - retryingWriteAll(Fd, reinterpret_cast<char *>(&Header), - reinterpret_cast<char *>(&Header) + sizeof(Header)); + LW->WriteAll(reinterpret_cast<char *>(&Header), + reinterpret_cast<char *>(&Header) + sizeof(Header)); + + // Release the current thread's buffer before we attempt to write out all the + // buffers. This ensures that in case we had only a single thread going, that + // we are able to capture the data nonetheless. + auto &TLD = getThreadLocalData(); + if (TLD.Controller != nullptr) + TLD.Controller->flush(); BQ->apply([&](const BufferQueue::Buffer &B) { // Starting at version 2 of the FDR logging implementation, we only write @@ -882,18 +365,18 @@ XRayLogFlushStatus fdrLoggingFlush() XRAY_NEVER_INSTRUMENT { // still use a Metadata record, but fill in the extents instead for the // data. MetadataRecord ExtentsRecord; - auto BufferExtents = atomic_load(&B.Extents->Size, memory_order_acquire); + auto BufferExtents = atomic_load(B.Extents, memory_order_acquire); DCHECK(BufferExtents <= B.Size); ExtentsRecord.Type = uint8_t(RecordType::Metadata); ExtentsRecord.RecordKind = uint8_t(MetadataRecord::RecordKinds::BufferExtents); internal_memcpy(ExtentsRecord.Data, &BufferExtents, sizeof(BufferExtents)); if (BufferExtents > 0) { - retryingWriteAll(Fd, reinterpret_cast<char *>(&ExtentsRecord), - reinterpret_cast<char *>(&ExtentsRecord) + - sizeof(MetadataRecord)); - retryingWriteAll(Fd, reinterpret_cast<char *>(B.Data), - reinterpret_cast<char *>(B.Data) + BufferExtents); + LW->WriteAll(reinterpret_cast<char *>(&ExtentsRecord), + reinterpret_cast<char *>(&ExtentsRecord) + + sizeof(MetadataRecord)); + LW->WriteAll(reinterpret_cast<char *>(B.Data), + reinterpret_cast<char *>(B.Data) + BufferExtents); } }); @@ -914,7 +397,12 @@ XRayLogInitStatus fdrLoggingFinalize() XRAY_NEVER_INSTRUMENT { // Do special things to make the log finalize itself, and not allow any more // operations to be performed until re-initialized. - BQ->finalize(); + if (BQ == nullptr) { + if (Verbosity()) + Report("Attempting to finalize an uninitialized global buffer!\n"); + } else { + BQ->finalize(); + } atomic_store(&LoggingStatus, XRayLogInitStatus::XRAY_LOG_FINALIZED, memory_order_release); @@ -935,7 +423,8 @@ static TSCAndCPU getTimestamp() XRAY_NEVER_INSTRUMENT { // Test once for required CPU features static pthread_once_t OnceProbe = PTHREAD_ONCE_INIT; static bool TSCSupported = true; - pthread_once(&OnceProbe, +[] { TSCSupported = probeRequiredCPUFeatures(); }); + pthread_once( + &OnceProbe, +[] { TSCSupported = probeRequiredCPUFeatures(); }); if (TSCSupported) { Result.TSC = __xray::readTSC(Result.CPU); @@ -953,16 +442,115 @@ static TSCAndCPU getTimestamp() XRAY_NEVER_INSTRUMENT { return Result; } +thread_local atomic_uint8_t Running{0}; + +static bool setupTLD(ThreadLocalData &TLD) XRAY_NEVER_INSTRUMENT { + // Check if we're finalizing, before proceeding. + { + auto Status = atomic_load(&LoggingStatus, memory_order_acquire); + if (Status == XRayLogInitStatus::XRAY_LOG_FINALIZING || + Status == XRayLogInitStatus::XRAY_LOG_FINALIZED) { + if (TLD.Controller != nullptr) { + TLD.Controller->flush(); + TLD.Controller = nullptr; + } + return false; + } + } + + if (UNLIKELY(TLD.Controller == nullptr)) { + // Set up the TLD buffer queue. + if (UNLIKELY(BQ == nullptr)) + return false; + TLD.BQ = BQ; + + // Check that we have a valid buffer. + if (TLD.Buffer.Generation != BQ->generation() && + TLD.BQ->releaseBuffer(TLD.Buffer) != BufferQueue::ErrorCode::Ok) + return false; + + // Set up a buffer, before setting up the log writer. Bail out on failure. + if (TLD.BQ->getBuffer(TLD.Buffer) != BufferQueue::ErrorCode::Ok) + return false; + + // Set up the Log Writer for this thread. + if (UNLIKELY(TLD.Writer == nullptr)) { + auto *LWStorage = reinterpret_cast<FDRLogWriter *>(&TLD.LWStorage); + new (LWStorage) FDRLogWriter(TLD.Buffer); + TLD.Writer = LWStorage; + } else { + TLD.Writer->resetRecord(); + } + + auto *CStorage = reinterpret_cast<FDRController<> *>(&TLD.CStorage); + new (CStorage) + FDRController<>(TLD.BQ, TLD.Buffer, *TLD.Writer, clock_gettime, + atomic_load_relaxed(&ThresholdTicks)); + TLD.Controller = CStorage; + } + + DCHECK_NE(TLD.Controller, nullptr); + return true; +} + void fdrLoggingHandleArg0(int32_t FuncId, XRayEntryType Entry) XRAY_NEVER_INSTRUMENT { auto TC = getTimestamp(); - processFunctionHook(FuncId, Entry, TC.TSC, TC.CPU, 0, clock_gettime); + auto &TSC = TC.TSC; + auto &CPU = TC.CPU; + RecursionGuard Guard{Running}; + if (!Guard) + return; + + auto &TLD = getThreadLocalData(); + if (!setupTLD(TLD)) + return; + + switch (Entry) { + case XRayEntryType::ENTRY: + case XRayEntryType::LOG_ARGS_ENTRY: + TLD.Controller->functionEnter(FuncId, TSC, CPU); + return; + case XRayEntryType::EXIT: + TLD.Controller->functionExit(FuncId, TSC, CPU); + return; + case XRayEntryType::TAIL: + TLD.Controller->functionTailExit(FuncId, TSC, CPU); + return; + case XRayEntryType::CUSTOM_EVENT: + case XRayEntryType::TYPED_EVENT: + break; + } } void fdrLoggingHandleArg1(int32_t FuncId, XRayEntryType Entry, uint64_t Arg) XRAY_NEVER_INSTRUMENT { auto TC = getTimestamp(); - processFunctionHook(FuncId, Entry, TC.TSC, TC.CPU, Arg, clock_gettime); + auto &TSC = TC.TSC; + auto &CPU = TC.CPU; + RecursionGuard Guard{Running}; + if (!Guard) + return; + + auto &TLD = getThreadLocalData(); + if (!setupTLD(TLD)) + return; + + switch (Entry) { + case XRayEntryType::ENTRY: + case XRayEntryType::LOG_ARGS_ENTRY: + TLD.Controller->functionEnterArg(FuncId, TSC, CPU, Arg); + return; + case XRayEntryType::EXIT: + TLD.Controller->functionExit(FuncId, TSC, CPU); + return; + case XRayEntryType::TAIL: + TLD.Controller->functionTailExit(FuncId, TSC, CPU); + return; + case XRayEntryType::CUSTOM_EVENT: + case XRayEntryType::TYPED_EVENT: + break; + } } void fdrLoggingHandleCustomEvent(void *Event, @@ -973,40 +561,25 @@ void fdrLoggingHandleCustomEvent(void *Event, RecursionGuard Guard{Running}; if (!Guard) return; - if (EventSize > std::numeric_limits<int32_t>::max()) { + + // Complain when we ever get at least one custom event that's larger than what + // we can possibly support. + if (EventSize > + static_cast<std::size_t>(std::numeric_limits<int32_t>::max())) { static pthread_once_t Once = PTHREAD_ONCE_INIT; - pthread_once(&Once, +[] { Report("Event size too large.\n"); }); + pthread_once( + &Once, +[] { + Report("Custom event size too large; truncating to %d.\n", + std::numeric_limits<int32_t>::max()); + }); } - int32_t ReducedEventSize = static_cast<int32_t>(EventSize); - auto &TLD = getThreadLocalData(); - if (!isLogInitializedAndReady(TLD.BQ, TSC, CPU, clock_gettime)) - return; - // Here we need to prepare the log to handle: - // - The metadata record we're going to write. (16 bytes) - // - The additional data we're going to write. Currently, that's the size - // of the event we're going to dump into the log as free-form bytes. - if (!prepareBuffer(TSC, CPU, clock_gettime, MetadataRecSize + EventSize)) { - TLD.BQ = nullptr; + auto &TLD = getThreadLocalData(); + if (!setupTLD(TLD)) return; - } - // Write the custom event metadata record, which consists of the following - // information: - // - 8 bytes (64-bits) for the full TSC when the event started. - // - 4 bytes (32-bits) for the length of the data. - MetadataRecord CustomEvent; - CustomEvent.Type = uint8_t(RecordType::Metadata); - CustomEvent.RecordKind = - uint8_t(MetadataRecord::RecordKinds::CustomEventMarker); - constexpr auto TSCSize = sizeof(TC.TSC); - internal_memcpy(&CustomEvent.Data, &ReducedEventSize, sizeof(int32_t)); - internal_memcpy(&CustomEvent.Data[sizeof(int32_t)], &TSC, TSCSize); - internal_memcpy(TLD.RecordPtr, &CustomEvent, sizeof(CustomEvent)); - TLD.RecordPtr += sizeof(CustomEvent); - internal_memcpy(TLD.RecordPtr, Event, ReducedEventSize); - incrementExtents(MetadataRecSize + EventSize); - endBufferIfFull(); + int32_t ReducedEventSize = static_cast<int32_t>(EventSize); + TLD.Controller->customEvent(TSC, CPU, Event, ReducedEventSize); } void fdrLoggingHandleTypedEvent( @@ -1018,50 +591,28 @@ void fdrLoggingHandleTypedEvent( RecursionGuard Guard{Running}; if (!Guard) return; - if (EventSize > std::numeric_limits<int32_t>::max()) { + + // Complain when we ever get at least one typed event that's larger than what + // we can possibly support. + if (EventSize > + static_cast<std::size_t>(std::numeric_limits<int32_t>::max())) { static pthread_once_t Once = PTHREAD_ONCE_INIT; - pthread_once(&Once, +[] { Report("Event size too large.\n"); }); + pthread_once( + &Once, +[] { + Report("Typed event size too large; truncating to %d.\n", + std::numeric_limits<int32_t>::max()); + }); } - int32_t ReducedEventSize = static_cast<int32_t>(EventSize); + auto &TLD = getThreadLocalData(); - if (!isLogInitializedAndReady(TLD.BQ, TSC, CPU, clock_gettime)) + if (!setupTLD(TLD)) return; - // Here we need to prepare the log to handle: - // - The metadata record we're going to write. (16 bytes) - // - The additional data we're going to write. Currently, that's the size - // of the event we're going to dump into the log as free-form bytes. - if (!prepareBuffer(TSC, CPU, clock_gettime, MetadataRecSize + EventSize)) { - TLD.BQ = nullptr; - return; - } - // Write the custom event metadata record, which consists of the following - // information: - // - 8 bytes (64-bits) for the full TSC when the event started. - // - 4 bytes (32-bits) for the length of the data. - // - 2 bytes (16-bits) for the event type. 3 bytes remain since one of the - // bytes has the record type (Metadata Record) and kind (TypedEvent). - // We'll log the error if the event type is greater than 2 bytes. - // Event types are generated sequentially, so 2^16 is enough. - MetadataRecord TypedEvent; - TypedEvent.Type = uint8_t(RecordType::Metadata); - TypedEvent.RecordKind = - uint8_t(MetadataRecord::RecordKinds::TypedEventMarker); - constexpr auto TSCSize = sizeof(TC.TSC); - internal_memcpy(&TypedEvent.Data, &ReducedEventSize, sizeof(int32_t)); - internal_memcpy(&TypedEvent.Data[sizeof(int32_t)], &TSC, TSCSize); - internal_memcpy(&TypedEvent.Data[sizeof(int32_t) + TSCSize], &EventType, - sizeof(EventType)); - internal_memcpy(TLD.RecordPtr, &TypedEvent, sizeof(TypedEvent)); - - TLD.RecordPtr += sizeof(TypedEvent); - internal_memcpy(TLD.RecordPtr, Event, ReducedEventSize); - incrementExtents(MetadataRecSize + EventSize); - endBufferIfFull(); + int32_t ReducedEventSize = static_cast<int32_t>(EventSize); + TLD.Controller->typedEvent(TSC, CPU, EventType, Event, ReducedEventSize); } -XRayLogInitStatus fdrLoggingInit(size_t BufferSize, size_t BufferMax, - void *Options, +XRayLogInitStatus fdrLoggingInit(size_t, size_t, void *Options, size_t OptionsSize) XRAY_NEVER_INSTRUMENT { if (Options == nullptr) return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; @@ -1075,107 +626,81 @@ XRayLogInitStatus fdrLoggingInit(size_t BufferSize, size_t BufferMax, return static_cast<XRayLogInitStatus>(CurrentStatus); } - // Because of __xray_log_init_mode(...) which guarantees that this will be - // called with BufferSize == 0 and BufferMax == 0 we parse the configuration - // provided in the Options pointer as a string instead. - if (BufferSize == 0 && BufferMax == 0) { - if (Verbosity()) - Report("Initializing FDR mode with options: %s\n", - static_cast<const char *>(Options)); - - // TODO: Factor out the flags specific to the FDR mode implementation. For - // now, use the global/single definition of the flags, since the FDR mode - // flags are already defined there. - FlagParser FDRParser; - FDRFlags FDRFlags; - registerXRayFDRFlags(&FDRParser, &FDRFlags); - FDRFlags.setDefaults(); - - // Override first from the general XRAY_DEFAULT_OPTIONS compiler-provided - // options until we migrate everyone to use the XRAY_FDR_OPTIONS - // compiler-provided options. - FDRParser.ParseString(useCompilerDefinedFlags()); - FDRParser.ParseString(useCompilerDefinedFDRFlags()); - auto *EnvOpts = GetEnv("XRAY_FDR_OPTIONS"); - if (EnvOpts == nullptr) - EnvOpts = ""; - FDRParser.ParseString(EnvOpts); - - // FIXME: Remove this when we fully remove the deprecated flags. - if (internal_strlen(EnvOpts) == 0) { - FDRFlags.func_duration_threshold_us = - flags()->xray_fdr_log_func_duration_threshold_us; - FDRFlags.grace_period_ms = flags()->xray_fdr_log_grace_period_ms; - } - - // The provided options should always override the compiler-provided and - // environment-variable defined options. - FDRParser.ParseString(static_cast<const char *>(Options)); - *fdrFlags() = FDRFlags; - BufferSize = FDRFlags.buffer_size; - BufferMax = FDRFlags.buffer_max; - SpinMutexLock Guard(&FDROptionsMutex); - FDROptions.Fd = -1; - FDROptions.ReportErrors = true; - } else if (OptionsSize != sizeof(FDRLoggingOptions)) { - // FIXME: This is deprecated, and should really be removed. - // At this point we use the flag parser specific to the FDR mode - // implementation. - if (Verbosity()) - Report("Cannot initialize FDR logging; wrong size for options: %d\n", - OptionsSize); - return static_cast<XRayLogInitStatus>( - atomic_load(&LoggingStatus, memory_order_acquire)); - } else { - if (Verbosity()) - Report("XRay FDR: struct-based init is deprecated, please use " - "string-based configuration instead.\n"); - SpinMutexLock Guard(&FDROptionsMutex); - internal_memcpy(&FDROptions, Options, OptionsSize); - } - - bool Success = false; - - if (BQ != nullptr) { - BQ->~BufferQueue(); - InternalFree(BQ); - BQ = nullptr; - } + if (Verbosity()) + Report("Initializing FDR mode with options: %s\n", + static_cast<const char *>(Options)); + + // TODO: Factor out the flags specific to the FDR mode implementation. For + // now, use the global/single definition of the flags, since the FDR mode + // flags are already defined there. + FlagParser FDRParser; + FDRFlags FDRFlags; + registerXRayFDRFlags(&FDRParser, &FDRFlags); + FDRFlags.setDefaults(); + + // Override first from the general XRAY_DEFAULT_OPTIONS compiler-provided + // options until we migrate everyone to use the XRAY_FDR_OPTIONS + // compiler-provided options. + FDRParser.ParseString(useCompilerDefinedFlags()); + FDRParser.ParseString(useCompilerDefinedFDRFlags()); + auto *EnvOpts = GetEnv("XRAY_FDR_OPTIONS"); + if (EnvOpts == nullptr) + EnvOpts = ""; + FDRParser.ParseString(EnvOpts); + + // FIXME: Remove this when we fully remove the deprecated flags. + if (internal_strlen(EnvOpts) == 0) { + FDRFlags.func_duration_threshold_us = + flags()->xray_fdr_log_func_duration_threshold_us; + FDRFlags.grace_period_ms = flags()->xray_fdr_log_grace_period_ms; + } + + // The provided options should always override the compiler-provided and + // environment-variable defined options. + FDRParser.ParseString(static_cast<const char *>(Options)); + *fdrFlags() = FDRFlags; + auto BufferSize = FDRFlags.buffer_size; + auto BufferMax = FDRFlags.buffer_max; if (BQ == nullptr) { - BQ = reinterpret_cast<BufferQueue *>( - InternalAlloc(sizeof(BufferQueue), nullptr, 64)); + bool Success = false; + BQ = reinterpret_cast<BufferQueue *>(&BufferQueueStorage); new (BQ) BufferQueue(BufferSize, BufferMax, Success); - } - - if (!Success) { - Report("BufferQueue init failed.\n"); - if (BQ != nullptr) { - BQ->~BufferQueue(); - InternalFree(BQ); - BQ = nullptr; + if (!Success) { + Report("BufferQueue init failed.\n"); + return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; + } + } else { + if (BQ->init(BufferSize, BufferMax) != BufferQueue::ErrorCode::Ok) { + if (Verbosity()) + Report("Failed to re-initialize global buffer queue. Init failed.\n"); + return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; } - return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; } static pthread_once_t OnceInit = PTHREAD_ONCE_INIT; - pthread_once(&OnceInit, +[] { - atomic_store(&TicksPerSec, - probeRequiredCPUFeatures() ? getTSCFrequency() - : __xray::NanosecondsPerSecond, - memory_order_release); - pthread_key_create(&Key, +[](void *TLDPtr) { - if (TLDPtr == nullptr) - return; - auto &TLD = *reinterpret_cast<ThreadLocalData *>(TLDPtr); - if (TLD.BQ == nullptr) - return; - auto EC = TLD.BQ->releaseBuffer(TLD.Buffer); - if (EC != BufferQueue::ErrorCode::Ok) - Report("At thread exit, failed to release buffer at %p; error=%s\n", - TLD.Buffer.Data, BufferQueue::getErrorString(EC)); - }); - }); + pthread_once( + &OnceInit, +[] { + atomic_store(&TicksPerSec, + probeRequiredCPUFeatures() ? getTSCFrequency() + : __xray::NanosecondsPerSecond, + memory_order_release); + pthread_key_create( + &Key, +[](void *TLDPtr) { + if (TLDPtr == nullptr) + return; + auto &TLD = *reinterpret_cast<ThreadLocalData *>(TLDPtr); + if (TLD.BQ == nullptr) + return; + if (TLD.Buffer.Data == nullptr) + return; + auto EC = TLD.BQ->releaseBuffer(TLD.Buffer); + if (EC != BufferQueue::ErrorCode::Ok) + Report("At thread exit, failed to release buffer at %p; " + "error=%s\n", + TLD.Buffer.Data, BufferQueue::getErrorString(EC)); + }); + }); atomic_store(&ThresholdTicks, atomic_load_relaxed(&TicksPerSec) * @@ -1209,11 +734,22 @@ bool fdrLogDynamicInitializer() XRAY_NEVER_INSTRUMENT { }; auto RegistrationResult = __xray_log_register_mode("xray-fdr", Impl); if (RegistrationResult != XRayLogRegisterStatus::XRAY_REGISTRATION_OK && - Verbosity()) + Verbosity()) { Report("Cannot register XRay FDR mode to 'xray-fdr'; error = %d\n", RegistrationResult); - if (flags()->xray_fdr_log || !internal_strcmp(flags()->xray_mode, "xray-fdr")) - __xray_set_log_impl(Impl); + return false; + } + + if (flags()->xray_fdr_log || + !internal_strcmp(flags()->xray_mode, "xray-fdr")) { + auto SelectResult = __xray_log_select_mode("xray-fdr"); + if (SelectResult != XRayLogRegisterStatus::XRAY_REGISTRATION_OK && + Verbosity()) { + Report("Cannot select XRay FDR mode as 'xray-fdr'; error = %d\n", + SelectResult); + return false; + } + } return true; } diff --git a/contrib/compiler-rt/lib/xray/xray_function_call_trie.h b/contrib/compiler-rt/lib/xray/xray_function_call_trie.h index 2acf14aa5625..d01ad20e3d71 100644 --- a/contrib/compiler-rt/lib/xray/xray_function_call_trie.h +++ b/contrib/compiler-rt/lib/xray/xray_function_call_trie.h @@ -15,9 +15,11 @@ #ifndef XRAY_FUNCTION_CALL_TRIE_H #define XRAY_FUNCTION_CALL_TRIE_H -#include "sanitizer_common/sanitizer_allocator_internal.h" +#include "xray_buffer_queue.h" +#include "xray_defs.h" #include "xray_profiling_flags.h" #include "xray_segmented_array.h" +#include <limits> #include <memory> // For placement new. #include <utility> @@ -97,9 +99,6 @@ public: struct NodeIdPair { Node *NodePtr; int32_t FId; - - // Constructor for inplace-construction. - NodeIdPair(Node *N, int32_t F) : NodePtr(N), FId(F) {} }; using NodeIdPairArray = Array<NodeIdPair>; @@ -113,17 +112,10 @@ public: struct Node { Node *Parent; NodeIdPairArray Callees; - int64_t CallCount; - int64_t CumulativeLocalTime; // Typically in TSC deltas, not wall-time. + uint64_t CallCount; + uint64_t CumulativeLocalTime; // Typically in TSC deltas, not wall-time. int32_t FId; - // We add a constructor here to allow us to inplace-construct through - // Array<...>'s AppendEmplace. - Node(Node *P, NodeIdPairAllocatorType &A, int64_t CC, int64_t CLT, - int32_t F) - : Parent(P), Callees(A), CallCount(CC), CumulativeLocalTime(CLT), - FId(F) {} - // TODO: Include the compact histogram. }; @@ -131,10 +123,7 @@ private: struct ShadowStackEntry { uint64_t EntryTSC; Node *NodePtr; - - // We add a constructor here to allow us to inplace-construct through - // Array<...>'s AppendEmplace. - ShadowStackEntry(uint64_t T, Node *N) : EntryTSC{T}, NodePtr{N} {} + uint16_t EntryCPU; }; using NodeArray = Array<Node>; @@ -149,103 +138,184 @@ public: using RootAllocatorType = RootArray::AllocatorType; using ShadowStackAllocatorType = ShadowStackArray::AllocatorType; + // Use hosted aligned storage members to allow for trivial move and init. + // This also allows us to sidestep the potential-failing allocation issue. + typename std::aligned_storage<sizeof(NodeAllocatorType), + alignof(NodeAllocatorType)>::type + NodeAllocatorStorage; + typename std::aligned_storage<sizeof(RootAllocatorType), + alignof(RootAllocatorType)>::type + RootAllocatorStorage; + typename std::aligned_storage<sizeof(ShadowStackAllocatorType), + alignof(ShadowStackAllocatorType)>::type + ShadowStackAllocatorStorage; + typename std::aligned_storage<sizeof(NodeIdPairAllocatorType), + alignof(NodeIdPairAllocatorType)>::type + NodeIdPairAllocatorStorage; + NodeAllocatorType *NodeAllocator = nullptr; RootAllocatorType *RootAllocator = nullptr; ShadowStackAllocatorType *ShadowStackAllocator = nullptr; NodeIdPairAllocatorType *NodeIdPairAllocator = nullptr; - Allocators() {} + Allocators() = default; Allocators(const Allocators &) = delete; Allocators &operator=(const Allocators &) = delete; - Allocators(Allocators &&O) - : NodeAllocator(O.NodeAllocator), RootAllocator(O.RootAllocator), - ShadowStackAllocator(O.ShadowStackAllocator), - NodeIdPairAllocator(O.NodeIdPairAllocator) { + struct Buffers { + BufferQueue::Buffer NodeBuffer; + BufferQueue::Buffer RootsBuffer; + BufferQueue::Buffer ShadowStackBuffer; + BufferQueue::Buffer NodeIdPairBuffer; + }; + + explicit Allocators(Buffers &B) XRAY_NEVER_INSTRUMENT { + new (&NodeAllocatorStorage) + NodeAllocatorType(B.NodeBuffer.Data, B.NodeBuffer.Size); + NodeAllocator = + reinterpret_cast<NodeAllocatorType *>(&NodeAllocatorStorage); + + new (&RootAllocatorStorage) + RootAllocatorType(B.RootsBuffer.Data, B.RootsBuffer.Size); + RootAllocator = + reinterpret_cast<RootAllocatorType *>(&RootAllocatorStorage); + + new (&ShadowStackAllocatorStorage) ShadowStackAllocatorType( + B.ShadowStackBuffer.Data, B.ShadowStackBuffer.Size); + ShadowStackAllocator = reinterpret_cast<ShadowStackAllocatorType *>( + &ShadowStackAllocatorStorage); + + new (&NodeIdPairAllocatorStorage) NodeIdPairAllocatorType( + B.NodeIdPairBuffer.Data, B.NodeIdPairBuffer.Size); + NodeIdPairAllocator = reinterpret_cast<NodeIdPairAllocatorType *>( + &NodeIdPairAllocatorStorage); + } + + explicit Allocators(uptr Max) XRAY_NEVER_INSTRUMENT { + new (&NodeAllocatorStorage) NodeAllocatorType(Max); + NodeAllocator = + reinterpret_cast<NodeAllocatorType *>(&NodeAllocatorStorage); + + new (&RootAllocatorStorage) RootAllocatorType(Max); + RootAllocator = + reinterpret_cast<RootAllocatorType *>(&RootAllocatorStorage); + + new (&ShadowStackAllocatorStorage) ShadowStackAllocatorType(Max); + ShadowStackAllocator = reinterpret_cast<ShadowStackAllocatorType *>( + &ShadowStackAllocatorStorage); + + new (&NodeIdPairAllocatorStorage) NodeIdPairAllocatorType(Max); + NodeIdPairAllocator = reinterpret_cast<NodeIdPairAllocatorType *>( + &NodeIdPairAllocatorStorage); + } + + Allocators(Allocators &&O) XRAY_NEVER_INSTRUMENT { + // Here we rely on the safety of memcpy'ing contents of the storage + // members, and then pointing the source pointers to nullptr. + internal_memcpy(&NodeAllocatorStorage, &O.NodeAllocatorStorage, + sizeof(NodeAllocatorType)); + internal_memcpy(&RootAllocatorStorage, &O.RootAllocatorStorage, + sizeof(RootAllocatorType)); + internal_memcpy(&ShadowStackAllocatorStorage, + &O.ShadowStackAllocatorStorage, + sizeof(ShadowStackAllocatorType)); + internal_memcpy(&NodeIdPairAllocatorStorage, + &O.NodeIdPairAllocatorStorage, + sizeof(NodeIdPairAllocatorType)); + + NodeAllocator = + reinterpret_cast<NodeAllocatorType *>(&NodeAllocatorStorage); + RootAllocator = + reinterpret_cast<RootAllocatorType *>(&RootAllocatorStorage); + ShadowStackAllocator = reinterpret_cast<ShadowStackAllocatorType *>( + &ShadowStackAllocatorStorage); + NodeIdPairAllocator = reinterpret_cast<NodeIdPairAllocatorType *>( + &NodeIdPairAllocatorStorage); + O.NodeAllocator = nullptr; O.RootAllocator = nullptr; O.ShadowStackAllocator = nullptr; O.NodeIdPairAllocator = nullptr; } - Allocators &operator=(Allocators &&O) { - { - auto Tmp = O.NodeAllocator; - O.NodeAllocator = this->NodeAllocator; - this->NodeAllocator = Tmp; - } - { - auto Tmp = O.RootAllocator; - O.RootAllocator = this->RootAllocator; - this->RootAllocator = Tmp; - } - { - auto Tmp = O.ShadowStackAllocator; - O.ShadowStackAllocator = this->ShadowStackAllocator; - this->ShadowStackAllocator = Tmp; - } - { - auto Tmp = O.NodeIdPairAllocator; - O.NodeIdPairAllocator = this->NodeIdPairAllocator; - this->NodeIdPairAllocator = Tmp; - } - return *this; - } - - ~Allocators() { - // Note that we cannot use delete on these pointers, as they need to be - // returned to the sanitizer_common library's internal memory tracking - // system. - if (NodeAllocator != nullptr) { + Allocators &operator=(Allocators &&O) XRAY_NEVER_INSTRUMENT { + // When moving into an existing instance, we ensure that we clean up the + // current allocators. + if (NodeAllocator) NodeAllocator->~NodeAllocatorType(); - InternalFree(NodeAllocator); + if (O.NodeAllocator) { + new (&NodeAllocatorStorage) + NodeAllocatorType(std::move(*O.NodeAllocator)); + NodeAllocator = + reinterpret_cast<NodeAllocatorType *>(&NodeAllocatorStorage); + O.NodeAllocator = nullptr; + } else { NodeAllocator = nullptr; } - if (RootAllocator != nullptr) { + + if (RootAllocator) RootAllocator->~RootAllocatorType(); - InternalFree(RootAllocator); + if (O.RootAllocator) { + new (&RootAllocatorStorage) + RootAllocatorType(std::move(*O.RootAllocator)); + RootAllocator = + reinterpret_cast<RootAllocatorType *>(&RootAllocatorStorage); + O.RootAllocator = nullptr; + } else { RootAllocator = nullptr; } - if (ShadowStackAllocator != nullptr) { + + if (ShadowStackAllocator) ShadowStackAllocator->~ShadowStackAllocatorType(); - InternalFree(ShadowStackAllocator); + if (O.ShadowStackAllocator) { + new (&ShadowStackAllocatorStorage) + ShadowStackAllocatorType(std::move(*O.ShadowStackAllocator)); + ShadowStackAllocator = reinterpret_cast<ShadowStackAllocatorType *>( + &ShadowStackAllocatorStorage); + O.ShadowStackAllocator = nullptr; + } else { ShadowStackAllocator = nullptr; } - if (NodeIdPairAllocator != nullptr) { + + if (NodeIdPairAllocator) NodeIdPairAllocator->~NodeIdPairAllocatorType(); - InternalFree(NodeIdPairAllocator); + if (O.NodeIdPairAllocator) { + new (&NodeIdPairAllocatorStorage) + NodeIdPairAllocatorType(std::move(*O.NodeIdPairAllocator)); + NodeIdPairAllocator = reinterpret_cast<NodeIdPairAllocatorType *>( + &NodeIdPairAllocatorStorage); + O.NodeIdPairAllocator = nullptr; + } else { NodeIdPairAllocator = nullptr; } + + return *this; + } + + ~Allocators() XRAY_NEVER_INSTRUMENT { + if (NodeAllocator != nullptr) + NodeAllocator->~NodeAllocatorType(); + if (RootAllocator != nullptr) + RootAllocator->~RootAllocatorType(); + if (ShadowStackAllocator != nullptr) + ShadowStackAllocator->~ShadowStackAllocatorType(); + if (NodeIdPairAllocator != nullptr) + NodeIdPairAllocator->~NodeIdPairAllocatorType(); } }; - // TODO: Support configuration of options through the arguments. - static Allocators InitAllocators() { + static Allocators InitAllocators() XRAY_NEVER_INSTRUMENT { return InitAllocatorsCustom(profilingFlags()->per_thread_allocator_max); } - static Allocators InitAllocatorsCustom(uptr Max) { - Allocators A; - auto NodeAllocator = reinterpret_cast<Allocators::NodeAllocatorType *>( - InternalAlloc(sizeof(Allocators::NodeAllocatorType))); - new (NodeAllocator) Allocators::NodeAllocatorType(Max); - A.NodeAllocator = NodeAllocator; - - auto RootAllocator = reinterpret_cast<Allocators::RootAllocatorType *>( - InternalAlloc(sizeof(Allocators::RootAllocatorType))); - new (RootAllocator) Allocators::RootAllocatorType(Max); - A.RootAllocator = RootAllocator; - - auto ShadowStackAllocator = - reinterpret_cast<Allocators::ShadowStackAllocatorType *>( - InternalAlloc(sizeof(Allocators::ShadowStackAllocatorType))); - new (ShadowStackAllocator) Allocators::ShadowStackAllocatorType(Max); - A.ShadowStackAllocator = ShadowStackAllocator; - - auto NodeIdPairAllocator = reinterpret_cast<NodeIdPairAllocatorType *>( - InternalAlloc(sizeof(NodeIdPairAllocatorType))); - new (NodeIdPairAllocator) NodeIdPairAllocatorType(Max); - A.NodeIdPairAllocator = NodeIdPairAllocator; + static Allocators InitAllocatorsCustom(uptr Max) XRAY_NEVER_INSTRUMENT { + Allocators A(Max); + return A; + } + + static Allocators + InitAllocatorsFromBuffers(Allocators::Buffers &Bufs) XRAY_NEVER_INSTRUMENT { + Allocators A(Bufs); return A; } @@ -253,65 +323,135 @@ private: NodeArray Nodes; RootArray Roots; ShadowStackArray ShadowStack; - NodeIdPairAllocatorType *NodeIdPairAllocator = nullptr; + NodeIdPairAllocatorType *NodeIdPairAllocator; + uint32_t OverflowedFunctions; public: - explicit FunctionCallTrie(const Allocators &A) - : Nodes(*A.NodeAllocator), Roots(*A.RootAllocator), + explicit FunctionCallTrie(const Allocators &A) XRAY_NEVER_INSTRUMENT + : Nodes(*A.NodeAllocator), + Roots(*A.RootAllocator), ShadowStack(*A.ShadowStackAllocator), - NodeIdPairAllocator(A.NodeIdPairAllocator) {} + NodeIdPairAllocator(A.NodeIdPairAllocator), + OverflowedFunctions(0) {} + + FunctionCallTrie() = delete; + FunctionCallTrie(const FunctionCallTrie &) = delete; + FunctionCallTrie &operator=(const FunctionCallTrie &) = delete; + + FunctionCallTrie(FunctionCallTrie &&O) XRAY_NEVER_INSTRUMENT + : Nodes(std::move(O.Nodes)), + Roots(std::move(O.Roots)), + ShadowStack(std::move(O.ShadowStack)), + NodeIdPairAllocator(O.NodeIdPairAllocator), + OverflowedFunctions(O.OverflowedFunctions) {} + + FunctionCallTrie &operator=(FunctionCallTrie &&O) XRAY_NEVER_INSTRUMENT { + Nodes = std::move(O.Nodes); + Roots = std::move(O.Roots); + ShadowStack = std::move(O.ShadowStack); + NodeIdPairAllocator = O.NodeIdPairAllocator; + OverflowedFunctions = O.OverflowedFunctions; + return *this; + } + + ~FunctionCallTrie() XRAY_NEVER_INSTRUMENT {} - void enterFunction(const int32_t FId, uint64_t TSC) { + void enterFunction(const int32_t FId, uint64_t TSC, + uint16_t CPU) XRAY_NEVER_INSTRUMENT { DCHECK_NE(FId, 0); - // This function primarily deals with ensuring that the ShadowStack is - // consistent and ready for when an exit event is encountered. + + // If we're already overflowed the function call stack, do not bother + // attempting to record any more function entries. + if (UNLIKELY(OverflowedFunctions)) { + ++OverflowedFunctions; + return; + } + + // If this is the first function we've encountered, we want to set up the + // node(s) and treat it as a root. if (UNLIKELY(ShadowStack.empty())) { - auto NewRoot = - Nodes.AppendEmplace(nullptr, *NodeIdPairAllocator, 0, 0, FId); + auto *NewRoot = Nodes.AppendEmplace( + nullptr, NodeIdPairArray(*NodeIdPairAllocator), 0u, 0u, FId); if (UNLIKELY(NewRoot == nullptr)) return; - Roots.Append(NewRoot); - ShadowStack.AppendEmplace(TSC, NewRoot); + if (Roots.AppendEmplace(NewRoot) == nullptr) { + Nodes.trim(1); + return; + } + if (ShadowStack.AppendEmplace(TSC, NewRoot, CPU) == nullptr) { + Nodes.trim(1); + Roots.trim(1); + ++OverflowedFunctions; + return; + } return; } - auto &Top = ShadowStack.back(); - auto TopNode = Top.NodePtr; + // From this point on, we require that the stack is not empty. + DCHECK(!ShadowStack.empty()); + auto TopNode = ShadowStack.back().NodePtr; DCHECK_NE(TopNode, nullptr); - // If we've seen this callee before, then we just access that node and place - // that on the top of the stack. - auto Callee = TopNode->Callees.find_element( + // If we've seen this callee before, then we access that node and place that + // on the top of the stack. + auto* Callee = TopNode->Callees.find_element( [FId](const NodeIdPair &NR) { return NR.FId == FId; }); if (Callee != nullptr) { CHECK_NE(Callee->NodePtr, nullptr); - ShadowStack.AppendEmplace(TSC, Callee->NodePtr); + if (ShadowStack.AppendEmplace(TSC, Callee->NodePtr, CPU) == nullptr) + ++OverflowedFunctions; return; } // This means we've never seen this stack before, create a new node here. - auto NewNode = - Nodes.AppendEmplace(TopNode, *NodeIdPairAllocator, 0, 0, FId); + auto* NewNode = Nodes.AppendEmplace( + TopNode, NodeIdPairArray(*NodeIdPairAllocator), 0u, 0u, FId); if (UNLIKELY(NewNode == nullptr)) return; DCHECK_NE(NewNode, nullptr); TopNode->Callees.AppendEmplace(NewNode, FId); - ShadowStack.AppendEmplace(TSC, NewNode); - DCHECK_NE(ShadowStack.back().NodePtr, nullptr); + if (ShadowStack.AppendEmplace(TSC, NewNode, CPU) == nullptr) + ++OverflowedFunctions; return; } - void exitFunction(int32_t FId, uint64_t TSC) { + void exitFunction(int32_t FId, uint64_t TSC, + uint16_t CPU) XRAY_NEVER_INSTRUMENT { + // If we're exiting functions that have "overflowed" or don't fit into the + // stack due to allocator constraints, we then decrement that count first. + if (OverflowedFunctions) { + --OverflowedFunctions; + return; + } + // When we exit a function, we look up the ShadowStack to see whether we've // entered this function before. We do as little processing here as we can, // since most of the hard work would have already been done at function // entry. uint64_t CumulativeTreeTime = 0; + while (!ShadowStack.empty()) { const auto &Top = ShadowStack.back(); auto TopNode = Top.NodePtr; DCHECK_NE(TopNode, nullptr); - auto LocalTime = TSC - Top.EntryTSC; + + // We may encounter overflow on the TSC we're provided, which may end up + // being less than the TSC when we first entered the function. + // + // To get the accurate measurement of cycles, we need to check whether + // we've overflowed (TSC < Top.EntryTSC) and then account the difference + // between the entry TSC and the max for the TSC counter (max of uint64_t) + // then add the value of TSC. We can prove that the maximum delta we will + // get is at most the 64-bit unsigned value, since the difference between + // a TSC of 0 and a Top.EntryTSC of 1 is (numeric_limits<uint64_t>::max() + // - 1) + 1. + // + // NOTE: This assumes that TSCs are synchronised across CPUs. + // TODO: Count the number of times we've seen CPU migrations. + uint64_t LocalTime = + Top.EntryTSC > TSC + ? (std::numeric_limits<uint64_t>::max() - Top.EntryTSC) + TSC + : TSC - Top.EntryTSC; TopNode->CallCount++; TopNode->CumulativeLocalTime += LocalTime - CumulativeTreeTime; CumulativeTreeTime += LocalTime; @@ -323,7 +463,7 @@ public: } } - const RootArray &getRoots() const { return Roots; } + const RootArray &getRoots() const XRAY_NEVER_INSTRUMENT { return Roots; } // The deepCopyInto operation will update the provided FunctionCallTrie by // re-creating the contents of this particular FunctionCallTrie in the other @@ -338,7 +478,7 @@ public: // synchronisation of both "this" and |O|. // // This function must *not* be called with a non-empty FunctionCallTrie |O|. - void deepCopyInto(FunctionCallTrie &O) const { + void deepCopyInto(FunctionCallTrie &O) const XRAY_NEVER_INSTRUMENT { DCHECK(O.getRoots().empty()); // We then push the root into a stack, to use as the parent marker for new @@ -356,18 +496,20 @@ public: for (const auto Root : getRoots()) { // Add a node in O for this root. auto NewRoot = O.Nodes.AppendEmplace( - nullptr, *O.NodeIdPairAllocator, Root->CallCount, + nullptr, NodeIdPairArray(*O.NodeIdPairAllocator), Root->CallCount, Root->CumulativeLocalTime, Root->FId); // Because we cannot allocate more memory we should bail out right away. if (UNLIKELY(NewRoot == nullptr)) return; - O.Roots.Append(NewRoot); + if (UNLIKELY(O.Roots.Append(NewRoot) == nullptr)) + return; // TODO: Figure out what to do if we fail to allocate any more stack // space. Maybe warn or report once? - DFSStack.AppendEmplace(Root, NewRoot); + if (DFSStack.AppendEmplace(Root, NewRoot) == nullptr) + return; while (!DFSStack.empty()) { NodeAndParent NP = DFSStack.back(); DCHECK_NE(NP.Node, nullptr); @@ -375,12 +517,17 @@ public: DFSStack.trim(1); for (const auto Callee : NP.Node->Callees) { auto NewNode = O.Nodes.AppendEmplace( - NP.NewNode, *O.NodeIdPairAllocator, Callee.NodePtr->CallCount, - Callee.NodePtr->CumulativeLocalTime, Callee.FId); + NP.NewNode, NodeIdPairArray(*O.NodeIdPairAllocator), + Callee.NodePtr->CallCount, Callee.NodePtr->CumulativeLocalTime, + Callee.FId); if (UNLIKELY(NewNode == nullptr)) return; - NP.NewNode->Callees.AppendEmplace(NewNode, Callee.FId); - DFSStack.AppendEmplace(Callee.NodePtr, NewNode); + if (UNLIKELY(NP.NewNode->Callees.AppendEmplace(NewNode, Callee.FId) == + nullptr)) + return; + if (UNLIKELY(DFSStack.AppendEmplace(Callee.NodePtr, NewNode) == + nullptr)) + return; } } } @@ -394,7 +541,7 @@ public: // // This function is *not* thread-safe, and may require external // synchronisation of both "this" and |O|. - void mergeInto(FunctionCallTrie &O) const { + void mergeInto(FunctionCallTrie &O) const XRAY_NEVER_INSTRUMENT { struct NodeAndTarget { FunctionCallTrie::Node *OrigNode; FunctionCallTrie::Node *TargetNode; @@ -409,8 +556,9 @@ public: auto R = O.Roots.find_element( [&](const Node *Node) { return Node->FId == Root->FId; }); if (R == nullptr) { - TargetRoot = O.Nodes.AppendEmplace(nullptr, *O.NodeIdPairAllocator, 0, - 0, Root->FId); + TargetRoot = O.Nodes.AppendEmplace( + nullptr, NodeIdPairArray(*O.NodeIdPairAllocator), 0u, 0u, + Root->FId); if (UNLIKELY(TargetRoot == nullptr)) return; @@ -419,7 +567,7 @@ public: TargetRoot = *R; } - DFSStack.Append(NodeAndTarget{Root, TargetRoot}); + DFSStack.AppendEmplace(Root, TargetRoot); while (!DFSStack.empty()) { NodeAndTarget NT = DFSStack.back(); DCHECK_NE(NT.OrigNode, nullptr); @@ -435,7 +583,8 @@ public: }); if (TargetCallee == nullptr) { auto NewTargetNode = O.Nodes.AppendEmplace( - NT.TargetNode, *O.NodeIdPairAllocator, 0, 0, Callee.FId); + NT.TargetNode, NodeIdPairArray(*O.NodeIdPairAllocator), 0u, 0u, + Callee.FId); if (UNLIKELY(NewTargetNode == nullptr)) return; diff --git a/contrib/compiler-rt/lib/xray/xray_init.cc b/contrib/compiler-rt/lib/xray/xray_init.cc index b4e069795195..b0922aa8e379 100644 --- a/contrib/compiler-rt/lib/xray/xray_init.cc +++ b/contrib/compiler-rt/lib/xray/xray_init.cc @@ -27,6 +27,15 @@ extern const XRaySledEntry __start_xray_instr_map[] __attribute__((weak)); extern const XRaySledEntry __stop_xray_instr_map[] __attribute__((weak)); extern const XRayFunctionSledIndex __start_xray_fn_idx[] __attribute__((weak)); extern const XRayFunctionSledIndex __stop_xray_fn_idx[] __attribute__((weak)); + +#if SANITIZER_MAC +// HACK: This is a temporary workaround to make XRay build on +// Darwin, but it will probably not work at runtime. +const XRaySledEntry __start_xray_instr_map[] = {}; +extern const XRaySledEntry __stop_xray_instr_map[] = {}; +extern const XRayFunctionSledIndex __start_xray_fn_idx[] = {}; +extern const XRayFunctionSledIndex __stop_xray_fn_idx[] = {}; +#endif } using namespace __xray; @@ -58,6 +67,9 @@ void __xray_init() XRAY_NEVER_INSTRUMENT { if (atomic_load(&XRayInitialized, memory_order_acquire)) return; + // XRAY is not compatible with PaX MPROTECT + CheckMPROTECT(); + if (!atomic_load(&XRayFlagsInitialized, memory_order_acquire)) { initializeFlags(); atomic_store(&XRayFlagsInitialized, true, memory_order_release); @@ -97,8 +109,8 @@ __attribute__((section(".preinit_array"), #else // If we cannot use the .preinit_array section, we should instead use dynamic // initialisation. -static bool UNUSED __local_xray_dyninit = [] { +__attribute__ ((constructor (0))) +static void __local_xray_dyninit() { __xray_init(); - return true; -}(); +} #endif diff --git a/contrib/compiler-rt/lib/xray/xray_interface.cc b/contrib/compiler-rt/lib/xray/xray_interface.cc index 01bf6ddc607e..6f7b6615b2c0 100644 --- a/contrib/compiler-rt/lib/xray/xray_interface.cc +++ b/contrib/compiler-rt/lib/xray/xray_interface.cc @@ -22,6 +22,13 @@ #include <string.h> #include <sys/mman.h> +#if SANITIZER_FUCHSIA +#include <zircon/process.h> +#include <zircon/sanitizer.h> +#include <zircon/status.h> +#include <zircon/syscalls.h> +#endif + #include "sanitizer_common/sanitizer_addrhashmap.h" #include "sanitizer_common/sanitizer_common.h" @@ -92,22 +99,48 @@ class MProtectHelper { public: explicit MProtectHelper(void *PageAlignedAddr, - std::size_t MProtectLen) XRAY_NEVER_INSTRUMENT + std::size_t MProtectLen, + std::size_t PageSize) XRAY_NEVER_INSTRUMENT : PageAlignedAddr(PageAlignedAddr), MProtectLen(MProtectLen), - MustCleanup(false) {} + MustCleanup(false) { +#if SANITIZER_FUCHSIA + MProtectLen = RoundUpTo(MProtectLen, PageSize); +#endif + } int MakeWriteable() XRAY_NEVER_INSTRUMENT { +#if SANITIZER_FUCHSIA + auto R = __sanitizer_change_code_protection( + reinterpret_cast<uintptr_t>(PageAlignedAddr), MProtectLen, true); + if (R != ZX_OK) { + Report("XRay: cannot change code protection: %s\n", + _zx_status_get_string(R)); + return -1; + } + MustCleanup = true; + return 0; +#else auto R = mprotect(PageAlignedAddr, MProtectLen, PROT_READ | PROT_WRITE | PROT_EXEC); if (R != -1) MustCleanup = true; return R; +#endif } ~MProtectHelper() XRAY_NEVER_INSTRUMENT { if (MustCleanup) { +#if SANITIZER_FUCHSIA + auto R = __sanitizer_change_code_protection( + reinterpret_cast<uintptr_t>(PageAlignedAddr), MProtectLen, false); + if (R != ZX_OK) { + Report("XRay: cannot change code protection: %s\n", + _zx_status_get_string(R)); + } +#else mprotect(PageAlignedAddr, MProtectLen, PROT_READ | PROT_EXEC); +#endif } } }; @@ -254,7 +287,7 @@ XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT { reinterpret_cast<void *>(MinSled.Address & ~(PageSize - 1)); size_t MProtectLen = (MaxSled.Address - reinterpret_cast<uptr>(PageAlignedAddr)) + cSledLength; - MProtectHelper Protector(PageAlignedAddr, MProtectLen); + MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize); if (Protector.MakeWriteable() == -1) { Report("Failed mprotect: %d\n", errno); return XRayPatchingStatus::FAILED; @@ -319,7 +352,7 @@ XRayPatchingStatus mprotectAndPatchFunction(int32_t FuncId, reinterpret_cast<void *>(MinSled.Address & ~(PageSize - 1)); size_t MProtectLen = (MaxSled.Address - reinterpret_cast<uptr>(PageAlignedAddr)) + cSledLength; - MProtectHelper Protector(PageAlignedAddr, MProtectLen); + MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize); if (Protector.MakeWriteable() == -1) { Report("Failed mprotect: %d\n", errno); return XRayPatchingStatus::FAILED; diff --git a/contrib/compiler-rt/lib/xray/xray_profile_collector.cc b/contrib/compiler-rt/lib/xray/xray_profile_collector.cc index 17a611eeacb8..dc3a82069840 100644 --- a/contrib/compiler-rt/lib/xray/xray_profile_collector.cc +++ b/contrib/compiler-rt/lib/xray/xray_profile_collector.cc @@ -13,10 +13,11 @@ // //===----------------------------------------------------------------------===// #include "xray_profile_collector.h" -#include "sanitizer_common/sanitizer_allocator_internal.h" #include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_vector.h" +#include "xray_allocator.h" +#include "xray_defs.h" #include "xray_profiling_flags.h" +#include "xray_segmented_array.h" #include <memory> #include <pthread.h> #include <utility> @@ -29,7 +30,7 @@ namespace { SpinMutex GlobalMutex; struct ThreadTrie { tid_t TId; - FunctionCallTrie *Trie; + typename std::aligned_storage<sizeof(FunctionCallTrie)>::type TrieStorage; }; struct ProfileBuffer { @@ -56,65 +57,91 @@ struct BlockHeader { u64 ThreadId; }; -// These need to be pointers that point to heap/internal-allocator-allocated -// objects because these are accessed even at program exit. -Vector<ThreadTrie> *ThreadTries = nullptr; -Vector<ProfileBuffer> *ProfileBuffers = nullptr; -FunctionCallTrie::Allocators *GlobalAllocators = nullptr; +struct ThreadData { + BufferQueue *BQ; + FunctionCallTrie::Allocators::Buffers Buffers; + FunctionCallTrie::Allocators Allocators; + FunctionCallTrie FCT; + tid_t TId; +}; + +using ThreadDataArray = Array<ThreadData>; +using ThreadDataAllocator = ThreadDataArray::AllocatorType; + +// We use a separate buffer queue for the backing store for the allocator used +// by the ThreadData array. This lets us host the buffers, allocators, and tries +// associated with a thread by moving the data into the array instead of +// attempting to copy the data to a separately backed set of tries. +static typename std::aligned_storage< + sizeof(BufferQueue), alignof(BufferQueue)>::type BufferQueueStorage; +static BufferQueue *BQ = nullptr; +static BufferQueue::Buffer Buffer; +static typename std::aligned_storage<sizeof(ThreadDataAllocator), + alignof(ThreadDataAllocator)>::type + ThreadDataAllocatorStorage; +static typename std::aligned_storage<sizeof(ThreadDataArray), + alignof(ThreadDataArray)>::type + ThreadDataArrayStorage; + +static ThreadDataAllocator *TDAllocator = nullptr; +static ThreadDataArray *TDArray = nullptr; + +using ProfileBufferArray = Array<ProfileBuffer>; +using ProfileBufferArrayAllocator = typename ProfileBufferArray::AllocatorType; + +// These need to be global aligned storage to avoid dynamic initialization. We +// need these to be aligned to allow us to placement new objects into the +// storage, and have pointers to those objects be appropriately aligned. +static typename std::aligned_storage<sizeof(ProfileBufferArray)>::type + ProfileBuffersStorage; +static typename std::aligned_storage<sizeof(ProfileBufferArrayAllocator)>::type + ProfileBufferArrayAllocatorStorage; + +static ProfileBufferArrayAllocator *ProfileBuffersAllocator = nullptr; +static ProfileBufferArray *ProfileBuffers = nullptr; + +// Use a global flag to determine whether the collector implementation has been +// initialized. +static atomic_uint8_t CollectorInitialized{0}; } // namespace -void post(const FunctionCallTrie &T, tid_t TId) { - static pthread_once_t Once = PTHREAD_ONCE_INIT; - pthread_once(&Once, +[] { - SpinMutexLock Lock(&GlobalMutex); - GlobalAllocators = reinterpret_cast<FunctionCallTrie::Allocators *>( - InternalAlloc(sizeof(FunctionCallTrie::Allocators))); - new (GlobalAllocators) FunctionCallTrie::Allocators(); - *GlobalAllocators = FunctionCallTrie::InitAllocatorsCustom( - profilingFlags()->global_allocator_max); - ThreadTries = reinterpret_cast<Vector<ThreadTrie> *>( - InternalAlloc(sizeof(Vector<ThreadTrie>))); - new (ThreadTries) Vector<ThreadTrie>(); - ProfileBuffers = reinterpret_cast<Vector<ProfileBuffer> *>( - InternalAlloc(sizeof(Vector<ProfileBuffer>))); - new (ProfileBuffers) Vector<ProfileBuffer>(); - }); - DCHECK_NE(GlobalAllocators, nullptr); - DCHECK_NE(ThreadTries, nullptr); - DCHECK_NE(ProfileBuffers, nullptr); - - ThreadTrie *Item = nullptr; +void post(BufferQueue *Q, FunctionCallTrie &&T, + FunctionCallTrie::Allocators &&A, + FunctionCallTrie::Allocators::Buffers &&B, + tid_t TId) XRAY_NEVER_INSTRUMENT { + DCHECK_NE(Q, nullptr); + + // Bail out early if the collector has not been initialized. + if (!atomic_load(&CollectorInitialized, memory_order_acquire)) { + T.~FunctionCallTrie(); + A.~Allocators(); + Q->releaseBuffer(B.NodeBuffer); + Q->releaseBuffer(B.RootsBuffer); + Q->releaseBuffer(B.ShadowStackBuffer); + Q->releaseBuffer(B.NodeIdPairBuffer); + B.~Buffers(); + return; + } + { SpinMutexLock Lock(&GlobalMutex); - if (GlobalAllocators == nullptr) - return; - - Item = ThreadTries->PushBack(); - Item->TId = TId; - - // Here we're using the internal allocator instead of the managed allocator - // because: - // - // 1) We're not using the segmented array data structure to host - // FunctionCallTrie objects. We're using a Vector (from sanitizer_common) - // which works like a std::vector<...> keeping elements contiguous in - // memory. The segmented array data structure assumes that elements are - // trivially destructible, where FunctionCallTrie isn't. - // - // 2) Using a managed allocator means we need to manage that separately, - // which complicates the nature of this code. To get around that, we're - // using the internal allocator instead, which has its own global state - // and is decoupled from the lifetime management required by the managed - // allocator we have in XRay. - // - Item->Trie = reinterpret_cast<FunctionCallTrie *>(InternalAlloc( - sizeof(FunctionCallTrie), nullptr, alignof(FunctionCallTrie))); - DCHECK_NE(Item->Trie, nullptr); - new (Item->Trie) FunctionCallTrie(*GlobalAllocators); + DCHECK_NE(TDAllocator, nullptr); + DCHECK_NE(TDArray, nullptr); + + if (TDArray->AppendEmplace(Q, std::move(B), std::move(A), std::move(T), + TId) == nullptr) { + // If we fail to add the data to the array, we should destroy the objects + // handed us. + T.~FunctionCallTrie(); + A.~Allocators(); + Q->releaseBuffer(B.NodeBuffer); + Q->releaseBuffer(B.RootsBuffer); + Q->releaseBuffer(B.ShadowStackBuffer); + Q->releaseBuffer(B.NodeIdPairBuffer); + B.~Buffers(); + } } - - T.deepCopyInto(*Item->Trie); } // A PathArray represents the function id's representing a stack trace. In this @@ -127,18 +154,8 @@ struct ProfileRecord { // The Path in this record is the function id's from the leaf to the root of // the function call stack as represented from a FunctionCallTrie. - PathArray *Path = nullptr; - const FunctionCallTrie::Node *Node = nullptr; - - // Constructor for in-place construction. - ProfileRecord(PathAllocator &A, const FunctionCallTrie::Node *N) - : Path([&] { - auto P = - reinterpret_cast<PathArray *>(InternalAlloc(sizeof(PathArray))); - new (P) PathArray(A); - return P; - }()), - Node(N) {} + PathArray Path; + const FunctionCallTrie::Node *Node; }; namespace { @@ -147,19 +164,21 @@ using ProfileRecordArray = Array<ProfileRecord>; // Walk a depth-first traversal of each root of the FunctionCallTrie to generate // the path(s) and the data associated with the path. -static void populateRecords(ProfileRecordArray &PRs, - ProfileRecord::PathAllocator &PA, - const FunctionCallTrie &Trie) { +static void +populateRecords(ProfileRecordArray &PRs, ProfileRecord::PathAllocator &PA, + const FunctionCallTrie &Trie) XRAY_NEVER_INSTRUMENT { using StackArray = Array<const FunctionCallTrie::Node *>; using StackAllocator = typename StackArray::AllocatorType; StackAllocator StackAlloc(profilingFlags()->stack_allocator_max); StackArray DFSStack(StackAlloc); - for (const auto R : Trie.getRoots()) { + for (const auto *R : Trie.getRoots()) { DFSStack.Append(R); while (!DFSStack.empty()) { - auto Node = DFSStack.back(); + auto *Node = DFSStack.back(); DFSStack.trim(1); - auto Record = PRs.AppendEmplace(PA, Node); + if (Node == nullptr) + continue; + auto Record = PRs.AppendEmplace(PathArray{PA}, Node); if (Record == nullptr) return; DCHECK_NE(Record, nullptr); @@ -167,8 +186,8 @@ static void populateRecords(ProfileRecordArray &PRs, // Traverse the Node's parents and as we're doing so, get the FIds in // the order they appear. for (auto N = Node; N != nullptr; N = N->Parent) - Record->Path->Append(N->FId); - DCHECK(!Record->Path->empty()); + Record->Path.Append(N->FId); + DCHECK(!Record->Path.empty()); for (const auto C : Node->Callees) DFSStack.Append(C.NodePtr); @@ -177,67 +196,89 @@ static void populateRecords(ProfileRecordArray &PRs, } static void serializeRecords(ProfileBuffer *Buffer, const BlockHeader &Header, - const ProfileRecordArray &ProfileRecords) { - auto NextPtr = static_cast<char *>( + const ProfileRecordArray &ProfileRecords) + XRAY_NEVER_INSTRUMENT { + auto NextPtr = static_cast<uint8_t *>( internal_memcpy(Buffer->Data, &Header, sizeof(Header))) + sizeof(Header); for (const auto &Record : ProfileRecords) { // List of IDs follow: - for (const auto FId : *Record.Path) + for (const auto FId : Record.Path) NextPtr = - static_cast<char *>(internal_memcpy(NextPtr, &FId, sizeof(FId))) + + static_cast<uint8_t *>(internal_memcpy(NextPtr, &FId, sizeof(FId))) + sizeof(FId); // Add the sentinel here. constexpr int32_t SentinelFId = 0; - NextPtr = static_cast<char *>( + NextPtr = static_cast<uint8_t *>( internal_memset(NextPtr, SentinelFId, sizeof(SentinelFId))) + sizeof(SentinelFId); // Add the node data here. NextPtr = - static_cast<char *>(internal_memcpy(NextPtr, &Record.Node->CallCount, - sizeof(Record.Node->CallCount))) + + static_cast<uint8_t *>(internal_memcpy( + NextPtr, &Record.Node->CallCount, sizeof(Record.Node->CallCount))) + sizeof(Record.Node->CallCount); - NextPtr = static_cast<char *>( + NextPtr = static_cast<uint8_t *>( internal_memcpy(NextPtr, &Record.Node->CumulativeLocalTime, sizeof(Record.Node->CumulativeLocalTime))) + sizeof(Record.Node->CumulativeLocalTime); } - DCHECK_EQ(NextPtr - static_cast<char *>(Buffer->Data), Buffer->Size); + DCHECK_EQ(NextPtr - static_cast<uint8_t *>(Buffer->Data), Buffer->Size); } } // namespace -void serialize() { +void serialize() XRAY_NEVER_INSTRUMENT { + if (!atomic_load(&CollectorInitialized, memory_order_acquire)) + return; + SpinMutexLock Lock(&GlobalMutex); - // Clear out the global ProfileBuffers. - for (uptr I = 0; I < ProfileBuffers->Size(); ++I) - InternalFree((*ProfileBuffers)[I].Data); - ProfileBuffers->Reset(); + // Clear out the global ProfileBuffers, if it's not empty. + for (auto &B : *ProfileBuffers) + deallocateBuffer(reinterpret_cast<unsigned char *>(B.Data), B.Size); + ProfileBuffers->trim(ProfileBuffers->size()); - if (ThreadTries->Size() == 0) + DCHECK_NE(TDArray, nullptr); + if (TDArray->empty()) return; // Then repopulate the global ProfileBuffers. - for (u32 I = 0; I < ThreadTries->Size(); ++I) { + u32 I = 0; + auto MaxSize = profilingFlags()->global_allocator_max; + auto ProfileArena = allocateBuffer(MaxSize); + if (ProfileArena == nullptr) + return; + + auto ProfileArenaCleanup = at_scope_exit( + [&]() XRAY_NEVER_INSTRUMENT { deallocateBuffer(ProfileArena, MaxSize); }); + + auto PathArena = allocateBuffer(profilingFlags()->global_allocator_max); + if (PathArena == nullptr) + return; + + auto PathArenaCleanup = at_scope_exit( + [&]() XRAY_NEVER_INSTRUMENT { deallocateBuffer(PathArena, MaxSize); }); + + for (const auto &ThreadTrie : *TDArray) { using ProfileRecordAllocator = typename ProfileRecordArray::AllocatorType; - ProfileRecordAllocator PRAlloc(profilingFlags()->global_allocator_max); + ProfileRecordAllocator PRAlloc(ProfileArena, + profilingFlags()->global_allocator_max); ProfileRecord::PathAllocator PathAlloc( - profilingFlags()->global_allocator_max); + PathArena, profilingFlags()->global_allocator_max); ProfileRecordArray ProfileRecords(PRAlloc); // First, we want to compute the amount of space we're going to need. We'll // use a local allocator and an __xray::Array<...> to store the intermediary // data, then compute the size as we're going along. Then we'll allocate the // contiguous space to contain the thread buffer data. - const auto &Trie = *(*ThreadTries)[I].Trie; - if (Trie.getRoots().empty()) + if (ThreadTrie.FCT.getRoots().empty()) continue; - populateRecords(ProfileRecords, PathAlloc, Trie); - DCHECK(!Trie.getRoots().empty()); + + populateRecords(ProfileRecords, PathAlloc, ThreadTrie.FCT); + DCHECK(!ThreadTrie.FCT.getRoots().empty()); DCHECK(!ProfileRecords.empty()); // Go through each record, to compute the sizes. @@ -251,75 +292,103 @@ void serialize() { // + end of record (8 bytes) u32 CumulativeSizes = 0; for (const auto &Record : ProfileRecords) - CumulativeSizes += 20 + (4 * Record.Path->size()); - - BlockHeader Header{16 + CumulativeSizes, I, (*ThreadTries)[I].TId}; - auto Buffer = ProfileBuffers->PushBack(); - Buffer->Size = sizeof(Header) + CumulativeSizes; - Buffer->Data = InternalAlloc(Buffer->Size, nullptr, 64); - DCHECK_NE(Buffer->Data, nullptr); - serializeRecords(Buffer, Header, ProfileRecords); - - // Now clean up the ProfileRecords array, one at a time. - for (auto &Record : ProfileRecords) { - Record.Path->~PathArray(); - InternalFree(Record.Path); - } + CumulativeSizes += 20 + (4 * Record.Path.size()); + + BlockHeader Header{16 + CumulativeSizes, I++, ThreadTrie.TId}; + auto B = ProfileBuffers->Append({}); + B->Size = sizeof(Header) + CumulativeSizes; + B->Data = allocateBuffer(B->Size); + DCHECK_NE(B->Data, nullptr); + serializeRecords(B, Header, ProfileRecords); } } -void reset() { +void reset() XRAY_NEVER_INSTRUMENT { + atomic_store(&CollectorInitialized, 0, memory_order_release); SpinMutexLock Lock(&GlobalMutex); + if (ProfileBuffers != nullptr) { // Clear out the profile buffers that have been serialized. - for (uptr I = 0; I < ProfileBuffers->Size(); ++I) - InternalFree((*ProfileBuffers)[I].Data); - ProfileBuffers->Reset(); - InternalFree(ProfileBuffers); + for (auto &B : *ProfileBuffers) + deallocateBuffer(reinterpret_cast<uint8_t *>(B.Data), B.Size); + ProfileBuffers->trim(ProfileBuffers->size()); ProfileBuffers = nullptr; } - if (ThreadTries != nullptr) { - // Clear out the function call tries per thread. - for (uptr I = 0; I < ThreadTries->Size(); ++I) { - auto &T = (*ThreadTries)[I]; - T.Trie->~FunctionCallTrie(); - InternalFree(T.Trie); + if (TDArray != nullptr) { + // Release the resources as required. + for (auto &TD : *TDArray) { + TD.BQ->releaseBuffer(TD.Buffers.NodeBuffer); + TD.BQ->releaseBuffer(TD.Buffers.RootsBuffer); + TD.BQ->releaseBuffer(TD.Buffers.ShadowStackBuffer); + TD.BQ->releaseBuffer(TD.Buffers.NodeIdPairBuffer); } - ThreadTries->Reset(); - InternalFree(ThreadTries); - ThreadTries = nullptr; + // We don't bother destroying the array here because we've already + // potentially freed the backing store for the array. Instead we're going to + // reset the pointer to nullptr, and re-use the storage later instead + // (placement-new'ing into the storage as-is). + TDArray = nullptr; + } + + if (TDAllocator != nullptr) { + TDAllocator->~Allocator(); + TDAllocator = nullptr; + } + + if (Buffer.Data != nullptr) { + BQ->releaseBuffer(Buffer); } - // Reset the global allocators. - if (GlobalAllocators != nullptr) { - GlobalAllocators->~Allocators(); - InternalFree(GlobalAllocators); - GlobalAllocators = nullptr; + if (BQ == nullptr) { + bool Success = false; + new (&BufferQueueStorage) + BufferQueue(profilingFlags()->global_allocator_max, 1, Success); + if (!Success) + return; + BQ = reinterpret_cast<BufferQueue *>(&BufferQueueStorage); + } else { + BQ->finalize(); + + if (BQ->init(profilingFlags()->global_allocator_max, 1) != + BufferQueue::ErrorCode::Ok) + return; } - GlobalAllocators = reinterpret_cast<FunctionCallTrie::Allocators *>( - InternalAlloc(sizeof(FunctionCallTrie::Allocators))); - new (GlobalAllocators) FunctionCallTrie::Allocators(); - *GlobalAllocators = FunctionCallTrie::InitAllocators(); - ThreadTries = reinterpret_cast<Vector<ThreadTrie> *>( - InternalAlloc(sizeof(Vector<ThreadTrie>))); - new (ThreadTries) Vector<ThreadTrie>(); - ProfileBuffers = reinterpret_cast<Vector<ProfileBuffer> *>( - InternalAlloc(sizeof(Vector<ProfileBuffer>))); - new (ProfileBuffers) Vector<ProfileBuffer>(); + + if (BQ->getBuffer(Buffer) != BufferQueue::ErrorCode::Ok) + return; + + new (&ProfileBufferArrayAllocatorStorage) + ProfileBufferArrayAllocator(profilingFlags()->global_allocator_max); + ProfileBuffersAllocator = reinterpret_cast<ProfileBufferArrayAllocator *>( + &ProfileBufferArrayAllocatorStorage); + + new (&ProfileBuffersStorage) ProfileBufferArray(*ProfileBuffersAllocator); + ProfileBuffers = + reinterpret_cast<ProfileBufferArray *>(&ProfileBuffersStorage); + + new (&ThreadDataAllocatorStorage) + ThreadDataAllocator(Buffer.Data, Buffer.Size); + TDAllocator = + reinterpret_cast<ThreadDataAllocator *>(&ThreadDataAllocatorStorage); + new (&ThreadDataArrayStorage) ThreadDataArray(*TDAllocator); + TDArray = reinterpret_cast<ThreadDataArray *>(&ThreadDataArrayStorage); + + atomic_store(&CollectorInitialized, 1, memory_order_release); } -XRayBuffer nextBuffer(XRayBuffer B) { +XRayBuffer nextBuffer(XRayBuffer B) XRAY_NEVER_INSTRUMENT { SpinMutexLock Lock(&GlobalMutex); - if (ProfileBuffers == nullptr || ProfileBuffers->Size() == 0) + if (ProfileBuffers == nullptr || ProfileBuffers->size() == 0) return {nullptr, 0}; static pthread_once_t Once = PTHREAD_ONCE_INIT; static typename std::aligned_storage<sizeof(XRayProfilingFileHeader)>::type FileHeaderStorage; - pthread_once(&Once, - +[] { new (&FileHeaderStorage) XRayProfilingFileHeader{}; }); + pthread_once( + &Once, +[]() XRAY_NEVER_INSTRUMENT { + new (&FileHeaderStorage) XRayProfilingFileHeader{}; + }); if (UNLIKELY(B.Data == nullptr)) { // The first buffer should always contain the file header information. @@ -336,7 +405,7 @@ XRayBuffer nextBuffer(XRayBuffer B) { BlockHeader Header; internal_memcpy(&Header, B.Data, sizeof(BlockHeader)); auto NextBlock = Header.BlockNum + 1; - if (NextBlock < ProfileBuffers->Size()) + if (NextBlock < ProfileBuffers->size()) return {(*ProfileBuffers)[NextBlock].Data, (*ProfileBuffers)[NextBlock].Size}; return {nullptr, 0}; diff --git a/contrib/compiler-rt/lib/xray/xray_profile_collector.h b/contrib/compiler-rt/lib/xray/xray_profile_collector.h index 335043db9526..86c4ce853797 100644 --- a/contrib/compiler-rt/lib/xray/xray_profile_collector.h +++ b/contrib/compiler-rt/lib/xray/xray_profile_collector.h @@ -33,27 +33,13 @@ namespace profileCollectorService { /// Posts the FunctionCallTrie associated with a specific Thread ID. This /// will: /// -/// - Make a copy of the FunctionCallTrie and store that against the Thread -/// ID. This will use the global allocator for the service-managed -/// FunctionCallTrie instances. -/// - Queue up a pointer to the FunctionCallTrie. -/// - If the queue is long enough (longer than some arbitrary threshold) we -/// then pre-calculate a single FunctionCallTrie for the whole process. +/// Moves the collection of FunctionCallTrie, Allocators, and Buffers associated +/// with a thread's data to the queue. This takes ownership of the memory +/// associated with a thread, and manages those exclusively. /// -/// -/// We are making a copy of the FunctionCallTrie because the intent is to have -/// this function be called at thread exit, or soon after the profiling -/// handler is finalized through the XRay APIs. By letting threads each -/// process their own thread-local FunctionCallTrie instances, we're removing -/// the need for synchronisation across threads while we're profiling. -/// However, once we're done profiling, we can then collect copies of these -/// FunctionCallTrie instances and pay the cost of the copy. -/// -/// NOTE: In the future, if this turns out to be more costly than "moving" the -/// FunctionCallTrie instances from the owning thread to the collector -/// service, then we can change the implementation to do it this way (moving) -/// instead. -void post(const FunctionCallTrie &T, tid_t TId); +void post(BufferQueue *Q, FunctionCallTrie &&T, + FunctionCallTrie::Allocators &&A, + FunctionCallTrie::Allocators::Buffers &&B, tid_t TId); /// The serialize will process all FunctionCallTrie instances in memory, and /// turn those into specifically formatted blocks, each describing the diff --git a/contrib/compiler-rt/lib/xray/xray_profiling.cc b/contrib/compiler-rt/lib/xray/xray_profiling.cc index d4b4345d764a..4323170cd1bb 100644 --- a/contrib/compiler-rt/lib/xray/xray_profiling.cc +++ b/contrib/compiler-rt/lib/xray/xray_profiling.cc @@ -19,7 +19,7 @@ #include "sanitizer_common/sanitizer_flags.h" #include "xray/xray_interface.h" #include "xray/xray_log_interface.h" - +#include "xray_buffer_queue.h" #include "xray_flags.h" #include "xray_profile_collector.h" #include "xray_profiling_flags.h" @@ -32,62 +32,167 @@ namespace __xray { namespace { -atomic_sint32_t ProfilerLogFlushStatus = { +static atomic_sint32_t ProfilerLogFlushStatus = { XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING}; -atomic_sint32_t ProfilerLogStatus = {XRayLogInitStatus::XRAY_LOG_UNINITIALIZED}; +static atomic_sint32_t ProfilerLogStatus = { + XRayLogInitStatus::XRAY_LOG_UNINITIALIZED}; -SpinMutex ProfilerOptionsMutex; +static SpinMutex ProfilerOptionsMutex; -struct alignas(64) ProfilingData { - FunctionCallTrie::Allocators *Allocators = nullptr; - FunctionCallTrie *FCT = nullptr; +struct ProfilingData { + atomic_uintptr_t Allocators; + atomic_uintptr_t FCT; }; static pthread_key_t ProfilingKey; -thread_local std::aligned_storage<sizeof(ProfilingData)>::type ThreadStorage{}; -static ProfilingData &getThreadLocalData() XRAY_NEVER_INSTRUMENT { - thread_local auto ThreadOnce = [] { - new (&ThreadStorage) ProfilingData{}; - pthread_setspecific(ProfilingKey, &ThreadStorage); +// We use a global buffer queue, which gets initialized once at initialisation +// time, and gets reset when profiling is "done". +static std::aligned_storage<sizeof(BufferQueue), alignof(BufferQueue)>::type + BufferQueueStorage; +static BufferQueue *BQ = nullptr; + +thread_local FunctionCallTrie::Allocators::Buffers ThreadBuffers; +thread_local std::aligned_storage<sizeof(FunctionCallTrie::Allocators), + alignof(FunctionCallTrie::Allocators)>::type + AllocatorsStorage; +thread_local std::aligned_storage<sizeof(FunctionCallTrie), + alignof(FunctionCallTrie)>::type + FunctionCallTrieStorage; +thread_local ProfilingData TLD{{0}, {0}}; +thread_local atomic_uint8_t ReentranceGuard{0}; + +// We use a separate guard for ensuring that for this thread, if we're already +// cleaning up, that any signal handlers don't attempt to cleanup nor +// initialise. +thread_local atomic_uint8_t TLDInitGuard{0}; + +// We also use a separate latch to signal that the thread is exiting, and +// non-essential work should be ignored (things like recording events, etc.). +thread_local atomic_uint8_t ThreadExitingLatch{0}; + +static ProfilingData *getThreadLocalData() XRAY_NEVER_INSTRUMENT { + thread_local auto ThreadOnce = []() XRAY_NEVER_INSTRUMENT { + pthread_setspecific(ProfilingKey, &TLD); return false; }(); (void)ThreadOnce; - auto &TLD = *reinterpret_cast<ProfilingData *>(&ThreadStorage); + RecursionGuard TLDInit(TLDInitGuard); + if (!TLDInit) + return nullptr; - // We need to check whether the global flag to finalizing/finalized has been - // switched. If it is, then we ought to not actually initialise the data. - auto Status = atomic_load(&ProfilerLogStatus, memory_order_acquire); - if (Status == XRayLogInitStatus::XRAY_LOG_FINALIZING || - Status == XRayLogInitStatus::XRAY_LOG_FINALIZED) - return TLD; - - // If we're live, then we re-initialize TLD if the pointers are not null. - if (UNLIKELY(TLD.Allocators == nullptr && TLD.FCT == nullptr)) { - TLD.Allocators = reinterpret_cast<FunctionCallTrie::Allocators *>( - InternalAlloc(sizeof(FunctionCallTrie::Allocators))); - new (TLD.Allocators) FunctionCallTrie::Allocators(); - *TLD.Allocators = FunctionCallTrie::InitAllocators(); - TLD.FCT = reinterpret_cast<FunctionCallTrie *>( - InternalAlloc(sizeof(FunctionCallTrie))); - new (TLD.FCT) FunctionCallTrie(*TLD.Allocators); + if (atomic_load_relaxed(&ThreadExitingLatch)) + return nullptr; + + uptr Allocators = 0; + if (atomic_compare_exchange_strong(&TLD.Allocators, &Allocators, 1, + memory_order_acq_rel)) { + bool Success = false; + auto AllocatorsUndo = at_scope_exit([&]() XRAY_NEVER_INSTRUMENT { + if (!Success) + atomic_store(&TLD.Allocators, 0, memory_order_release); + }); + + // Acquire a set of buffers for this thread. + if (BQ == nullptr) + return nullptr; + + if (BQ->getBuffer(ThreadBuffers.NodeBuffer) != BufferQueue::ErrorCode::Ok) + return nullptr; + auto NodeBufferUndo = at_scope_exit([&]() XRAY_NEVER_INSTRUMENT { + if (!Success) + BQ->releaseBuffer(ThreadBuffers.NodeBuffer); + }); + + if (BQ->getBuffer(ThreadBuffers.RootsBuffer) != BufferQueue::ErrorCode::Ok) + return nullptr; + auto RootsBufferUndo = at_scope_exit([&]() XRAY_NEVER_INSTRUMENT { + if (!Success) + BQ->releaseBuffer(ThreadBuffers.RootsBuffer); + }); + + if (BQ->getBuffer(ThreadBuffers.ShadowStackBuffer) != + BufferQueue::ErrorCode::Ok) + return nullptr; + auto ShadowStackBufferUndo = at_scope_exit([&]() XRAY_NEVER_INSTRUMENT { + if (!Success) + BQ->releaseBuffer(ThreadBuffers.ShadowStackBuffer); + }); + + if (BQ->getBuffer(ThreadBuffers.NodeIdPairBuffer) != + BufferQueue::ErrorCode::Ok) + return nullptr; + + Success = true; + new (&AllocatorsStorage) FunctionCallTrie::Allocators( + FunctionCallTrie::InitAllocatorsFromBuffers(ThreadBuffers)); + Allocators = reinterpret_cast<uptr>( + reinterpret_cast<FunctionCallTrie::Allocators *>(&AllocatorsStorage)); + atomic_store(&TLD.Allocators, Allocators, memory_order_release); + } + + if (Allocators == 1) + return nullptr; + + uptr FCT = 0; + if (atomic_compare_exchange_strong(&TLD.FCT, &FCT, 1, memory_order_acq_rel)) { + new (&FunctionCallTrieStorage) + FunctionCallTrie(*reinterpret_cast<FunctionCallTrie::Allocators *>( + atomic_load_relaxed(&TLD.Allocators))); + FCT = reinterpret_cast<uptr>( + reinterpret_cast<FunctionCallTrie *>(&FunctionCallTrieStorage)); + atomic_store(&TLD.FCT, FCT, memory_order_release); } - return TLD; + if (FCT == 1) + return nullptr; + + return &TLD; } static void cleanupTLD() XRAY_NEVER_INSTRUMENT { - auto &TLD = *reinterpret_cast<ProfilingData *>(&ThreadStorage); - if (TLD.Allocators != nullptr && TLD.FCT != nullptr) { - TLD.FCT->~FunctionCallTrie(); - TLD.Allocators->~Allocators(); - InternalFree(TLD.FCT); - InternalFree(TLD.Allocators); - TLD.FCT = nullptr; - TLD.Allocators = nullptr; - } + auto FCT = atomic_exchange(&TLD.FCT, 0, memory_order_acq_rel); + if (FCT == reinterpret_cast<uptr>(reinterpret_cast<FunctionCallTrie *>( + &FunctionCallTrieStorage))) + reinterpret_cast<FunctionCallTrie *>(FCT)->~FunctionCallTrie(); + + auto Allocators = atomic_exchange(&TLD.Allocators, 0, memory_order_acq_rel); + if (Allocators == + reinterpret_cast<uptr>( + reinterpret_cast<FunctionCallTrie::Allocators *>(&AllocatorsStorage))) + reinterpret_cast<FunctionCallTrie::Allocators *>(Allocators)->~Allocators(); +} + +static void postCurrentThreadFCT(ProfilingData &T) XRAY_NEVER_INSTRUMENT { + RecursionGuard TLDInit(TLDInitGuard); + if (!TLDInit) + return; + + uptr P = atomic_exchange(&T.FCT, 0, memory_order_acq_rel); + if (P != reinterpret_cast<uptr>( + reinterpret_cast<FunctionCallTrie *>(&FunctionCallTrieStorage))) + return; + + auto FCT = reinterpret_cast<FunctionCallTrie *>(P); + DCHECK_NE(FCT, nullptr); + + uptr A = atomic_exchange(&T.Allocators, 0, memory_order_acq_rel); + if (A != + reinterpret_cast<uptr>( + reinterpret_cast<FunctionCallTrie::Allocators *>(&AllocatorsStorage))) + return; + + auto Allocators = reinterpret_cast<FunctionCallTrie::Allocators *>(A); + DCHECK_NE(Allocators, nullptr); + + // Always move the data into the profile collector. + profileCollectorService::post(BQ, std::move(*FCT), std::move(*Allocators), + std::move(ThreadBuffers), GetTid()); + + // Re-initialize the ThreadBuffers object to a known "default" state. + ThreadBuffers = FunctionCallTrie::Allocators::Buffers{}; } } // namespace @@ -100,9 +205,6 @@ const char *profilingCompilerDefinedFlags() XRAY_NEVER_INSTRUMENT { #endif } -atomic_sint32_t ProfileFlushStatus = { - XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING}; - XRayLogFlushStatus profilingFlush() XRAY_NEVER_INSTRUMENT { if (atomic_load(&ProfilerLogStatus, memory_order_acquire) != XRayLogInitStatus::XRAY_LOG_FINALIZED) { @@ -111,12 +213,23 @@ XRayLogFlushStatus profilingFlush() XRAY_NEVER_INSTRUMENT { return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; } - s32 Result = XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; - if (!atomic_compare_exchange_strong(&ProfilerLogFlushStatus, &Result, - XRayLogFlushStatus::XRAY_LOG_FLUSHING, - memory_order_acq_rel)) { + RecursionGuard SignalGuard(ReentranceGuard); + if (!SignalGuard) { + if (Verbosity()) + Report("Cannot finalize properly inside a signal handler!\n"); + atomic_store(&ProfilerLogFlushStatus, + XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING, + memory_order_release); + return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; + } + + s32 Previous = atomic_exchange(&ProfilerLogFlushStatus, + XRayLogFlushStatus::XRAY_LOG_FLUSHING, + memory_order_acq_rel); + if (Previous == XRayLogFlushStatus::XRAY_LOG_FLUSHING) { if (Verbosity()) - Report("Not flushing profiles, implementation still finalizing.\n"); + Report("Not flushing profiles, implementation still flushing.\n"); + return XRayLogFlushStatus::XRAY_LOG_FLUSHING; } // At this point, we'll create the file that will contain the profile, but @@ -129,49 +242,33 @@ XRayLogFlushStatus profilingFlush() XRAY_NEVER_INSTRUMENT { if (Verbosity()) Report("profiling: No data to flush.\n"); } else { - int Fd = getLogFD(); - if (Fd == -1) { + LogWriter *LW = LogWriter::Open(); + if (LW == nullptr) { if (Verbosity()) Report("profiling: Failed to flush to file, dropping data.\n"); } else { // Now for each of the buffers, write out the profile data as we would // see it in memory, verbatim. while (B.Data != nullptr && B.Size != 0) { - retryingWriteAll(Fd, reinterpret_cast<const char *>(B.Data), - reinterpret_cast<const char *>(B.Data) + B.Size); + LW->WriteAll(reinterpret_cast<const char *>(B.Data), + reinterpret_cast<const char *>(B.Data) + B.Size); B = profileCollectorService::nextBuffer(B); } - // Then we close out the file. - internal_close(Fd); } + LogWriter::Close(LW); } } profileCollectorService::reset(); - // Flush the current thread's local data structures as well. - cleanupTLD(); - - atomic_store(&ProfilerLogStatus, XRayLogFlushStatus::XRAY_LOG_FLUSHED, + atomic_store(&ProfilerLogFlushStatus, XRayLogFlushStatus::XRAY_LOG_FLUSHED, + memory_order_release); + atomic_store(&ProfilerLogStatus, XRayLogInitStatus::XRAY_LOG_UNINITIALIZED, memory_order_release); return XRayLogFlushStatus::XRAY_LOG_FLUSHED; } -namespace { - -thread_local atomic_uint8_t ReentranceGuard{0}; - -static void postCurrentThreadFCT(ProfilingData &TLD) { - if (TLD.Allocators == nullptr || TLD.FCT == nullptr) - return; - - profileCollectorService::post(*TLD.FCT, GetTid()); - cleanupTLD(); -} - -} // namespace - void profilingHandleArg0(int32_t FuncId, XRayEntryType Entry) XRAY_NEVER_INSTRUMENT { unsigned char CPU; @@ -181,21 +278,29 @@ void profilingHandleArg0(int32_t FuncId, return; auto Status = atomic_load(&ProfilerLogStatus, memory_order_acquire); - auto &TLD = getThreadLocalData(); + if (UNLIKELY(Status == XRayLogInitStatus::XRAY_LOG_UNINITIALIZED || + Status == XRayLogInitStatus::XRAY_LOG_INITIALIZING)) + return; + if (UNLIKELY(Status == XRayLogInitStatus::XRAY_LOG_FINALIZED || Status == XRayLogInitStatus::XRAY_LOG_FINALIZING)) { postCurrentThreadFCT(TLD); return; } + auto T = getThreadLocalData(); + if (T == nullptr) + return; + + auto FCT = reinterpret_cast<FunctionCallTrie *>(atomic_load_relaxed(&T->FCT)); switch (Entry) { case XRayEntryType::ENTRY: case XRayEntryType::LOG_ARGS_ENTRY: - TLD.FCT->enterFunction(FuncId, TSC); + FCT->enterFunction(FuncId, TSC, CPU); break; case XRayEntryType::EXIT: case XRayEntryType::TAIL: - TLD.FCT->exitFunction(FuncId, TSC); + FCT->exitFunction(FuncId, TSC, CPU); break; default: // FIXME: Handle bugs. @@ -218,12 +323,22 @@ XRayLogInitStatus profilingFinalize() XRAY_NEVER_INSTRUMENT { return static_cast<XRayLogInitStatus>(CurrentStatus); } + // Mark then finalize the current generation of buffers. This allows us to let + // the threads currently holding onto new buffers still use them, but let the + // last reference do the memory cleanup. + DCHECK_NE(BQ, nullptr); + BQ->finalize(); + // Wait a grace period to allow threads to see that we're finalizing. SleepForMillis(profilingFlags()->grace_period_ms); - // We also want to make sure that the current thread's data is cleaned up, - // if we have any. - auto &TLD = getThreadLocalData(); + // If we for some reason are entering this function from an instrumented + // handler, we bail out. + RecursionGuard G(ReentranceGuard); + if (!G) + return static_cast<XRayLogInitStatus>(CurrentStatus); + + // Post the current thread's data if we have any. postCurrentThreadFCT(TLD); // Then we force serialize the log data. @@ -235,19 +350,16 @@ XRayLogInitStatus profilingFinalize() XRAY_NEVER_INSTRUMENT { } XRayLogInitStatus -profilingLoggingInit(size_t BufferSize, size_t BufferMax, void *Options, +profilingLoggingInit(size_t, size_t, void *Options, size_t OptionsSize) XRAY_NEVER_INSTRUMENT { - if (BufferSize != 0 || BufferMax != 0) { - if (Verbosity()) - Report("__xray_log_init() being used, and is unsupported. Use " - "__xray_log_init_mode(...) instead. Bailing out."); + RecursionGuard G(ReentranceGuard); + if (!G) return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; - } s32 CurrentStatus = XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; if (!atomic_compare_exchange_strong(&ProfilerLogStatus, &CurrentStatus, XRayLogInitStatus::XRAY_LOG_INITIALIZING, - memory_order_release)) { + memory_order_acq_rel)) { if (Verbosity()) Report("Cannot initialize already initialised profiling " "implementation.\n"); @@ -276,35 +388,88 @@ profilingLoggingInit(size_t BufferSize, size_t BufferMax, void *Options, // We need to reset the profile data collection implementation now. profileCollectorService::reset(); - // We need to set up the exit handlers. - static pthread_once_t Once = PTHREAD_ONCE_INIT; - pthread_once(&Once, +[] { - pthread_key_create(&ProfilingKey, +[](void *P) { - // This is the thread-exit handler. - auto &TLD = *reinterpret_cast<ProfilingData *>(P); - if (TLD.Allocators == nullptr && TLD.FCT == nullptr) - return; - - postCurrentThreadFCT(TLD); - }); + // Then also reset the buffer queue implementation. + if (BQ == nullptr) { + bool Success = false; + new (&BufferQueueStorage) + BufferQueue(profilingFlags()->per_thread_allocator_max, + profilingFlags()->buffers_max, Success); + if (!Success) { + if (Verbosity()) + Report("Failed to initialize preallocated memory buffers!"); + atomic_store(&ProfilerLogStatus, + XRayLogInitStatus::XRAY_LOG_UNINITIALIZED, + memory_order_release); + return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; + } - // We also need to set up an exit handler, so that we can get the profile - // information at exit time. We use the C API to do this, to not rely on C++ - // ABI functions for registering exit handlers. - Atexit(+[] { - // Finalize and flush. - if (profilingFinalize() != XRAY_LOG_FINALIZED) { - cleanupTLD(); - return; - } - if (profilingFlush() != XRAY_LOG_FLUSHED) { - cleanupTLD(); - return; - } + // If we've succeded, set the global pointer to the initialised storage. + BQ = reinterpret_cast<BufferQueue *>(&BufferQueueStorage); + } else { + BQ->finalize(); + auto InitStatus = BQ->init(profilingFlags()->per_thread_allocator_max, + profilingFlags()->buffers_max); + + if (InitStatus != BufferQueue::ErrorCode::Ok) { if (Verbosity()) - Report("XRay Profile flushed at exit."); - }); - }); + Report("Failed to initialize preallocated memory buffers; error: %s", + BufferQueue::getErrorString(InitStatus)); + atomic_store(&ProfilerLogStatus, + XRayLogInitStatus::XRAY_LOG_UNINITIALIZED, + memory_order_release); + return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; + } + + DCHECK(!BQ->finalizing()); + } + + // We need to set up the exit handlers. + static pthread_once_t Once = PTHREAD_ONCE_INIT; + pthread_once( + &Once, +[] { + pthread_key_create( + &ProfilingKey, +[](void *P) XRAY_NEVER_INSTRUMENT { + if (atomic_exchange(&ThreadExitingLatch, 1, memory_order_acq_rel)) + return; + + if (P == nullptr) + return; + + auto T = reinterpret_cast<ProfilingData *>(P); + if (atomic_load_relaxed(&T->Allocators) == 0) + return; + + { + // If we're somehow executing this while inside a + // non-reentrant-friendly context, we skip attempting to post + // the current thread's data. + RecursionGuard G(ReentranceGuard); + if (!G) + return; + + postCurrentThreadFCT(*T); + } + }); + + // We also need to set up an exit handler, so that we can get the + // profile information at exit time. We use the C API to do this, to not + // rely on C++ ABI functions for registering exit handlers. + Atexit(+[]() XRAY_NEVER_INSTRUMENT { + if (atomic_exchange(&ThreadExitingLatch, 1, memory_order_acq_rel)) + return; + + auto Cleanup = + at_scope_exit([]() XRAY_NEVER_INSTRUMENT { cleanupTLD(); }); + + // Finalize and flush. + if (profilingFinalize() != XRAY_LOG_FINALIZED || + profilingFlush() != XRAY_LOG_FLUSHED) + return; + + if (Verbosity()) + Report("XRay Profile flushed at exit."); + }); + }); __xray_log_set_buffer_iterator(profileCollectorService::nextBuffer); __xray_set_handler(profilingHandleArg0); diff --git a/contrib/compiler-rt/lib/xray/xray_profiling_flags.inc b/contrib/compiler-rt/lib/xray/xray_profiling_flags.inc index e9230ae64187..ccd70860bf61 100644 --- a/contrib/compiler-rt/lib/xray/xray_profiling_flags.inc +++ b/contrib/compiler-rt/lib/xray/xray_profiling_flags.inc @@ -14,7 +14,7 @@ #error "Define XRAY_FLAG prior to including this file!" #endif -XRAY_FLAG(uptr, per_thread_allocator_max, 2 << 20, +XRAY_FLAG(uptr, per_thread_allocator_max, 16384, "Maximum size of any single per-thread allocator.") XRAY_FLAG(uptr, global_allocator_max, 2 << 24, "Maximum size of the global allocator for profile storage.") @@ -27,3 +27,6 @@ XRAY_FLAG(int, grace_period_ms, 1, XRAY_FLAG(bool, no_flush, false, "Set to true if we want the profiling implementation to not write " "out files.") +XRAY_FLAG(int, buffers_max, 128, + "The number of buffers to pre-allocate used by the profiling " + "implementation.") diff --git a/contrib/compiler-rt/lib/xray/xray_segmented_array.h b/contrib/compiler-rt/lib/xray/xray_segmented_array.h index 11dd794fa520..bc7e9379f63b 100644 --- a/contrib/compiler-rt/lib/xray/xray_segmented_array.h +++ b/contrib/compiler-rt/lib/xray/xray_segmented_array.h @@ -32,14 +32,9 @@ namespace __xray { /// is destroyed. When an Array is destroyed, it will destroy elements in the /// backing store but will not free the memory. template <class T> class Array { - struct SegmentBase { - SegmentBase *Prev; - SegmentBase *Next; - }; - - // We want each segment of the array to be cache-line aligned, and elements of - // the array be offset from the beginning of the segment. - struct Segment : SegmentBase { + struct Segment { + Segment *Prev; + Segment *Next; char Data[1]; }; @@ -62,98 +57,46 @@ public: // kCacheLineSize-multiple segments, minus the size of two pointers. // // - Request cacheline-multiple sized elements from the allocator. - static constexpr size_t AlignedElementStorageSize = + static constexpr uint64_t AlignedElementStorageSize = sizeof(typename std::aligned_storage<sizeof(T), alignof(T)>::type); - static constexpr size_t SegmentSize = - nearest_boundary(sizeof(Segment) + next_pow2(sizeof(T)), kCacheLineSize); + static constexpr uint64_t SegmentControlBlockSize = sizeof(Segment *) * 2; + + static constexpr uint64_t SegmentSize = nearest_boundary( + SegmentControlBlockSize + next_pow2(sizeof(T)), kCacheLineSize); using AllocatorType = Allocator<SegmentSize>; - static constexpr size_t ElementsPerSegment = - (SegmentSize - sizeof(Segment)) / next_pow2(sizeof(T)); + static constexpr uint64_t ElementsPerSegment = + (SegmentSize - SegmentControlBlockSize) / next_pow2(sizeof(T)); static_assert(ElementsPerSegment > 0, "Must have at least 1 element per segment."); - static SegmentBase SentinelSegment; - -private: - AllocatorType *Alloc; - SegmentBase *Head = &SentinelSegment; - SegmentBase *Tail = &SentinelSegment; - size_t Size = 0; - - // Here we keep track of segments in the freelist, to allow us to re-use - // segments when elements are trimmed off the end. - SegmentBase *Freelist = &SentinelSegment; - - Segment *NewSegment() { - // We need to handle the case in which enough elements have been trimmed to - // allow us to re-use segments we've allocated before. For this we look into - // the Freelist, to see whether we need to actually allocate new blocks or - // just re-use blocks we've already seen before. - if (Freelist != &SentinelSegment) { - auto *FreeSegment = Freelist; - Freelist = FreeSegment->Next; - FreeSegment->Next = &SentinelSegment; - Freelist->Prev = &SentinelSegment; - return static_cast<Segment *>(FreeSegment); - } - - auto SegmentBlock = Alloc->Allocate(); - if (SegmentBlock.Data == nullptr) - return nullptr; - - // Placement-new the Segment element at the beginning of the SegmentBlock. - auto S = reinterpret_cast<Segment *>(SegmentBlock.Data); - new (S) SegmentBase{&SentinelSegment, &SentinelSegment}; - return S; - } - - Segment *InitHeadAndTail() { - DCHECK_EQ(Head, &SentinelSegment); - DCHECK_EQ(Tail, &SentinelSegment); - auto Segment = NewSegment(); - if (Segment == nullptr) - return nullptr; - DCHECK_EQ(Segment->Next, &SentinelSegment); - DCHECK_EQ(Segment->Prev, &SentinelSegment); - Head = Tail = static_cast<SegmentBase *>(Segment); - return Segment; - } + static Segment SentinelSegment; - Segment *AppendNewSegment() { - auto S = NewSegment(); - if (S == nullptr) - return nullptr; - DCHECK_NE(Tail, &SentinelSegment); - DCHECK_EQ(Tail->Next, &SentinelSegment); - DCHECK_EQ(S->Prev, &SentinelSegment); - DCHECK_EQ(S->Next, &SentinelSegment); - Tail->Next = S; - S->Prev = Tail; - Tail = S; - return static_cast<Segment *>(Tail); - } + using size_type = uint64_t; +private: // This Iterator models a BidirectionalIterator. template <class U> class Iterator { - SegmentBase *S = &SentinelSegment; - size_t Offset = 0; - size_t Size = 0; + Segment *S = &SentinelSegment; + uint64_t Offset = 0; + uint64_t Size = 0; public: - Iterator(SegmentBase *IS, size_t Off, size_t S) - : S(IS), Offset(Off), Size(S) {} - Iterator(const Iterator &) noexcept = default; - Iterator() noexcept = default; - Iterator(Iterator &&) noexcept = default; - Iterator &operator=(const Iterator &) = default; - Iterator &operator=(Iterator &&) = default; - ~Iterator() = default; - - Iterator &operator++() { + Iterator(Segment *IS, uint64_t Off, uint64_t S) XRAY_NEVER_INSTRUMENT + : S(IS), + Offset(Off), + Size(S) {} + Iterator(const Iterator &) NOEXCEPT XRAY_NEVER_INSTRUMENT = default; + Iterator() NOEXCEPT XRAY_NEVER_INSTRUMENT = default; + Iterator(Iterator &&) NOEXCEPT XRAY_NEVER_INSTRUMENT = default; + Iterator &operator=(const Iterator &) XRAY_NEVER_INSTRUMENT = default; + Iterator &operator=(Iterator &&) XRAY_NEVER_INSTRUMENT = default; + ~Iterator() XRAY_NEVER_INSTRUMENT = default; + + Iterator &operator++() XRAY_NEVER_INSTRUMENT { if (++Offset % ElementsPerSegment || Offset == Size) return *this; @@ -168,7 +111,7 @@ private: return *this; } - Iterator &operator--() { + Iterator &operator--() XRAY_NEVER_INSTRUMENT { DCHECK_NE(S, &SentinelSegment); DCHECK_GT(Offset, 0); @@ -181,107 +124,295 @@ private: return *this; } - Iterator operator++(int) { + Iterator operator++(int) XRAY_NEVER_INSTRUMENT { Iterator Copy(*this); ++(*this); return Copy; } - Iterator operator--(int) { + Iterator operator--(int) XRAY_NEVER_INSTRUMENT { Iterator Copy(*this); --(*this); return Copy; } template <class V, class W> - friend bool operator==(const Iterator<V> &L, const Iterator<W> &R) { + friend bool operator==(const Iterator<V> &L, + const Iterator<W> &R) XRAY_NEVER_INSTRUMENT { return L.S == R.S && L.Offset == R.Offset; } template <class V, class W> - friend bool operator!=(const Iterator<V> &L, const Iterator<W> &R) { + friend bool operator!=(const Iterator<V> &L, + const Iterator<W> &R) XRAY_NEVER_INSTRUMENT { return !(L == R); } - U &operator*() const { + U &operator*() const XRAY_NEVER_INSTRUMENT { DCHECK_NE(S, &SentinelSegment); auto RelOff = Offset % ElementsPerSegment; // We need to compute the character-aligned pointer, offset from the // segment's Data location to get the element in the position of Offset. - auto Base = static_cast<Segment *>(S)->Data; + auto Base = &S->Data; auto AlignedOffset = Base + (RelOff * AlignedElementStorageSize); return *reinterpret_cast<U *>(AlignedOffset); } - U *operator->() const { return &(**this); } + U *operator->() const XRAY_NEVER_INSTRUMENT { return &(**this); } }; + AllocatorType *Alloc; + Segment *Head; + Segment *Tail; + + // Here we keep track of segments in the freelist, to allow us to re-use + // segments when elements are trimmed off the end. + Segment *Freelist; + uint64_t Size; + + // =============================== + // In the following implementation, we work through the algorithms and the + // list operations using the following notation: + // + // - pred(s) is the predecessor (previous node accessor) and succ(s) is + // the successor (next node accessor). + // + // - S is a sentinel segment, which has the following property: + // + // pred(S) == succ(S) == S + // + // - @ is a loop operator, which can imply pred(s) == s if it appears on + // the left of s, or succ(s) == S if it appears on the right of s. + // + // - sL <-> sR : means a bidirectional relation between sL and sR, which + // means: + // + // succ(sL) == sR && pred(SR) == sL + // + // - sL -> sR : implies a unidirectional relation between sL and SR, + // with the following properties: + // + // succ(sL) == sR + // + // sL <- sR : implies a unidirectional relation between sR and sL, + // with the following properties: + // + // pred(sR) == sL + // + // =============================== + + Segment *NewSegment() XRAY_NEVER_INSTRUMENT { + // We need to handle the case in which enough elements have been trimmed to + // allow us to re-use segments we've allocated before. For this we look into + // the Freelist, to see whether we need to actually allocate new blocks or + // just re-use blocks we've already seen before. + if (Freelist != &SentinelSegment) { + // The current state of lists resemble something like this at this point: + // + // Freelist: @S@<-f0->...<->fN->@S@ + // ^ Freelist + // + // We want to perform a splice of `f0` from Freelist to a temporary list, + // which looks like: + // + // Templist: @S@<-f0->@S@ + // ^ FreeSegment + // + // Our algorithm preconditions are: + DCHECK_EQ(Freelist->Prev, &SentinelSegment); + + // Then the algorithm we implement is: + // + // SFS = Freelist + // Freelist = succ(Freelist) + // if (Freelist != S) + // pred(Freelist) = S + // succ(SFS) = S + // pred(SFS) = S + // + auto *FreeSegment = Freelist; + Freelist = Freelist->Next; + + // Note that we need to handle the case where Freelist is now pointing to + // S, which we don't want to be overwriting. + // TODO: Determine whether the cost of the branch is higher than the cost + // of the blind assignment. + if (Freelist != &SentinelSegment) + Freelist->Prev = &SentinelSegment; + + FreeSegment->Next = &SentinelSegment; + FreeSegment->Prev = &SentinelSegment; + + // Our postconditions are: + DCHECK_EQ(Freelist->Prev, &SentinelSegment); + DCHECK_NE(FreeSegment, &SentinelSegment); + return FreeSegment; + } + + auto SegmentBlock = Alloc->Allocate(); + if (SegmentBlock.Data == nullptr) + return nullptr; + + // Placement-new the Segment element at the beginning of the SegmentBlock. + new (SegmentBlock.Data) Segment{&SentinelSegment, &SentinelSegment, {0}}; + auto SB = reinterpret_cast<Segment *>(SegmentBlock.Data); + return SB; + } + + Segment *InitHeadAndTail() XRAY_NEVER_INSTRUMENT { + DCHECK_EQ(Head, &SentinelSegment); + DCHECK_EQ(Tail, &SentinelSegment); + auto S = NewSegment(); + if (S == nullptr) + return nullptr; + DCHECK_EQ(S->Next, &SentinelSegment); + DCHECK_EQ(S->Prev, &SentinelSegment); + DCHECK_NE(S, &SentinelSegment); + Head = S; + Tail = S; + DCHECK_EQ(Head, Tail); + DCHECK_EQ(Tail->Next, &SentinelSegment); + DCHECK_EQ(Tail->Prev, &SentinelSegment); + return S; + } + + Segment *AppendNewSegment() XRAY_NEVER_INSTRUMENT { + auto S = NewSegment(); + if (S == nullptr) + return nullptr; + DCHECK_NE(Tail, &SentinelSegment); + DCHECK_EQ(Tail->Next, &SentinelSegment); + DCHECK_EQ(S->Prev, &SentinelSegment); + DCHECK_EQ(S->Next, &SentinelSegment); + S->Prev = Tail; + Tail->Next = S; + Tail = S; + DCHECK_EQ(S, S->Prev->Next); + DCHECK_EQ(Tail->Next, &SentinelSegment); + return S; + } + public: - explicit Array(AllocatorType &A) : Alloc(&A) {} + explicit Array(AllocatorType &A) XRAY_NEVER_INSTRUMENT + : Alloc(&A), + Head(&SentinelSegment), + Tail(&SentinelSegment), + Freelist(&SentinelSegment), + Size(0) {} + + Array() XRAY_NEVER_INSTRUMENT : Alloc(nullptr), + Head(&SentinelSegment), + Tail(&SentinelSegment), + Freelist(&SentinelSegment), + Size(0) {} Array(const Array &) = delete; - Array(Array &&O) NOEXCEPT : Alloc(O.Alloc), - Head(O.Head), - Tail(O.Tail), - Size(O.Size) { + Array &operator=(const Array &) = delete; + + Array(Array &&O) XRAY_NEVER_INSTRUMENT : Alloc(O.Alloc), + Head(O.Head), + Tail(O.Tail), + Freelist(O.Freelist), + Size(O.Size) { + O.Alloc = nullptr; O.Head = &SentinelSegment; O.Tail = &SentinelSegment; O.Size = 0; + O.Freelist = &SentinelSegment; } - bool empty() const { return Size == 0; } + Array &operator=(Array &&O) XRAY_NEVER_INSTRUMENT { + Alloc = O.Alloc; + O.Alloc = nullptr; + Head = O.Head; + O.Head = &SentinelSegment; + Tail = O.Tail; + O.Tail = &SentinelSegment; + Freelist = O.Freelist; + O.Freelist = &SentinelSegment; + Size = O.Size; + O.Size = 0; + return *this; + } + + ~Array() XRAY_NEVER_INSTRUMENT { + for (auto &E : *this) + (&E)->~T(); + } - AllocatorType &allocator() const { + bool empty() const XRAY_NEVER_INSTRUMENT { return Size == 0; } + + AllocatorType &allocator() const XRAY_NEVER_INSTRUMENT { DCHECK_NE(Alloc, nullptr); return *Alloc; } - size_t size() const { return Size; } + uint64_t size() const XRAY_NEVER_INSTRUMENT { return Size; } - T *Append(const T &E) { - if (UNLIKELY(Head == &SentinelSegment)) - if (InitHeadAndTail() == nullptr) + template <class... Args> + T *AppendEmplace(Args &&... args) XRAY_NEVER_INSTRUMENT { + DCHECK((Size == 0 && Head == &SentinelSegment && Head == Tail) || + (Size != 0 && Head != &SentinelSegment && Tail != &SentinelSegment)); + if (UNLIKELY(Head == &SentinelSegment)) { + auto R = InitHeadAndTail(); + if (R == nullptr) return nullptr; + } + + DCHECK_NE(Head, &SentinelSegment); + DCHECK_NE(Tail, &SentinelSegment); auto Offset = Size % ElementsPerSegment; if (UNLIKELY(Size != 0 && Offset == 0)) if (AppendNewSegment() == nullptr) return nullptr; - auto Base = static_cast<Segment *>(Tail)->Data; + DCHECK_NE(Tail, &SentinelSegment); + auto Base = &Tail->Data; auto AlignedOffset = Base + (Offset * AlignedElementStorageSize); - auto Position = reinterpret_cast<T *>(AlignedOffset); - *Position = E; + DCHECK_LE(AlignedOffset + sizeof(T), + reinterpret_cast<unsigned char *>(Base) + SegmentSize); + + // In-place construct at Position. + new (AlignedOffset) T{std::forward<Args>(args)...}; ++Size; - return Position; + return reinterpret_cast<T *>(AlignedOffset); } - template <class... Args> T *AppendEmplace(Args &&... args) { - if (UNLIKELY(Head == &SentinelSegment)) - if (InitHeadAndTail() == nullptr) + T *Append(const T &E) XRAY_NEVER_INSTRUMENT { + // FIXME: This is a duplication of AppenEmplace with the copy semantics + // explicitly used, as a work-around to GCC 4.8 not invoking the copy + // constructor with the placement new with braced-init syntax. + DCHECK((Size == 0 && Head == &SentinelSegment && Head == Tail) || + (Size != 0 && Head != &SentinelSegment && Tail != &SentinelSegment)); + if (UNLIKELY(Head == &SentinelSegment)) { + auto R = InitHeadAndTail(); + if (R == nullptr) return nullptr; + } + + DCHECK_NE(Head, &SentinelSegment); + DCHECK_NE(Tail, &SentinelSegment); auto Offset = Size % ElementsPerSegment; - auto *LatestSegment = Tail; - if (UNLIKELY(Size != 0 && Offset == 0)) { - LatestSegment = AppendNewSegment(); - if (LatestSegment == nullptr) + if (UNLIKELY(Size != 0 && Offset == 0)) + if (AppendNewSegment() == nullptr) return nullptr; - } DCHECK_NE(Tail, &SentinelSegment); - auto Base = static_cast<Segment *>(LatestSegment)->Data; + auto Base = &Tail->Data; auto AlignedOffset = Base + (Offset * AlignedElementStorageSize); - auto Position = reinterpret_cast<T *>(AlignedOffset); + DCHECK_LE(AlignedOffset + sizeof(T), + reinterpret_cast<unsigned char *>(Tail) + SegmentSize); // In-place construct at Position. - new (Position) T{std::forward<Args>(args)...}; + new (AlignedOffset) T(E); ++Size; - return reinterpret_cast<T *>(Position); + return reinterpret_cast<T *>(AlignedOffset); } - T &operator[](size_t Offset) const { + T &operator[](uint64_t Offset) const XRAY_NEVER_INSTRUMENT { DCHECK_LE(Offset, Size); // We need to traverse the array enough times to find the element at Offset. auto S = Head; @@ -290,19 +421,19 @@ public: Offset -= ElementsPerSegment; DCHECK_NE(S, &SentinelSegment); } - auto Base = static_cast<Segment *>(S)->Data; + auto Base = &S->Data; auto AlignedOffset = Base + (Offset * AlignedElementStorageSize); auto Position = reinterpret_cast<T *>(AlignedOffset); return *reinterpret_cast<T *>(Position); } - T &front() const { + T &front() const XRAY_NEVER_INSTRUMENT { DCHECK_NE(Head, &SentinelSegment); DCHECK_NE(Size, 0u); return *begin(); } - T &back() const { + T &back() const XRAY_NEVER_INSTRUMENT { DCHECK_NE(Tail, &SentinelSegment); DCHECK_NE(Size, 0u); auto It = end(); @@ -310,7 +441,8 @@ public: return *It; } - template <class Predicate> T *find_element(Predicate P) const { + template <class Predicate> + T *find_element(Predicate P) const XRAY_NEVER_INSTRUMENT { if (empty()) return nullptr; @@ -324,51 +456,195 @@ public: /// Remove N Elements from the end. This leaves the blocks behind, and not /// require allocation of new blocks for new elements added after trimming. - void trim(size_t Elements) { - DCHECK_LE(Elements, Size); - DCHECK_GT(Size, 0); + void trim(uint64_t Elements) XRAY_NEVER_INSTRUMENT { auto OldSize = Size; + Elements = Elements > Size ? Size : Elements; Size -= Elements; - DCHECK_NE(Head, &SentinelSegment); - DCHECK_NE(Tail, &SentinelSegment); - - for (auto SegmentsToTrim = (nearest_boundary(OldSize, ElementsPerSegment) - - nearest_boundary(Size, ElementsPerSegment)) / - ElementsPerSegment; - SegmentsToTrim > 0; --SegmentsToTrim) { + // We compute the number of segments we're going to return from the tail by + // counting how many elements have been trimmed. Given the following: + // + // - Each segment has N valid positions, where N > 0 + // - The previous size > current size + // + // To compute the number of segments to return, we need to perform the + // following calculations for the number of segments required given 'x' + // elements: + // + // f(x) = { + // x == 0 : 0 + // , 0 < x <= N : 1 + // , N < x <= max : x / N + (x % N ? 1 : 0) + // } + // + // We can simplify this down to: + // + // f(x) = { + // x == 0 : 0, + // , 0 < x <= max : x / N + (x < N || x % N ? 1 : 0) + // } + // + // And further down to: + // + // f(x) = x ? x / N + (x < N || x % N ? 1 : 0) : 0 + // + // We can then perform the following calculation `s` which counts the number + // of segments we need to remove from the end of the data structure: + // + // s(p, c) = f(p) - f(c) + // + // If we treat p = previous size, and c = current size, and given the + // properties above, the possible range for s(...) is [0..max(typeof(p))/N] + // given that typeof(p) == typeof(c). + auto F = [](uint64_t X) { + return X ? (X / ElementsPerSegment) + + (X < ElementsPerSegment || X % ElementsPerSegment ? 1 : 0) + : 0; + }; + auto PS = F(OldSize); + auto CS = F(Size); + DCHECK_GE(PS, CS); + auto SegmentsToTrim = PS - CS; + for (auto I = 0uL; I < SegmentsToTrim; ++I) { + // Here we place the current tail segment to the freelist. To do this + // appropriately, we need to perform a splice operation on two + // bidirectional linked-lists. In particular, we have the current state of + // the doubly-linked list of segments: + // + // @S@ <- s0 <-> s1 <-> ... <-> sT -> @S@ + // DCHECK_NE(Head, &SentinelSegment); DCHECK_NE(Tail, &SentinelSegment); - // Put the tail into the Freelist. - auto *FreeSegment = Tail; - Tail = Tail->Prev; - if (Tail == &SentinelSegment) - Head = Tail; - else - Tail->Next = &SentinelSegment; - DCHECK_EQ(Tail->Next, &SentinelSegment); - FreeSegment->Next = Freelist; - FreeSegment->Prev = &SentinelSegment; - if (Freelist != &SentinelSegment) - Freelist->Prev = FreeSegment; - Freelist = FreeSegment; + + if (Freelist == &SentinelSegment) { + // Our two lists at this point are in this configuration: + // + // Freelist: (potentially) @S@ + // Mainlist: @S@<-s0<->s1<->...<->sPT<->sT->@S@ + // ^ Head ^ Tail + // + // The end state for us will be this configuration: + // + // Freelist: @S@<-sT->@S@ + // Mainlist: @S@<-s0<->s1<->...<->sPT->@S@ + // ^ Head ^ Tail + // + // The first step for us is to hold a reference to the tail of Mainlist, + // which in our notation is represented by sT. We call this our "free + // segment" which is the segment we are placing on the Freelist. + // + // sF = sT + // + // Then, we also hold a reference to the "pre-tail" element, which we + // call sPT: + // + // sPT = pred(sT) + // + // We want to splice sT into the beginning of the Freelist, which in + // an empty Freelist means placing a segment whose predecessor and + // successor is the sentinel segment. + // + // The splice operation then can be performed in the following + // algorithm: + // + // succ(sPT) = S + // pred(sT) = S + // succ(sT) = Freelist + // Freelist = sT + // Tail = sPT + // + auto SPT = Tail->Prev; + SPT->Next = &SentinelSegment; + Tail->Prev = &SentinelSegment; + Tail->Next = Freelist; + Freelist = Tail; + Tail = SPT; + + // Our post-conditions here are: + DCHECK_EQ(Tail->Next, &SentinelSegment); + DCHECK_EQ(Freelist->Prev, &SentinelSegment); + } else { + // In the other case, where the Freelist is not empty, we perform the + // following transformation instead: + // + // This transforms the current state: + // + // Freelist: @S@<-f0->@S@ + // ^ Freelist + // Mainlist: @S@<-s0<->s1<->...<->sPT<->sT->@S@ + // ^ Head ^ Tail + // + // Into the following: + // + // Freelist: @S@<-sT<->f0->@S@ + // ^ Freelist + // Mainlist: @S@<-s0<->s1<->...<->sPT->@S@ + // ^ Head ^ Tail + // + // The algorithm is: + // + // sFH = Freelist + // sPT = pred(sT) + // pred(SFH) = sT + // succ(sT) = Freelist + // pred(sT) = S + // succ(sPT) = S + // Tail = sPT + // Freelist = sT + // + auto SFH = Freelist; + auto SPT = Tail->Prev; + auto ST = Tail; + SFH->Prev = ST; + ST->Next = Freelist; + ST->Prev = &SentinelSegment; + SPT->Next = &SentinelSegment; + Tail = SPT; + Freelist = ST; + + // Our post-conditions here are: + DCHECK_EQ(Tail->Next, &SentinelSegment); + DCHECK_EQ(Freelist->Prev, &SentinelSegment); + DCHECK_EQ(Freelist->Next->Prev, Freelist); + } } + + // Now in case we've spliced all the segments in the end, we ensure that the + // main list is "empty", or both the head and tail pointing to the sentinel + // segment. + if (Tail == &SentinelSegment) + Head = Tail; + + DCHECK( + (Size == 0 && Head == &SentinelSegment && Tail == &SentinelSegment) || + (Size != 0 && Head != &SentinelSegment && Tail != &SentinelSegment)); + DCHECK( + (Freelist != &SentinelSegment && Freelist->Prev == &SentinelSegment) || + (Freelist == &SentinelSegment && Tail->Next == &SentinelSegment)); } // Provide iterators. - Iterator<T> begin() const { return Iterator<T>(Head, 0, Size); } - Iterator<T> end() const { return Iterator<T>(Tail, Size, Size); } - Iterator<const T> cbegin() const { return Iterator<const T>(Head, 0, Size); } - Iterator<const T> cend() const { return Iterator<const T>(Tail, Size, Size); } + Iterator<T> begin() const XRAY_NEVER_INSTRUMENT { + return Iterator<T>(Head, 0, Size); + } + Iterator<T> end() const XRAY_NEVER_INSTRUMENT { + return Iterator<T>(Tail, Size, Size); + } + Iterator<const T> cbegin() const XRAY_NEVER_INSTRUMENT { + return Iterator<const T>(Head, 0, Size); + } + Iterator<const T> cend() const XRAY_NEVER_INSTRUMENT { + return Iterator<const T>(Tail, Size, Size); + } }; // We need to have this storage definition out-of-line so that the compiler can // ensure that storage for the SentinelSegment is defined and has a single // address. template <class T> -typename Array<T>::SegmentBase Array<T>::SentinelSegment{ - &Array<T>::SentinelSegment, &Array<T>::SentinelSegment}; +typename Array<T>::Segment Array<T>::SentinelSegment{ + &Array<T>::SentinelSegment, &Array<T>::SentinelSegment, {'\0'}}; } // namespace __xray diff --git a/contrib/compiler-rt/lib/xray/xray_trampoline_x86_64.S b/contrib/compiler-rt/lib/xray/xray_trampoline_x86_64.S index 99ad3966ee3a..52985ffd19ab 100644 --- a/contrib/compiler-rt/lib/xray/xray_trampoline_x86_64.S +++ b/contrib/compiler-rt/lib/xray/xray_trampoline_x86_64.S @@ -19,6 +19,7 @@ .macro SAVE_REGISTERS + pushfq subq $240, %rsp CFI_DEF_CFA_OFFSET(248) movq %rbp, 232(%rsp) @@ -69,6 +70,7 @@ movq 8(%rsp), %r14 movq 0(%rsp), %r15 addq $240, %rsp + popfq CFI_DEF_CFA_OFFSET(8) .endm @@ -89,10 +91,10 @@ .text #if !defined(__APPLE__) .section .text + .file "xray_trampoline_x86.S" #else .section __TEXT,__text #endif - .file "xray_trampoline_x86.S" //===----------------------------------------------------------------------===// diff --git a/contrib/compiler-rt/lib/xray/xray_tsc.h b/contrib/compiler-rt/lib/xray/xray_tsc.h index 4507564e7cd2..180d6df188c1 100644 --- a/contrib/compiler-rt/lib/xray/xray_tsc.h +++ b/contrib/compiler-rt/lib/xray/xray_tsc.h @@ -13,10 +13,32 @@ #ifndef XRAY_EMULATE_TSC_H #define XRAY_EMULATE_TSC_H +#include "sanitizer_common/sanitizer_common.h" + namespace __xray { static constexpr uint64_t NanosecondsPerSecond = 1000ULL * 1000 * 1000; } +#if SANITIZER_FUCHSIA +#include <zircon/syscalls.h> + +namespace __xray { + +inline bool probeRequiredCPUFeatures() XRAY_NEVER_INSTRUMENT { return true; } + +ALWAYS_INLINE uint64_t readTSC(uint8_t &CPU) XRAY_NEVER_INSTRUMENT { + CPU = 0; + return _zx_ticks_get(); +} + +inline uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT { + return _zx_ticks_per_second(); +} + +} // namespace __xray + +#else // SANITIZER_FUCHSIA + #if defined(__x86_64__) #include "xray_x86_64.inc" #elif defined(__powerpc64__) @@ -64,5 +86,6 @@ inline uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT { #else #error Target architecture is not supported. #endif // CPU architecture +#endif // SANITIZER_FUCHSIA #endif // XRAY_EMULATE_TSC_H diff --git a/contrib/compiler-rt/lib/xray/xray_utils.cc b/contrib/compiler-rt/lib/xray/xray_utils.cc index 68f4e8c1094c..59ba6c3082b2 100644 --- a/contrib/compiler-rt/lib/xray/xray_utils.cc +++ b/contrib/compiler-rt/lib/xray/xray_utils.cc @@ -12,7 +12,9 @@ //===----------------------------------------------------------------------===// #include "xray_utils.h" +#include "sanitizer_common/sanitizer_allocator_internal.h" #include "sanitizer_common/sanitizer_common.h" +#include "xray_allocator.h" #include "xray_defs.h" #include "xray_flags.h" #include <cstdio> @@ -25,13 +27,113 @@ #include <unistd.h> #include <utility> +#if SANITIZER_FUCHSIA +#include "sanitizer_common/sanitizer_symbolizer_fuchsia.h" + +#include <inttypes.h> +#include <zircon/process.h> +#include <zircon/sanitizer.h> +#include <zircon/status.h> +#include <zircon/syscalls.h> +#endif + namespace __xray { -void printToStdErr(const char *Buffer) XRAY_NEVER_INSTRUMENT { - fprintf(stderr, "%s", Buffer); +#if SANITIZER_FUCHSIA +constexpr const char* ProfileSinkName = "llvm-xray"; + +LogWriter::~LogWriter() { + _zx_handle_close(Vmo); +} + +void LogWriter::WriteAll(const char *Begin, const char *End) XRAY_NEVER_INSTRUMENT { + if (Begin == End) + return; + auto TotalBytes = std::distance(Begin, End); + + const size_t PageSize = flags()->xray_page_size_override > 0 + ? flags()->xray_page_size_override + : GetPageSizeCached(); + if (RoundUpTo(Offset, PageSize) != RoundUpTo(Offset + TotalBytes, PageSize)) { + // Resize the VMO to ensure there's sufficient space for the data. + zx_status_t Status = _zx_vmo_set_size(Vmo, Offset + TotalBytes); + if (Status != ZX_OK) { + Report("Failed to resize VMO: %s\n", _zx_status_get_string(Status)); + return; + } + } + + // Write the data into VMO. + zx_status_t Status = _zx_vmo_write(Vmo, Begin, Offset, TotalBytes); + if (Status != ZX_OK) { + Report("Failed to write: %s\n", _zx_status_get_string(Status)); + return; + } + Offset += TotalBytes; +} + +void LogWriter::Flush() XRAY_NEVER_INSTRUMENT { + // Nothing to do here since WriteAll writes directly into the VMO. +} + +LogWriter *LogWriter::Open() XRAY_NEVER_INSTRUMENT { + // Create VMO to hold the profile data. + zx_handle_t Vmo; + zx_status_t Status = _zx_vmo_create(0, 0, &Vmo); + if (Status != ZX_OK) { + Report("XRay: cannot create VMO: %s\n", _zx_status_get_string(Status)); + return nullptr; + } + + // Get the KOID of the current process to use in the VMO name. + zx_info_handle_basic_t Info; + Status = _zx_object_get_info(_zx_process_self(), ZX_INFO_HANDLE_BASIC, &Info, + sizeof(Info), NULL, NULL); + if (Status != ZX_OK) { + Report("XRay: cannot get basic info about current process handle: %s\n", + _zx_status_get_string(Status)); + return nullptr; + } + + // Give the VMO a name including our process KOID so it's easy to spot. + char VmoName[ZX_MAX_NAME_LEN]; + internal_snprintf(VmoName, sizeof(VmoName), "%s.%zu", ProfileSinkName, + Info.koid); + _zx_object_set_property(Vmo, ZX_PROP_NAME, VmoName, strlen(VmoName)); + + // Duplicate the handle since __sanitizer_publish_data consumes it and + // LogWriter needs to hold onto it. + zx_handle_t Handle; + Status =_zx_handle_duplicate(Vmo, ZX_RIGHT_SAME_RIGHTS, &Handle); + if (Status != ZX_OK) { + Report("XRay: cannot duplicate VMO handle: %s\n", + _zx_status_get_string(Status)); + return nullptr; + } + + // Publish the VMO that receives the logging. Note the VMO's contents can + // grow and change after publication. The contents won't be read out until + // after the process exits. + __sanitizer_publish_data(ProfileSinkName, Handle); + + // Use the dumpfile symbolizer markup element to write the name of the VMO. + Report("XRay: " FORMAT_DUMPFILE "\n", ProfileSinkName, VmoName); + + LogWriter *LW = reinterpret_cast<LogWriter *>(InternalAlloc(sizeof(LogWriter))); + new (LW) LogWriter(Vmo); + return LW; +} + +void LogWriter::Close(LogWriter *LW) { + LW->~LogWriter(); + InternalFree(LW); +} +#else // SANITIZER_FUCHSIA +LogWriter::~LogWriter() { + internal_close(Fd); } -void retryingWriteAll(int Fd, const char *Begin, const char *End) XRAY_NEVER_INSTRUMENT { +void LogWriter::WriteAll(const char *Begin, const char *End) XRAY_NEVER_INSTRUMENT { if (Begin == End) return; auto TotalBytes = std::distance(Begin, End); @@ -49,50 +151,11 @@ void retryingWriteAll(int Fd, const char *Begin, const char *End) XRAY_NEVER_INS } } -std::pair<ssize_t, bool> retryingReadSome(int Fd, char *Begin, - char *End) XRAY_NEVER_INSTRUMENT { - auto BytesToRead = std::distance(Begin, End); - ssize_t BytesRead; - ssize_t TotalBytesRead = 0; - while (BytesToRead && (BytesRead = read(Fd, Begin, BytesToRead))) { - if (BytesRead == -1) { - if (errno == EINTR) - continue; - Report("Read error; errno = %d\n", errno); - return std::make_pair(TotalBytesRead, false); - } - - TotalBytesRead += BytesRead; - BytesToRead -= BytesRead; - Begin += BytesRead; - } - return std::make_pair(TotalBytesRead, true); -} - -bool readValueFromFile(const char *Filename, - long long *Value) XRAY_NEVER_INSTRUMENT { - int Fd = open(Filename, O_RDONLY | O_CLOEXEC); - if (Fd == -1) - return false; - static constexpr size_t BufSize = 256; - char Line[BufSize] = {}; - ssize_t BytesRead; - bool Success; - std::tie(BytesRead, Success) = retryingReadSome(Fd, Line, Line + BufSize); - if (!Success) - return false; - close(Fd); - const char *End = nullptr; - long long Tmp = internal_simple_strtoll(Line, &End, 10); - bool Result = false; - if (Line[0] != '\0' && (*End == '\n' || *End == '\0')) { - *Value = Tmp; - Result = true; - } - return Result; +void LogWriter::Flush() XRAY_NEVER_INSTRUMENT { + fsync(Fd); } -int getLogFD() XRAY_NEVER_INSTRUMENT { +LogWriter *LogWriter::Open() XRAY_NEVER_INSTRUMENT { // Open a temporary file once for the log. char TmpFilename[256] = {}; char TmpWildcardPattern[] = "XXXXXX"; @@ -103,24 +166,31 @@ int getLogFD() XRAY_NEVER_INSTRUMENT { if (LastSlash != nullptr) Progname = LastSlash + 1; - const int HalfLength = sizeof(TmpFilename) / 2 - sizeof(TmpWildcardPattern); int NeededLength = internal_snprintf( - TmpFilename, sizeof(TmpFilename), "%.*s%.*s.%s", HalfLength, - flags()->xray_logfile_base, HalfLength, Progname, TmpWildcardPattern); + TmpFilename, sizeof(TmpFilename), "%s%s.%s", + flags()->xray_logfile_base, Progname, TmpWildcardPattern); if (NeededLength > int(sizeof(TmpFilename))) { Report("XRay log file name too long (%d): %s\n", NeededLength, TmpFilename); - return -1; + return nullptr; } int Fd = mkstemp(TmpFilename); if (Fd == -1) { Report("XRay: Failed opening temporary file '%s'; not logging events.\n", TmpFilename); - return -1; + return nullptr; } if (Verbosity()) Report("XRay: Log file in '%s'\n", TmpFilename); - return Fd; + LogWriter *LW = allocate<LogWriter>(); + new (LW) LogWriter(Fd); + return LW; +} + +void LogWriter::Close(LogWriter *LW) { + LW->~LogWriter(); + deallocate(LW); } +#endif // SANITIZER_FUCHSIA } // namespace __xray diff --git a/contrib/compiler-rt/lib/xray/xray_utils.h b/contrib/compiler-rt/lib/xray/xray_utils.h index eafa16e1a9d5..60438973fbd0 100644 --- a/contrib/compiler-rt/lib/xray/xray_utils.h +++ b/contrib/compiler-rt/lib/xray/xray_utils.h @@ -20,23 +20,40 @@ #include <sys/types.h> #include <utility> -namespace __xray { - -// Default implementation of the reporting interface for sanitizer errors. -void printToStdErr(const char *Buffer); - -// EINTR-safe write routine, provided a file descriptor and a character range. -void retryingWriteAll(int Fd, const char *Begin, const char *End); +#include "sanitizer_common/sanitizer_common.h" +#if SANITIZER_FUCHSIA +#include <zircon/types.h> +#endif -// Reads a long long value from a provided file. -bool readValueFromFile(const char *Filename, long long *Value); - -// EINTR-safe read routine, providing a file descriptor and a character range. -std::pair<ssize_t, bool> retryingReadSome(int Fd, char *Begin, char *End); +namespace __xray { -// EINTR-safe open routine, uses flag-provided values for initialising a log -// file. -int getLogFD(); +class LogWriter { +public: +#if SANITIZER_FUCHSIA + LogWriter(zx_handle_t Vmo) : Vmo(Vmo) {} +#else + explicit LogWriter(int Fd) : Fd(Fd) {} +#endif + ~LogWriter(); + + // Write a character range into a log. + void WriteAll(const char *Begin, const char *End); + + void Flush(); + + // Returns a new log instance initialized using the flag-provided values. + static LogWriter *Open(); + // Closes and deallocates the log instance. + static void Close(LogWriter *LogWriter); + +private: +#if SANITIZER_FUCHSIA + zx_handle_t Vmo = ZX_HANDLE_INVALID; + uint64_t Offset = 0; +#else + int Fd = -1; +#endif +}; constexpr size_t gcd(size_t a, size_t b) { return (b == 0) ? a : gcd(b, a % b); diff --git a/contrib/compiler-rt/lib/xray/xray_x86_64.cc b/contrib/compiler-rt/lib/xray/xray_x86_64.cc index 51dc4ce43b1c..e63ee1b3bd02 100644 --- a/contrib/compiler-rt/lib/xray/xray_x86_64.cc +++ b/contrib/compiler-rt/lib/xray/xray_x86_64.cc @@ -1,15 +1,20 @@ #include "cpuid.h" #include "sanitizer_common/sanitizer_common.h" +#if !SANITIZER_FUCHSIA +#include "sanitizer_common/sanitizer_posix.h" +#endif #include "xray_defs.h" #include "xray_interface_internal.h" -#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD +#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD || SANITIZER_MAC #include <sys/types.h> #if SANITIZER_OPENBSD #include <sys/time.h> #include <machine/cpu.h> #endif #include <sys/sysctl.h> +#elif SANITIZER_FUCHSIA +#include <zircon/syscalls.h> #endif #include <atomic> @@ -81,17 +86,20 @@ uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT { } return TSCFrequency == -1 ? 0 : static_cast<uint64_t>(TSCFrequency); } -#elif SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD +#elif SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD || SANITIZER_MAC uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT { long long TSCFrequency = -1; size_t tscfreqsz = sizeof(TSCFrequency); #if SANITIZER_OPENBSD int Mib[2] = { CTL_MACHDEP, CPU_TSCFREQ }; - if (sysctl(Mib, 2, &TSCFrequency, &tscfreqsz, NULL, 0) != -1) { + if (internal_sysctl(Mib, 2, &TSCFrequency, &tscfreqsz, NULL, 0) != -1) { +#elif SANITIZER_MAC + if (internal_sysctlbyname("machdep.tsc.frequency", &TSCFrequency, + &tscfreqsz, NULL, 0) != -1) { #else - if (sysctlbyname("machdep.tsc_freq", &TSCFrequency, &tscfreqsz, - NULL, 0) != -1) { + if (internal_sysctlbyname("machdep.tsc_freq", &TSCFrequency, &tscfreqsz, + NULL, 0) != -1) { #endif return static_cast<uint64_t>(TSCFrequency); } else { @@ -100,7 +108,7 @@ uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT { return 0; } -#else +#elif !SANITIZER_FUCHSIA uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT { /* Not supported */ return 0; @@ -317,6 +325,7 @@ bool patchTypedEvent(const bool Enable, const uint32_t FuncId, return false; } +#if !SANITIZER_FUCHSIA // We determine whether the CPU we're running on has the correct features we // need. In x86_64 this will be rdtscp support. bool probeRequiredCPUFeatures() XRAY_NEVER_INSTRUMENT { @@ -339,5 +348,6 @@ bool probeRequiredCPUFeatures() XRAY_NEVER_INSTRUMENT { } return true; } +#endif } // namespace __xray |