diff options
author | Dimitry Andric <dim@FreeBSD.org> | 2021-07-29 20:15:26 +0000 |
---|---|---|
committer | Dimitry Andric <dim@FreeBSD.org> | 2021-07-29 20:15:26 +0000 |
commit | 344a3780b2e33f6ca763666c380202b18aab72a3 (patch) | |
tree | f0b203ee6eb71d7fdd792373e3c81eb18d6934dd /compiler-rt/lib/orc | |
parent | b60736ec1405bb0a8dd40989f67ef4c93da068ab (diff) | |
download | src-344a3780b2e33f6ca763666c380202b18aab72a3.tar.gz src-344a3780b2e33f6ca763666c380202b18aab72a3.zip |
Vendor import of llvm-project main 88e66fa60ae5, the last commit beforevendor/llvm-project/llvmorg-13-init-16847-g88e66fa60ae5vendor/llvm-project/llvmorg-12.0.1-rc2-0-ge7dac564cd0evendor/llvm-project/llvmorg-12.0.1-0-gfed41342a82f
the upstream release/13.x branch was created.
Diffstat (limited to 'compiler-rt/lib/orc')
-rw-r--r-- | compiler-rt/lib/orc/adt.h | 113 | ||||
-rw-r--r-- | compiler-rt/lib/orc/c_api.h | 208 | ||||
-rw-r--r-- | compiler-rt/lib/orc/common.h | 48 | ||||
-rw-r--r-- | compiler-rt/lib/orc/compiler.h | 65 | ||||
-rw-r--r-- | compiler-rt/lib/orc/endianness.h | 143 | ||||
-rw-r--r-- | compiler-rt/lib/orc/error.h | 428 | ||||
-rw-r--r-- | compiler-rt/lib/orc/executor_address.h | 208 | ||||
-rw-r--r-- | compiler-rt/lib/orc/extensible_rtti.cpp | 24 | ||||
-rw-r--r-- | compiler-rt/lib/orc/extensible_rtti.h | 145 | ||||
-rw-r--r-- | compiler-rt/lib/orc/log_error_to_stderr.cpp | 19 | ||||
-rw-r--r-- | compiler-rt/lib/orc/macho_platform.cpp | 731 | ||||
-rw-r--r-- | compiler-rt/lib/orc/macho_platform.h | 135 | ||||
-rw-r--r-- | compiler-rt/lib/orc/macho_tlv.x86-64.S | 68 | ||||
-rw-r--r-- | compiler-rt/lib/orc/run_program_wrapper.cpp | 51 | ||||
-rw-r--r-- | compiler-rt/lib/orc/simple_packed_serialization.h | 579 | ||||
-rw-r--r-- | compiler-rt/lib/orc/stl_extras.h | 46 | ||||
-rw-r--r-- | compiler-rt/lib/orc/wrapper_function_utils.h | 367 |
17 files changed, 3378 insertions, 0 deletions
diff --git a/compiler-rt/lib/orc/adt.h b/compiler-rt/lib/orc/adt.h new file mode 100644 index 000000000000..33b731082f88 --- /dev/null +++ b/compiler-rt/lib/orc/adt.h @@ -0,0 +1,113 @@ +//===----------------------- adt.h - Handy ADTs -----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_ADT_H +#define ORC_RT_ADT_H + +#include <cstring> +#include <limits> +#include <string> + +namespace __orc_rt { + +constexpr std::size_t dynamic_extent = std::numeric_limits<std::size_t>::max(); + +/// A substitute for std::span (and llvm::ArrayRef). +/// FIXME: Remove in favor of std::span once we can use c++20. +template <typename T, std::size_t Extent = dynamic_extent> class span { +public: + typedef T element_type; + typedef std::remove_cv<T> value_type; + typedef std::size_t size_type; + typedef std::ptrdiff_t difference_type; + typedef T *pointer; + typedef const T *const_pointer; + typedef T &reference; + typedef const T &const_reference; + + typedef pointer iterator; + + static constexpr std::size_t extent = Extent; + + constexpr span() noexcept = default; + constexpr span(T *first, size_type count) noexcept + : Data(first), Size(count) {} + + template <std::size_t N> + constexpr span(T (&arr)[N]) noexcept : Data(&arr[0]), Size(N) {} + + constexpr iterator begin() const noexcept { return Data; } + constexpr iterator end() const noexcept { return Data + Size; } + constexpr pointer data() const noexcept { return Data; } + constexpr reference operator[](size_type idx) const { return Data[idx]; } + constexpr size_type size() const noexcept { return Size; } + constexpr bool empty() const noexcept { return Size == 0; } + +private: + T *Data = nullptr; + size_type Size = 0; +}; + +/// A substitue for std::string_view (and llvm::StringRef). +/// FIXME: Remove in favor of std::string_view once we have c++17. +class string_view { +public: + typedef char value_type; + typedef char *pointer; + typedef const char *const_pointer; + typedef char &reference; + typedef const char &const_reference; + typedef std::size_t size_type; + typedef std::ptrdiff_t difference_type; + + typedef const_pointer const_iterator; + typedef const_iterator iterator; + + constexpr string_view() noexcept = default; + constexpr string_view(const char *S, size_type Count) + : Data(S), Size(Count) {} + string_view(const char *S) : Data(S), Size(strlen(S)) {} + + constexpr const_iterator begin() const noexcept { return Data; } + constexpr const_iterator end() const noexcept { return Data + Size; } + constexpr const_pointer data() const noexcept { return Data; } + constexpr const_reference operator[](size_type idx) { return Data[idx]; } + constexpr size_type size() const noexcept { return Size; } + constexpr bool empty() const noexcept { return Size == 0; } + + friend bool operator==(const string_view &LHS, const string_view &RHS) { + if (LHS.Size != RHS.Size) + return false; + if (LHS.Data == RHS.Data) + return true; + for (size_t I = 0; I != LHS.Size; ++I) + if (LHS.Data[I] != RHS.Data[I]) + return false; + return true; + } + + friend bool operator!=(const string_view &LHS, const string_view &RHS) { + return !(LHS == RHS); + } + +private: + const char *Data = nullptr; + size_type Size = 0; +}; + +inline std::string to_string(string_view SV) { + return std::string(SV.data(), SV.size()); +} + +} // end namespace __orc_rt + +#endif // ORC_RT_COMMON_H diff --git a/compiler-rt/lib/orc/c_api.h b/compiler-rt/lib/orc/c_api.h new file mode 100644 index 000000000000..6677da06ede5 --- /dev/null +++ b/compiler-rt/lib/orc/c_api.h @@ -0,0 +1,208 @@ +/*===- c_api.h - C API for the ORC runtime ------------------------*- C -*-===*\ +|* *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| +|* *| +|*===----------------------------------------------------------------------===*| +|* *| +|* This file defines the C API for the ORC runtime *| +|* *| +|* NOTE: The OrtRTWrapperFunctionResult type must be kept in sync with the *| +|* definition in llvm/include/llvm-c/OrcShared.h. *| +|* *| +\*===----------------------------------------------------------------------===*/ + +#ifndef ORC_RT_C_API_H +#define ORC_RT_C_API_H + +#include <assert.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +/* Helper to suppress strict prototype warnings. */ +#ifdef __clang__ +#define ORC_RT_C_STRICT_PROTOTYPES_BEGIN \ + _Pragma("clang diagnostic push") \ + _Pragma("clang diagnostic error \"-Wstrict-prototypes\"") +#define ORC_RT_C_STRICT_PROTOTYPES_END _Pragma("clang diagnostic pop") +#else +#define ORC_RT_C_STRICT_PROTOTYPES_BEGIN +#define ORC_RT_C_STRICT_PROTOTYPES_END +#endif + +/* Helper to wrap C code for C++ */ +#ifdef __cplusplus +#define ORC_RT_C_EXTERN_C_BEGIN \ + extern "C" { \ + ORC_RT_C_STRICT_PROTOTYPES_BEGIN +#define ORC_RT_C_EXTERN_C_END \ + ORC_RT_C_STRICT_PROTOTYPES_END \ + } +#else +#define ORC_RT_C_EXTERN_C_BEGIN ORC_RT_C_STRICT_PROTOTYPES_BEGIN +#define ORC_RT_C_EXTERN_C_END ORC_RT_C_STRICT_PROTOTYPES_END +#endif + +ORC_RT_C_EXTERN_C_BEGIN + +typedef union { + char *ValuePtr; + char Value[sizeof(ValuePtr)]; +} __orc_rt_CWrapperFunctionResultDataUnion; + +/** + * __orc_rt_CWrapperFunctionResult is a kind of C-SmallVector with an + * out-of-band error state. + * + * If Size == 0 and Data.ValuePtr is non-zero then the value is in the + * 'out-of-band error' state, and Data.ValuePtr points at a malloc-allocated, + * null-terminated string error message. + * + * If Size <= sizeof(__orc_rt_CWrapperFunctionResultData) then the value is in + * the 'small' state and the content is held in the first Size bytes of + * Data.Value. + * + * If Size > sizeof(OrtRTCWrapperFunctionResultData) then the value is in the + * 'large' state and the content is held in the first Size bytes of the + * memory pointed to by Data.ValuePtr. This memory must have been allocated by + * malloc, and will be freed with free when this value is destroyed. + */ +typedef struct { + __orc_rt_CWrapperFunctionResultDataUnion Data; + size_t Size; +} __orc_rt_CWrapperFunctionResult; + +typedef struct __orc_rt_CSharedOpaqueJITProcessControl + *__orc_rt_SharedJITProcessControlRef; + +/** + * Zero-initialize an __orc_rt_CWrapperFunctionResult. + */ +static inline void +__orc_rt_CWrapperFunctionResultInit(__orc_rt_CWrapperFunctionResult *R) { + R->Size = 0; + R->Data.ValuePtr = 0; +} + +/** + * Create an __orc_rt_CWrapperFunctionResult with an uninitialized buffer of + * size Size. The buffer is returned via the DataPtr argument. + */ +static inline char * +__orc_rt_CWrapperFunctionResultAllocate(__orc_rt_CWrapperFunctionResult *R, + size_t Size) { + R->Size = Size; + if (Size <= sizeof(R->Data.Value)) + return R->Data.Value; + + R->Data.ValuePtr = (char *)malloc(Size); + return R->Data.ValuePtr; +} + +/** + * Create an __orc_rt_WrapperFunctionResult from the given data range. + */ +static inline __orc_rt_CWrapperFunctionResult +__orc_rt_CreateCWrapperFunctionResultFromRange(const char *Data, size_t Size) { + __orc_rt_CWrapperFunctionResult R; + R.Size = Size; + if (R.Size > sizeof(R.Data.Value)) { + char *Tmp = (char *)malloc(Size); + memcpy(Tmp, Data, Size); + R.Data.ValuePtr = Tmp; + } else + memcpy(R.Data.Value, Data, Size); + return R; +} + +/** + * Create an __orc_rt_CWrapperFunctionResult by copying the given string, + * including the null-terminator. + * + * This function copies the input string. The client is responsible for freeing + * the ErrMsg arg. + */ +static inline __orc_rt_CWrapperFunctionResult +__orc_rt_CreateCWrapperFunctionResultFromString(const char *Source) { + return __orc_rt_CreateCWrapperFunctionResultFromRange(Source, + strlen(Source) + 1); +} + +/** + * Create an __orc_rt_CWrapperFunctionResult representing an out-of-band + * error. + * + * This function takes ownership of the string argument which must have been + * allocated with malloc. + */ +static inline __orc_rt_CWrapperFunctionResult +__orc_rt_CreateCWrapperFunctionResultFromOutOfBandError(const char *ErrMsg) { + __orc_rt_CWrapperFunctionResult R; + R.Size = 0; + char *Tmp = (char *)malloc(strlen(ErrMsg) + 1); + strcpy(Tmp, ErrMsg); + R.Data.ValuePtr = Tmp; + return R; +} + +/** + * This should be called to destroy __orc_rt_CWrapperFunctionResult values + * regardless of their state. + */ +static inline void +__orc_rt_DisposeCWrapperFunctionResult(__orc_rt_CWrapperFunctionResult *R) { + if (R->Size > sizeof(R->Data.Value) || + (R->Size == 0 && R->Data.ValuePtr)) + free(R->Data.ValuePtr); +} + +/** + * Get a pointer to the data contained in the given + * __orc_rt_CWrapperFunctionResult. + */ +static inline const char * +__orc_rt_CWrapperFunctionResultData(const __orc_rt_CWrapperFunctionResult *R) { + assert((R->Size != 0 || R->Data.ValuePtr == nullptr) && + "Cannot get data for out-of-band error value"); + return R->Size > sizeof(R->Data.Value) ? R->Data.ValuePtr : R->Data.Value; +} + +/** + * Safely get the size of the given __orc_rt_CWrapperFunctionResult. + * + * Asserts that we're not trying to access the size of an error value. + */ +static inline size_t +__orc_rt_CWrapperFunctionResultSize(const __orc_rt_CWrapperFunctionResult *R) { + assert((R->Size != 0 || R->Data.ValuePtr == nullptr) && + "Cannot get size for out-of-band error value"); + return R->Size; +} + +/** + * Returns 1 if this value is equivalent to a value just initialized by + * __orc_rt_CWrapperFunctionResultInit, 0 otherwise. + */ +static inline size_t +__orc_rt_CWrapperFunctionResultEmpty(const __orc_rt_CWrapperFunctionResult *R) { + return R->Size == 0 && R->Data.ValuePtr == 0; +} + +/** + * Returns a pointer to the out-of-band error string for this + * __orc_rt_CWrapperFunctionResult, or null if there is no error. + * + * The __orc_rt_CWrapperFunctionResult retains ownership of the error + * string, so it should be copied if the caller wishes to preserve it. + */ +static inline const char *__orc_rt_CWrapperFunctionResultGetOutOfBandError( + const __orc_rt_CWrapperFunctionResult *R) { + return R->Size == 0 ? R->Data.ValuePtr : 0; +} + +ORC_RT_C_EXTERN_C_END + +#endif /* ORC_RT_C_API_H */ diff --git a/compiler-rt/lib/orc/common.h b/compiler-rt/lib/orc/common.h new file mode 100644 index 000000000000..54e613ecb42e --- /dev/null +++ b/compiler-rt/lib/orc/common.h @@ -0,0 +1,48 @@ +//===- common.h - Common utilities for the ORC runtime ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_COMMON_H +#define ORC_RT_COMMON_H + +#include "c_api.h" +#include "compiler.h" +#include <type_traits> + +/// This macro should be used to define tags that will be associated with +/// handlers in the JIT process, and call can be used to define tags f +#define ORC_RT_JIT_DISPATCH_TAG(X) \ +extern "C" char X; \ +char X = 0; + +/// Opaque struct for external symbols. +struct __orc_rt_Opaque {}; + +/// Error reporting function. +extern "C" void __orc_rt_log_error(const char *ErrMsg); + +/// Context object for dispatching calls to the JIT object. +/// +/// This is declared for use by the runtime, but should be implemented in the +/// executor or provided by a definition added to the JIT before the runtime +/// is loaded. +extern "C" __orc_rt_Opaque __orc_rt_jit_dispatch_ctx ORC_RT_WEAK_IMPORT; + +/// For dispatching calls to the JIT object. +/// +/// This is declared for use by the runtime, but should be implemented in the +/// executor or provided by a definition added to the JIT before the runtime +/// is loaded. +extern "C" __orc_rt_CWrapperFunctionResult +__orc_rt_jit_dispatch(__orc_rt_Opaque *DispatchCtx, const void *FnTag, + const char *Data, size_t Size) ORC_RT_WEAK_IMPORT; + +#endif // ORC_RT_COMMON_H diff --git a/compiler-rt/lib/orc/compiler.h b/compiler-rt/lib/orc/compiler.h new file mode 100644 index 000000000000..2e4cd144e335 --- /dev/null +++ b/compiler-rt/lib/orc/compiler.h @@ -0,0 +1,65 @@ +//===--------- compiler.h - Compiler abstraction support --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +// Most functionality in this file was swiped from llvm/Support/Compiler.h. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_COMPILER_H +#define ORC_RT_COMPILER_H + +#define ORC_RT_INTERFACE extern "C" __attribute__((visibility("default"))) +#define ORC_RT_HIDDEN __attribute__((visibility("hidden"))) + +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif + +// Only use __has_cpp_attribute in C++ mode. GCC defines __has_cpp_attribute in +// C mode, but the :: in __has_cpp_attribute(scoped::attribute) is invalid. +#ifndef ORC_RT_HAS_CPP_ATTRIBUTE +#if defined(__cplusplus) && defined(__has_cpp_attribute) +#define ORC_RT_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +#define ORC_RT_HAS_CPP_ATTRIBUTE(x) 0 +#endif +#endif + +// Use the 'nodiscard' attribute in C++17 or newer mode. +#if defined(__cplusplus) && __cplusplus > 201402L && \ + ORC_RT_HAS_CPP_ATTRIBUTE(nodiscard) +#define ORC_RT_NODISCARD [[nodiscard]] +#elif ORC_RT_HAS_CPP_ATTRIBUTE(clang::warn_unused_result) +#define ORC_RT_NODISCARD [[clang::warn_unused_result]] +// Clang in C++14 mode claims that it has the 'nodiscard' attribute, but also +// warns in the pedantic mode that 'nodiscard' is a C++17 extension (PR33518). +// Use the 'nodiscard' attribute in C++14 mode only with GCC. +// TODO: remove this workaround when PR33518 is resolved. +#elif defined(__GNUC__) && ORC_RT_HAS_CPP_ATTRIBUTE(nodiscard) +#define ORC_RT_NODISCARD [[nodiscard]] +#else +#define ORC_RT_NODISCARD +#endif + +#if __has_builtin(__builtin_expect) +#define ORC_RT_LIKELY(EXPR) __builtin_expect((bool)(EXPR), true) +#define ORC_RT_UNLIKELY(EXPR) __builtin_expect((bool)(EXPR), false) +#else +#define ORC_RT_LIKELY(EXPR) (EXPR) +#define ORC_RT_UNLIKELY(EXPR) (EXPR) +#endif + +#ifdef __APPLE__ +#define ORC_RT_WEAK_IMPORT __attribute__((weak_import)) +#else +#define ORC_RT_WEAK_IMPORT __attribute__((weak)) +#endif + +#endif // ORC_RT_COMPILER_H diff --git a/compiler-rt/lib/orc/endianness.h b/compiler-rt/lib/orc/endianness.h new file mode 100644 index 000000000000..4ee5505ce6dd --- /dev/null +++ b/compiler-rt/lib/orc/endianness.h @@ -0,0 +1,143 @@ +//===- endian.h - Endianness support ----------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file declares generic and optimized functions to swap the byte order of +// an integral type. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_ENDIAN_H +#define ORC_RT_ENDIAN_H + +#include <cstddef> +#include <cstdint> +#include <type_traits> +#if defined(_MSC_VER) && !defined(_DEBUG) +#include <stdlib.h> +#endif + +#if defined(__linux__) || defined(__GNU__) || defined(__HAIKU__) || \ + defined(__Fuchsia__) || defined(__EMSCRIPTEN__) +#include <endian.h> +#elif defined(_AIX) +#include <sys/machine.h> +#elif defined(__sun) +/* Solaris provides _BIG_ENDIAN/_LITTLE_ENDIAN selector in sys/types.h */ +#include <sys/types.h> +#define BIG_ENDIAN 4321 +#define LITTLE_ENDIAN 1234 +#if defined(_BIG_ENDIAN) +#define BYTE_ORDER BIG_ENDIAN +#else +#define BYTE_ORDER LITTLE_ENDIAN +#endif +#elif defined(__MVS__) +#define BIG_ENDIAN 4321 +#define LITTLE_ENDIAN 1234 +#define BYTE_ORDER BIG_ENDIAN +#else +#if !defined(BYTE_ORDER) && !defined(_WIN32) +#include <machine/endian.h> +#endif +#endif + +namespace __orc_rt { + +/// ByteSwap_16 - This function returns a byte-swapped representation of +/// the 16-bit argument. +inline uint16_t ByteSwap_16(uint16_t value) { +#if defined(_MSC_VER) && !defined(_DEBUG) + // The DLL version of the runtime lacks these functions (bug!?), but in a + // release build they're replaced with BSWAP instructions anyway. + return _byteswap_ushort(value); +#else + uint16_t Hi = value << 8; + uint16_t Lo = value >> 8; + return Hi | Lo; +#endif +} + +/// This function returns a byte-swapped representation of the 32-bit argument. +inline uint32_t ByteSwap_32(uint32_t value) { +#if defined(__llvm__) || (defined(__GNUC__) && !defined(__ICC)) + return __builtin_bswap32(value); +#elif defined(_MSC_VER) && !defined(_DEBUG) + return _byteswap_ulong(value); +#else + uint32_t Byte0 = value & 0x000000FF; + uint32_t Byte1 = value & 0x0000FF00; + uint32_t Byte2 = value & 0x00FF0000; + uint32_t Byte3 = value & 0xFF000000; + return (Byte0 << 24) | (Byte1 << 8) | (Byte2 >> 8) | (Byte3 >> 24); +#endif +} + +/// This function returns a byte-swapped representation of the 64-bit argument. +inline uint64_t ByteSwap_64(uint64_t value) { +#if defined(__llvm__) || (defined(__GNUC__) && !defined(__ICC)) + return __builtin_bswap64(value); +#elif defined(_MSC_VER) && !defined(_DEBUG) + return _byteswap_uint64(value); +#else + uint64_t Hi = ByteSwap_32(uint32_t(value)); + uint32_t Lo = ByteSwap_32(uint32_t(value >> 32)); + return (Hi << 32) | Lo; +#endif +} + +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && BYTE_ORDER == BIG_ENDIAN +constexpr bool IsBigEndianHost = true; +#else +constexpr bool IsBigEndianHost = false; +#endif + +static const bool IsLittleEndianHost = !IsBigEndianHost; + +inline unsigned char getSwappedBytes(unsigned char C) { return C; } +inline signed char getSwappedBytes(signed char C) { return C; } +inline char getSwappedBytes(char C) { return C; } + +inline unsigned short getSwappedBytes(unsigned short C) { + return ByteSwap_16(C); +} +inline signed short getSwappedBytes(signed short C) { return ByteSwap_16(C); } + +inline unsigned int getSwappedBytes(unsigned int C) { return ByteSwap_32(C); } +inline signed int getSwappedBytes(signed int C) { return ByteSwap_32(C); } + +inline unsigned long getSwappedBytes(unsigned long C) { + // Handle LLP64 and LP64 platforms. + return sizeof(long) == sizeof(int) ? ByteSwap_32((uint32_t)C) + : ByteSwap_64((uint64_t)C); +} +inline signed long getSwappedBytes(signed long C) { + // Handle LLP64 and LP64 platforms. + return sizeof(long) == sizeof(int) ? ByteSwap_32((uint32_t)C) + : ByteSwap_64((uint64_t)C); +} + +inline unsigned long long getSwappedBytes(unsigned long long C) { + return ByteSwap_64(C); +} +inline signed long long getSwappedBytes(signed long long C) { + return ByteSwap_64(C); +} + +template <typename T> +inline std::enable_if_t<std::is_enum<T>::value, T> getSwappedBytes(T C) { + return static_cast<T>( + getSwappedBytes(static_cast<std::underlying_type_t<T>>(C))); +} + +template <typename T> inline void swapByteOrder(T &Value) { + Value = getSwappedBytes(Value); +} + +} // end namespace __orc_rt + +#endif // ORC_RT_ENDIAN_H diff --git a/compiler-rt/lib/orc/error.h b/compiler-rt/lib/orc/error.h new file mode 100644 index 000000000000..92ac5a884ac6 --- /dev/null +++ b/compiler-rt/lib/orc/error.h @@ -0,0 +1,428 @@ +//===-------- Error.h - Enforced error checking for ORC RT ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_ERROR_H +#define ORC_RT_ERROR_H + +#include "compiler.h" +#include "extensible_rtti.h" +#include "stl_extras.h" + +#include <cassert> +#include <memory> +#include <string> +#include <type_traits> + +namespace __orc_rt { + +/// Base class for all errors. +class ErrorInfoBase : public RTTIExtends<ErrorInfoBase, RTTIRoot> { +public: + virtual std::string toString() const = 0; +}; + +/// Represents an environmental error. +class ORC_RT_NODISCARD Error { + + template <typename ErrT, typename... ArgTs> + friend Error make_error(ArgTs &&...Args); + + friend Error repackage_error(std::unique_ptr<ErrorInfoBase>); + + template <typename ErrT> friend std::unique_ptr<ErrT> error_cast(Error &); + + template <typename T> friend class Expected; + +public: + /// Destroy this error. Aborts if error was not checked, or was checked but + /// not handled. + ~Error() { assertIsChecked(); } + + Error(const Error &) = delete; + Error &operator=(const Error &) = delete; + + /// Move-construct an error. The newly constructed error is considered + /// unchecked, even if the source error had been checked. The original error + /// becomes a checked success value. + Error(Error &&Other) { + setChecked(true); + *this = std::move(Other); + } + + /// Move-assign an error value. The current error must represent success, you + /// you cannot overwrite an unhandled error. The current error is then + /// considered unchecked. The source error becomes a checked success value, + /// regardless of its original state. + Error &operator=(Error &&Other) { + // Don't allow overwriting of unchecked values. + assertIsChecked(); + setPtr(Other.getPtr()); + + // This Error is unchecked, even if the source error was checked. + setChecked(false); + + // Null out Other's payload and set its checked bit. + Other.setPtr(nullptr); + Other.setChecked(true); + + return *this; + } + + /// Create a success value. + static Error success() { return Error(); } + + /// Error values convert to true for failure values, false otherwise. + explicit operator bool() { + setChecked(getPtr() == nullptr); + return getPtr() != nullptr; + } + + /// Return true if this Error contains a failure value of the given type. + template <typename ErrT> bool isA() const { + return getPtr() && getPtr()->isA<ErrT>(); + } + +private: + Error() = default; + + Error(std::unique_ptr<ErrorInfoBase> ErrInfo) { + auto RawErrPtr = reinterpret_cast<uintptr_t>(ErrInfo.release()); + assert((RawErrPtr & 0x1) == 0 && "ErrorInfo is insufficiently aligned"); + ErrPtr = RawErrPtr | 0x1; + } + + void assertIsChecked() { + if (ORC_RT_UNLIKELY(!isChecked() || getPtr())) { + fprintf(stderr, "Error must be checked prior to destruction.\n"); + abort(); // Some sort of JIT program abort? + } + } + + template <typename ErrT = ErrorInfoBase> ErrT *getPtr() const { + return reinterpret_cast<ErrT *>(ErrPtr & ~uintptr_t(1)); + } + + void setPtr(ErrorInfoBase *Ptr) { + ErrPtr = (reinterpret_cast<uintptr_t>(Ptr) & ~uintptr_t(1)) | (ErrPtr & 1); + } + + bool isChecked() const { return ErrPtr & 0x1; } + + void setChecked(bool Checked) { + ErrPtr = (reinterpret_cast<uintptr_t>(ErrPtr) & ~uintptr_t(1)) | Checked; + } + + template <typename ErrT = ErrorInfoBase> std::unique_ptr<ErrT> takePayload() { + static_assert(std::is_base_of<ErrorInfoBase, ErrT>::value, + "ErrT is not an ErrorInfoBase subclass"); + std::unique_ptr<ErrT> Tmp(getPtr<ErrT>()); + setPtr(nullptr); + setChecked(true); + return Tmp; + } + + uintptr_t ErrPtr = 0; +}; + +/// Construct an error of ErrT with the given arguments. +template <typename ErrT, typename... ArgTs> Error make_error(ArgTs &&...Args) { + static_assert(std::is_base_of<ErrorInfoBase, ErrT>::value, + "ErrT is not an ErrorInfoBase subclass"); + return Error(std::make_unique<ErrT>(std::forward<ArgTs>(Args)...)); +} + +/// Construct an error of ErrT using a std::unique_ptr<ErrorInfoBase>. The +/// primary use-case for this is 're-packaging' errors after inspecting them +/// using error_cast, hence the name. +inline Error repackage_error(std::unique_ptr<ErrorInfoBase> EIB) { + return Error(std::move(EIB)); +} + +/// If the argument is an error of type ErrT then this function unpacks it +/// and returns a std::unique_ptr<ErrT>. Otherwise returns a nullptr and +/// leaves the error untouched. Common usage looks like: +/// +/// \code{.cpp} +/// if (Error E = foo()) { +/// if (auto EV1 = error_cast<ErrorType1>(E)) { +/// // use unwrapped EV1 value. +/// } else if (EV2 = error_cast<ErrorType2>(E)) { +/// // use unwrapped EV2 value. +/// } ... +/// } +/// \endcode +template <typename ErrT> std::unique_ptr<ErrT> error_cast(Error &Err) { + static_assert(std::is_base_of<ErrorInfoBase, ErrT>::value, + "ErrT is not an ErrorInfoBase subclass"); + if (Err.isA<ErrT>()) + return Err.takePayload<ErrT>(); + return nullptr; +} + +/// Helper for Errors used as out-parameters. +/// Sets the 'checked' flag on construction, resets it on destruction. +class ErrorAsOutParameter { +public: + ErrorAsOutParameter(Error *Err) : Err(Err) { + // Raise the checked bit if Err is success. + if (Err) + (void)!!*Err; + } + + ~ErrorAsOutParameter() { + // Clear the checked bit. + if (Err && !*Err) + *Err = Error::success(); + } + +private: + Error *Err; +}; + +template <typename T> class ORC_RT_NODISCARD Expected { + + template <class OtherT> friend class Expected; + + static constexpr bool IsRef = std::is_reference<T>::value; + using wrap = std::reference_wrapper<std::remove_reference_t<T>>; + using error_type = std::unique_ptr<ErrorInfoBase>; + using storage_type = std::conditional_t<IsRef, wrap, T>; + using value_type = T; + + using reference = std::remove_reference_t<T> &; + using const_reference = const std::remove_reference_t<T> &; + using pointer = std::remove_reference_t<T> *; + using const_pointer = const std::remove_reference_t<T> *; + +public: + /// Create an Expected from a failure value. + Expected(Error Err) : HasError(true), Unchecked(true) { + assert(Err && "Cannot create Expected<T> from Error success value"); + new (getErrorStorage()) error_type(Err.takePayload()); + } + + /// Create an Expected from a T value. + template <typename OtherT> + Expected(OtherT &&Val, + std::enable_if_t<std::is_convertible<OtherT, T>::value> * = nullptr) + : HasError(false), Unchecked(true) { + new (getStorage()) storage_type(std::forward<OtherT>(Val)); + } + + /// Move-construct an Expected<T> from an Expected<OtherT>. + Expected(Expected &&Other) { moveConstruct(std::move(Other)); } + + /// Move construct an Expected<T> value from an Expected<OtherT>, where OtherT + /// must be convertible to T. + template <class OtherT> + Expected( + Expected<OtherT> &&Other, + std::enable_if_t<std::is_convertible<OtherT, T>::value> * = nullptr) { + moveConstruct(std::move(Other)); + } + + /// Move construct an Expected<T> value from an Expected<OtherT>, where OtherT + /// isn't convertible to T. + template <class OtherT> + explicit Expected( + Expected<OtherT> &&Other, + std::enable_if_t<!std::is_convertible<OtherT, T>::value> * = nullptr) { + moveConstruct(std::move(Other)); + } + + /// Move-assign from another Expected<T>. + Expected &operator=(Expected &&Other) { + moveAssign(std::move(Other)); + return *this; + } + + /// Destroy an Expected<T>. + ~Expected() { + assertIsChecked(); + if (!HasError) + getStorage()->~storage_type(); + else + getErrorStorage()->~error_type(); + } + + /// Returns true if this Expected value is in a success state (holding a T), + /// and false if this Expected value is in a failure state. + explicit operator bool() { + Unchecked = HasError; + return !HasError; + } + + /// Returns true if this Expected value holds an Error of type error_type. + template <typename ErrT> bool isFailureOfType() const { + return HasError && (*getErrorStorage())->template isFailureOfType<ErrT>(); + } + + /// Take ownership of the stored error. + /// + /// If this Expected value is in a success state (holding a T) then this + /// method is a no-op and returns Error::success. + /// + /// If thsi Expected value is in a failure state (holding an Error) then this + /// method returns the contained error and leaves this Expected in an + /// 'empty' state from which it may be safely destructed but not otherwise + /// accessed. + Error takeError() { + Unchecked = false; + return HasError ? Error(std::move(*getErrorStorage())) : Error::success(); + } + + /// Returns a pointer to the stored T value. + pointer operator->() { + assertIsChecked(); + return toPointer(getStorage()); + } + + /// Returns a pointer to the stored T value. + const_pointer operator->() const { + assertIsChecked(); + return toPointer(getStorage()); + } + + /// Returns a reference to the stored T value. + reference operator*() { + assertIsChecked(); + return *getStorage(); + } + + /// Returns a reference to the stored T value. + const_reference operator*() const { + assertIsChecked(); + return *getStorage(); + } + +private: + template <class T1> + static bool compareThisIfSameType(const T1 &a, const T1 &b) { + return &a == &b; + } + + template <class T1, class T2> + static bool compareThisIfSameType(const T1 &a, const T2 &b) { + return false; + } + + template <class OtherT> void moveConstruct(Expected<OtherT> &&Other) { + HasError = Other.HasError; + Unchecked = true; + Other.Unchecked = false; + + if (!HasError) + new (getStorage()) storage_type(std::move(*Other.getStorage())); + else + new (getErrorStorage()) error_type(std::move(*Other.getErrorStorage())); + } + + template <class OtherT> void moveAssign(Expected<OtherT> &&Other) { + assertIsChecked(); + + if (compareThisIfSameType(*this, Other)) + return; + + this->~Expected(); + new (this) Expected(std::move(Other)); + } + + pointer toPointer(pointer Val) { return Val; } + + const_pointer toPointer(const_pointer Val) const { return Val; } + + pointer toPointer(wrap *Val) { return &Val->get(); } + + const_pointer toPointer(const wrap *Val) const { return &Val->get(); } + + storage_type *getStorage() { + assert(!HasError && "Cannot get value when an error exists!"); + return reinterpret_cast<storage_type *>(&TStorage); + } + + const storage_type *getStorage() const { + assert(!HasError && "Cannot get value when an error exists!"); + return reinterpret_cast<const storage_type *>(&TStorage); + } + + error_type *getErrorStorage() { + assert(HasError && "Cannot get error when a value exists!"); + return reinterpret_cast<error_type *>(&ErrorStorage); + } + + const error_type *getErrorStorage() const { + assert(HasError && "Cannot get error when a value exists!"); + return reinterpret_cast<const error_type *>(&ErrorStorage); + } + + void assertIsChecked() { + if (ORC_RT_UNLIKELY(Unchecked)) { + fprintf(stderr, + "Expected<T> must be checked before access or destruction.\n"); + abort(); + } + } + + union { + std::aligned_union_t<1, storage_type> TStorage; + std::aligned_union_t<1, error_type> ErrorStorage; + }; + + bool HasError : 1; + bool Unchecked : 1; +}; + +/// Consume an error without doing anything. +inline void consumeError(Error Err) { + if (Err) + (void)error_cast<ErrorInfoBase>(Err); +} + +/// Consumes success values. It is a programmatic error to call this function +/// on a failure value. +inline void cantFail(Error Err) { + assert(!Err && "cantFail called on failure value"); + consumeError(std::move(Err)); +} + +/// Auto-unwrap an Expected<T> value in the success state. It is a programmatic +/// error to call this function on a failure value. +template <typename T> T cantFail(Expected<T> E) { + assert(E && "cantFail called on failure value"); + consumeError(E.takeError()); + return std::move(*E); +} + +/// Auto-unwrap an Expected<T> value in the success state. It is a programmatic +/// error to call this function on a failure value. +template <typename T> T &cantFail(Expected<T &> E) { + assert(E && "cantFail called on failure value"); + consumeError(E.takeError()); + return *E; +} + +/// Convert the given error to a string. The error value is consumed in the +/// process. +inline std::string toString(Error Err) { + if (auto EIB = error_cast<ErrorInfoBase>(Err)) + return EIB->toString(); + return {}; +} + +class StringError : public RTTIExtends<StringError, ErrorInfoBase> { +public: + StringError(std::string ErrMsg) : ErrMsg(std::move(ErrMsg)) {} + std::string toString() const override { return ErrMsg; } + +private: + std::string ErrMsg; +}; + +} // end namespace __orc_rt + +#endif // ORC_RT_ERROR_H diff --git a/compiler-rt/lib/orc/executor_address.h b/compiler-rt/lib/orc/executor_address.h new file mode 100644 index 000000000000..cfe985bdb60f --- /dev/null +++ b/compiler-rt/lib/orc/executor_address.h @@ -0,0 +1,208 @@ +//===------ ExecutorAddress.h - Executing process address -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Represents an address in the executing program. +// +// This file was derived from +// llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_EXECUTOR_ADDRESS_H +#define ORC_RT_EXECUTOR_ADDRESS_H + +#include "adt.h" +#include "simple_packed_serialization.h" + +#include <cassert> +#include <type_traits> + +namespace __orc_rt { + +/// Represents the difference between two addresses in the executor process. +class ExecutorAddrDiff { +public: + ExecutorAddrDiff() = default; + explicit ExecutorAddrDiff(uint64_t Value) : Value(Value) {} + + uint64_t getValue() const { return Value; } + +private: + int64_t Value = 0; +}; + +/// Represents an address in the executor process. +class ExecutorAddress { +public: + ExecutorAddress() = default; + explicit ExecutorAddress(uint64_t Addr) : Addr(Addr) {} + + /// Create an ExecutorAddress from the given pointer. + /// Warning: This should only be used when JITing in-process. + template <typename T> static ExecutorAddress fromPtr(T *Value) { + return ExecutorAddress( + static_cast<uint64_t>(reinterpret_cast<uintptr_t>(Value))); + } + + /// Cast this ExecutorAddress to a pointer of the given type. + /// Warning: This should only be esude when JITing in-process. + template <typename T> T toPtr() const { + static_assert(std::is_pointer<T>::value, "T must be a pointer type"); + uintptr_t IntPtr = static_cast<uintptr_t>(Addr); + assert(IntPtr == Addr && + "JITTargetAddress value out of range for uintptr_t"); + return reinterpret_cast<T>(IntPtr); + } + + uint64_t getValue() const { return Addr; } + void setValue(uint64_t Addr) { this->Addr = Addr; } + bool isNull() const { return Addr == 0; } + + explicit operator bool() const { return Addr != 0; } + + friend bool operator==(const ExecutorAddress &LHS, + const ExecutorAddress &RHS) { + return LHS.Addr == RHS.Addr; + } + + friend bool operator!=(const ExecutorAddress &LHS, + const ExecutorAddress &RHS) { + return LHS.Addr != RHS.Addr; + } + + friend bool operator<(const ExecutorAddress &LHS, + const ExecutorAddress &RHS) { + return LHS.Addr < RHS.Addr; + } + + friend bool operator<=(const ExecutorAddress &LHS, + const ExecutorAddress &RHS) { + return LHS.Addr <= RHS.Addr; + } + + friend bool operator>(const ExecutorAddress &LHS, + const ExecutorAddress &RHS) { + return LHS.Addr > RHS.Addr; + } + + friend bool operator>=(const ExecutorAddress &LHS, + const ExecutorAddress &RHS) { + return LHS.Addr >= RHS.Addr; + } + + ExecutorAddress &operator++() { + ++Addr; + return *this; + } + ExecutorAddress &operator--() { + --Addr; + return *this; + } + ExecutorAddress operator++(int) { return ExecutorAddress(Addr++); } + ExecutorAddress operator--(int) { return ExecutorAddress(Addr++); } + + ExecutorAddress &operator+=(const ExecutorAddrDiff Delta) { + Addr += Delta.getValue(); + return *this; + } + + ExecutorAddress &operator-=(const ExecutorAddrDiff Delta) { + Addr -= Delta.getValue(); + return *this; + } + +private: + uint64_t Addr = 0; +}; + +/// Subtracting two addresses yields an offset. +inline ExecutorAddrDiff operator-(const ExecutorAddress &LHS, + const ExecutorAddress &RHS) { + return ExecutorAddrDiff(LHS.getValue() - RHS.getValue()); +} + +/// Adding an offset and an address yields an address. +inline ExecutorAddress operator+(const ExecutorAddress &LHS, + const ExecutorAddrDiff &RHS) { + return ExecutorAddress(LHS.getValue() + RHS.getValue()); +} + +/// Adding an address and an offset yields an address. +inline ExecutorAddress operator+(const ExecutorAddrDiff &LHS, + const ExecutorAddress &RHS) { + return ExecutorAddress(LHS.getValue() + RHS.getValue()); +} + +/// Represents an address range in the exceutor process. +struct ExecutorAddressRange { + ExecutorAddressRange() = default; + ExecutorAddressRange(ExecutorAddress StartAddress, ExecutorAddress EndAddress) + : StartAddress(StartAddress), EndAddress(EndAddress) {} + + bool empty() const { return StartAddress == EndAddress; } + ExecutorAddrDiff size() const { return EndAddress - StartAddress; } + + template <typename T> span<T> toSpan() const { + assert(size().getValue() % sizeof(T) == 0 && + "AddressRange is not a multiple of sizeof(T)"); + return span<T>(StartAddress.toPtr<T *>(), size().getValue() / sizeof(T)); + } + + ExecutorAddress StartAddress; + ExecutorAddress EndAddress; +}; + +/// SPS serializatior for ExecutorAddress. +template <> class SPSSerializationTraits<SPSExecutorAddress, ExecutorAddress> { +public: + static size_t size(const ExecutorAddress &EA) { + return SPSArgList<uint64_t>::size(EA.getValue()); + } + + static bool serialize(SPSOutputBuffer &BOB, const ExecutorAddress &EA) { + return SPSArgList<uint64_t>::serialize(BOB, EA.getValue()); + } + + static bool deserialize(SPSInputBuffer &BIB, ExecutorAddress &EA) { + uint64_t Tmp; + if (!SPSArgList<uint64_t>::deserialize(BIB, Tmp)) + return false; + EA = ExecutorAddress(Tmp); + return true; + } +}; + +using SPSExecutorAddressRange = + SPSTuple<SPSExecutorAddress, SPSExecutorAddress>; + +/// Serialization traits for address ranges. +template <> +class SPSSerializationTraits<SPSExecutorAddressRange, ExecutorAddressRange> { +public: + static size_t size(const ExecutorAddressRange &Value) { + return SPSArgList<SPSExecutorAddress, SPSExecutorAddress>::size( + Value.StartAddress, Value.EndAddress); + } + + static bool serialize(SPSOutputBuffer &BOB, + const ExecutorAddressRange &Value) { + return SPSArgList<SPSExecutorAddress, SPSExecutorAddress>::serialize( + BOB, Value.StartAddress, Value.EndAddress); + } + + static bool deserialize(SPSInputBuffer &BIB, ExecutorAddressRange &Value) { + return SPSArgList<SPSExecutorAddress, SPSExecutorAddress>::deserialize( + BIB, Value.StartAddress, Value.EndAddress); + } +}; + +using SPSExecutorAddressRangeSequence = SPSSequence<SPSExecutorAddressRange>; + +} // End namespace __orc_rt + +#endif // ORC_RT_EXECUTOR_ADDRESS_H diff --git a/compiler-rt/lib/orc/extensible_rtti.cpp b/compiler-rt/lib/orc/extensible_rtti.cpp new file mode 100644 index 000000000000..c6951a449a3d --- /dev/null +++ b/compiler-rt/lib/orc/extensible_rtti.cpp @@ -0,0 +1,24 @@ +//===- extensible_rtti.cpp ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +// Note: +// This source file was adapted from lib/Support/ExtensibleRTTI.cpp, however +// the data structures are not shared and the code need not be kept in sync. +// +//===----------------------------------------------------------------------===// + +#include "extensible_rtti.h" + +namespace __orc_rt { + +char RTTIRoot::ID = 0; +void RTTIRoot::anchor() {} + +} // end namespace __orc_rt diff --git a/compiler-rt/lib/orc/extensible_rtti.h b/compiler-rt/lib/orc/extensible_rtti.h new file mode 100644 index 000000000000..72f68242e7c4 --- /dev/null +++ b/compiler-rt/lib/orc/extensible_rtti.h @@ -0,0 +1,145 @@ +//===------ extensible_rtti.h - Extensible RTTI for ORC RT ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// \file +// +// Provides an extensible RTTI mechanism, that can be used regardless of whether +// the runtime is built with -frtti or not. This is predominantly used to +// support error handling. +// +// The RTTIRoot class defines methods for comparing type ids. Implementations +// of these methods can be injected into new classes using the RTTIExtends +// class template. +// +// E.g. +// +// @code{.cpp} +// class MyBaseClass : public RTTIExtends<MyBaseClass, RTTIRoot> { +// public: +// static char ID; +// virtual void foo() = 0; +// }; +// +// class MyDerivedClass1 : public RTTIExtends<MyDerivedClass1, MyBaseClass> { +// public: +// static char ID; +// void foo() override {} +// }; +// +// class MyDerivedClass2 : public RTTIExtends<MyDerivedClass2, MyBaseClass> { +// public: +// static char ID; +// void foo() override {} +// }; +// +// char MyBaseClass::ID = 0; +// char MyDerivedClass1::ID = 0; +// char MyDerivedClass2:: ID = 0; +// +// void fn() { +// std::unique_ptr<MyBaseClass> B = std::make_unique<MyDerivedClass1>(); +// outs() << isa<MyBaseClass>(B) << "\n"; // Outputs "1". +// outs() << isa<MyDerivedClass1>(B) << "\n"; // Outputs "1". +// outs() << isa<MyDerivedClass2>(B) << "\n"; // Outputs "0'. +// } +// +// @endcode +// +// Note: +// This header was adapted from llvm/Support/ExtensibleRTTI.h, however the +// data structures are not shared and the code need not be kept in sync. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_EXTENSIBLE_RTTI_H +#define ORC_RT_EXTENSIBLE_RTTI_H + +namespace __orc_rt { + +template <typename ThisT, typename ParentT> class RTTIExtends; + +/// Base class for the extensible RTTI hierarchy. +/// +/// This class defines virtual methods, dynamicClassID and isA, that enable +/// type comparisons. +class RTTIRoot { +public: + virtual ~RTTIRoot() = default; + + /// Returns the class ID for this type. + static const void *classID() { return &ID; } + + /// Returns the class ID for the dynamic type of this RTTIRoot instance. + virtual const void *dynamicClassID() const = 0; + + /// Returns true if this class's ID matches the given class ID. + virtual bool isA(const void *const ClassID) const { + return ClassID == classID(); + } + + /// Check whether this instance is a subclass of QueryT. + template <typename QueryT> bool isA() const { return isA(QueryT::classID()); } + + static bool classof(const RTTIRoot *R) { return R->isA<RTTIRoot>(); } + +private: + virtual void anchor(); + + static char ID; +}; + +/// Inheritance utility for extensible RTTI. +/// +/// Supports single inheritance only: A class can only have one +/// ExtensibleRTTI-parent (i.e. a parent for which the isa<> test will work), +/// though it can have many non-ExtensibleRTTI parents. +/// +/// RTTIExtents uses CRTP so the first template argument to RTTIExtends is the +/// newly introduced type, and the *second* argument is the parent class. +/// +/// class MyType : public RTTIExtends<MyType, RTTIRoot> { +/// public: +/// static char ID; +/// }; +/// +/// class MyDerivedType : public RTTIExtends<MyDerivedType, MyType> { +/// public: +/// static char ID; +/// }; +/// +template <typename ThisT, typename ParentT> class RTTIExtends : public ParentT { +public: + // Inherit constructors and isA methods from ParentT. + using ParentT::isA; + using ParentT::ParentT; + + static char ID; + + static const void *classID() { return &ThisT::ID; } + + const void *dynamicClassID() const override { return &ThisT::ID; } + + bool isA(const void *const ClassID) const override { + return ClassID == classID() || ParentT::isA(ClassID); + } + + static bool classof(const RTTIRoot *R) { return R->isA<ThisT>(); } +}; + +template <typename ThisT, typename ParentT> +char RTTIExtends<ThisT, ParentT>::ID = 0; + +/// Returns true if the given value is an instance of the template type +/// parameter. +template <typename To, typename From> bool isa(const From &Value) { + return To::classof(&Value); +} + +} // end namespace __orc_rt + +#endif // ORC_RT_EXTENSIBLE_RTTI_H diff --git a/compiler-rt/lib/orc/log_error_to_stderr.cpp b/compiler-rt/lib/orc/log_error_to_stderr.cpp new file mode 100644 index 000000000000..4fabbc0d212a --- /dev/null +++ b/compiler-rt/lib/orc/log_error_to_stderr.cpp @@ -0,0 +1,19 @@ +//===-- log_error_to_stderr.cpp -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +#include "compiler.h" + +#include <stdio.h> + +ORC_RT_INTERFACE void __orc_rt_log_error_to_stderr(const char *ErrMsg) { + fprintf(stderr, "orc runtime error: %s\n", ErrMsg); +} diff --git a/compiler-rt/lib/orc/macho_platform.cpp b/compiler-rt/lib/orc/macho_platform.cpp new file mode 100644 index 000000000000..2a960fb548fa --- /dev/null +++ b/compiler-rt/lib/orc/macho_platform.cpp @@ -0,0 +1,731 @@ +//===- macho_platform.cpp -------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains code required to load the rest of the MachO runtime. +// +//===----------------------------------------------------------------------===// + +#include "macho_platform.h" +#include "common.h" +#include "error.h" +#include "wrapper_function_utils.h" + +#include <map> +#include <mutex> +#include <sstream> +#include <unordered_map> +#include <vector> + +using namespace __orc_rt; +using namespace __orc_rt::macho; + +// Declare function tags for functions in the JIT process. +ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_get_initializers_tag) +ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_get_deinitializers_tag) +ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_symbol_lookup_tag) + +// eh-frame registration functions. +// We expect these to be available for all processes. +extern "C" void __register_frame(const void *); +extern "C" void __deregister_frame(const void *); + +// Objective-C types. +struct objc_class; +struct objc_image_info; +struct objc_object; +struct objc_selector; + +using Class = objc_class *; +using id = objc_object *; +using SEL = objc_selector *; + +// Objective-C registration functions. +// These are weakly imported. If the Objective-C runtime has not been loaded +// then code containing Objective-C sections will generate an error. +extern "C" id objc_msgSend(id, SEL, ...) ORC_RT_WEAK_IMPORT; +extern "C" Class objc_readClassPair(Class, + const objc_image_info *) ORC_RT_WEAK_IMPORT; +extern "C" SEL sel_registerName(const char *) ORC_RT_WEAK_IMPORT; + +// Swift types. +class ProtocolRecord; +class ProtocolConformanceRecord; + +extern "C" void +swift_registerProtocols(const ProtocolRecord *begin, + const ProtocolRecord *end) ORC_RT_WEAK_IMPORT; + +extern "C" void swift_registerProtocolConformances( + const ProtocolConformanceRecord *begin, + const ProtocolConformanceRecord *end) ORC_RT_WEAK_IMPORT; + +namespace { + +template <typename HandleFDEFn> +void walkEHFrameSection(span<const char> EHFrameSection, + HandleFDEFn HandleFDE) { + const char *CurCFIRecord = EHFrameSection.data(); + uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord); + + while (CurCFIRecord != EHFrameSection.end() && Size != 0) { + const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4); + if (Size == 0xffffffff) + Size = *reinterpret_cast<const uint64_t *>(CurCFIRecord + 4) + 12; + else + Size += 4; + uint32_t Offset = *reinterpret_cast<const uint32_t *>(OffsetField); + + if (Offset != 0) + HandleFDE(CurCFIRecord); + + CurCFIRecord += Size; + Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord); + } +} + +Error validatePointerSectionExtent(const char *SectionName, + const ExecutorAddressRange &SE) { + if (SE.size().getValue() % sizeof(uintptr_t)) { + std::ostringstream ErrMsg; + ErrMsg << std::hex << "Size of " << SectionName << " 0x" + << SE.StartAddress.getValue() << " -- 0x" << SE.EndAddress.getValue() + << " is not a pointer multiple"; + return make_error<StringError>(ErrMsg.str()); + } + return Error::success(); +} + +Error registerObjCSelectors( + const std::vector<ExecutorAddressRange> &ObjCSelRefsSections, + const MachOJITDylibInitializers &MOJDIs) { + + if (ORC_RT_UNLIKELY(!sel_registerName)) + return make_error<StringError>("sel_registerName is not available"); + + for (const auto &ObjCSelRefs : ObjCSelRefsSections) { + + if (auto Err = validatePointerSectionExtent("__objc_selrefs", ObjCSelRefs)) + return Err; + + fprintf(stderr, "Processing selrefs section at 0x%llx\n", + ObjCSelRefs.StartAddress.getValue()); + for (uintptr_t SelEntry : ObjCSelRefs.toSpan<uintptr_t>()) { + const char *SelName = reinterpret_cast<const char *>(SelEntry); + fprintf(stderr, "Registering selector \"%s\"\n", SelName); + auto Sel = sel_registerName(SelName); + *reinterpret_cast<SEL *>(SelEntry) = Sel; + } + } + + return Error::success(); +} + +Error registerObjCClasses( + const std::vector<ExecutorAddressRange> &ObjCClassListSections, + const MachOJITDylibInitializers &MOJDIs) { + + if (ObjCClassListSections.empty()) + return Error::success(); + + if (ORC_RT_UNLIKELY(!objc_msgSend)) + return make_error<StringError>("objc_msgSend is not available"); + if (ORC_RT_UNLIKELY(!objc_readClassPair)) + return make_error<StringError>("objc_readClassPair is not available"); + + struct ObjCClassCompiled { + void *Metaclass; + void *Parent; + void *Cache1; + void *Cache2; + void *Data; + }; + + auto *ImageInfo = + MOJDIs.ObjCImageInfoAddress.toPtr<const objc_image_info *>(); + auto ClassSelector = sel_registerName("class"); + + for (const auto &ObjCClassList : ObjCClassListSections) { + + if (auto Err = + validatePointerSectionExtent("__objc_classlist", ObjCClassList)) + return Err; + + for (uintptr_t ClassPtr : ObjCClassList.toSpan<uintptr_t>()) { + auto *Cls = reinterpret_cast<Class>(ClassPtr); + auto *ClassCompiled = reinterpret_cast<ObjCClassCompiled *>(ClassPtr); + objc_msgSend(reinterpret_cast<id>(ClassCompiled->Parent), ClassSelector); + auto Registered = objc_readClassPair(Cls, ImageInfo); + + // FIXME: Improve diagnostic by reporting the failed class's name. + if (Registered != Cls) + return make_error<StringError>("Unable to register Objective-C class"); + } + } + return Error::success(); +} + +Error registerSwift5Protocols( + const std::vector<ExecutorAddressRange> &Swift5ProtocolSections, + const MachOJITDylibInitializers &MOJDIs) { + + if (ORC_RT_UNLIKELY(!Swift5ProtocolSections.empty() && + !swift_registerProtocols)) + return make_error<StringError>("swift_registerProtocols is not available"); + + for (const auto &Swift5Protocols : Swift5ProtocolSections) + swift_registerProtocols( + Swift5Protocols.StartAddress.toPtr<const ProtocolRecord *>(), + Swift5Protocols.EndAddress.toPtr<const ProtocolRecord *>()); + + return Error::success(); +} + +Error registerSwift5ProtocolConformances( + const std::vector<ExecutorAddressRange> &Swift5ProtocolConformanceSections, + const MachOJITDylibInitializers &MOJDIs) { + + if (ORC_RT_UNLIKELY(!Swift5ProtocolConformanceSections.empty() && + !swift_registerProtocolConformances)) + return make_error<StringError>( + "swift_registerProtocolConformances is not available"); + + for (const auto &ProtoConfSec : Swift5ProtocolConformanceSections) + swift_registerProtocolConformances( + ProtoConfSec.StartAddress.toPtr<const ProtocolConformanceRecord *>(), + ProtoConfSec.EndAddress.toPtr<const ProtocolConformanceRecord *>()); + + return Error::success(); +} + +Error runModInits(const std::vector<ExecutorAddressRange> &ModInitsSections, + const MachOJITDylibInitializers &MOJDIs) { + + for (const auto &ModInits : ModInitsSections) { + if (auto Err = validatePointerSectionExtent("__mod_inits", ModInits)) + return Err; + + using InitFunc = void (*)(); + for (auto *Init : ModInits.toSpan<InitFunc>()) + (*Init)(); + } + + return Error::success(); +} + +struct TLVDescriptor { + void *(*Thunk)(TLVDescriptor *) = nullptr; + unsigned long Key = 0; + unsigned long DataAddress = 0; +}; + +class MachOPlatformRuntimeState { +private: + struct AtExitEntry { + void (*Func)(void *); + void *Arg; + }; + + using AtExitsVector = std::vector<AtExitEntry>; + + struct PerJITDylibState { + void *Header = nullptr; + size_t RefCount = 0; + bool AllowReinitialization = false; + AtExitsVector AtExits; + }; + +public: + static void initialize(); + static MachOPlatformRuntimeState &get(); + static void destroy(); + + MachOPlatformRuntimeState() = default; + + // Delete copy and move constructors. + MachOPlatformRuntimeState(const MachOPlatformRuntimeState &) = delete; + MachOPlatformRuntimeState & + operator=(const MachOPlatformRuntimeState &) = delete; + MachOPlatformRuntimeState(MachOPlatformRuntimeState &&) = delete; + MachOPlatformRuntimeState &operator=(MachOPlatformRuntimeState &&) = delete; + + Error registerObjectSections(MachOPerObjectSectionsToRegister POSR); + Error deregisterObjectSections(MachOPerObjectSectionsToRegister POSR); + + const char *dlerror(); + void *dlopen(string_view Name, int Mode); + int dlclose(void *DSOHandle); + void *dlsym(void *DSOHandle, string_view Symbol); + + int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle); + void runAtExits(void *DSOHandle); + + /// Returns the base address of the section containing ThreadData. + Expected<std::pair<const char *, size_t>> + getThreadDataSectionFor(const char *ThreadData); + +private: + PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle); + PerJITDylibState *getJITDylibStateByName(string_view Path); + PerJITDylibState &getOrCreateJITDylibState(MachOJITDylibInitializers &MOJDIs); + + Error registerThreadDataSection(span<const char> ThreadDataSec); + + Expected<ExecutorAddress> lookupSymbolInJITDylib(void *DSOHandle, + string_view Symbol); + + Expected<MachOJITDylibInitializerSequence> + getJITDylibInitializersByName(string_view Path); + Expected<void *> dlopenInitialize(string_view Path, int Mode); + Error initializeJITDylib(MachOJITDylibInitializers &MOJDIs); + + static MachOPlatformRuntimeState *MOPS; + + using InitSectionHandler = + Error (*)(const std::vector<ExecutorAddressRange> &Sections, + const MachOJITDylibInitializers &MOJDIs); + const std::vector<std::pair<const char *, InitSectionHandler>> InitSections = + {{"__DATA,__objc_selrefs", registerObjCSelectors}, + {"__DATA,__objc_classlist", registerObjCClasses}, + {"__TEXT,__swift5_protos", registerSwift5Protocols}, + {"__TEXT,__swift5_proto", registerSwift5ProtocolConformances}, + {"__DATA,__mod_init_func", runModInits}}; + + // FIXME: Move to thread-state. + std::string DLFcnError; + + std::recursive_mutex JDStatesMutex; + std::unordered_map<void *, PerJITDylibState> JDStates; + std::unordered_map<std::string, void *> JDNameToHeader; + + std::mutex ThreadDataSectionsMutex; + std::map<const char *, size_t> ThreadDataSections; +}; + +MachOPlatformRuntimeState *MachOPlatformRuntimeState::MOPS = nullptr; + +void MachOPlatformRuntimeState::initialize() { + assert(!MOPS && "MachOPlatformRuntimeState should be null"); + MOPS = new MachOPlatformRuntimeState(); +} + +MachOPlatformRuntimeState &MachOPlatformRuntimeState::get() { + assert(MOPS && "MachOPlatformRuntimeState not initialized"); + return *MOPS; +} + +void MachOPlatformRuntimeState::destroy() { + assert(MOPS && "MachOPlatformRuntimeState not initialized"); + delete MOPS; +} + +Error MachOPlatformRuntimeState::registerObjectSections( + MachOPerObjectSectionsToRegister POSR) { + if (POSR.EHFrameSection.StartAddress) + walkEHFrameSection(POSR.EHFrameSection.toSpan<const char>(), + __register_frame); + + if (POSR.ThreadDataSection.StartAddress) { + if (auto Err = registerThreadDataSection( + POSR.ThreadDataSection.toSpan<const char>())) + return Err; + } + + return Error::success(); +} + +Error MachOPlatformRuntimeState::deregisterObjectSections( + MachOPerObjectSectionsToRegister POSR) { + if (POSR.EHFrameSection.StartAddress) + walkEHFrameSection(POSR.EHFrameSection.toSpan<const char>(), + __deregister_frame); + + return Error::success(); +} + +const char *MachOPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); } + +void *MachOPlatformRuntimeState::dlopen(string_view Path, int Mode) { + std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); + + // Use fast path if all JITDylibs are already loaded and don't require + // re-running initializers. + if (auto *JDS = getJITDylibStateByName(Path)) { + if (!JDS->AllowReinitialization) { + ++JDS->RefCount; + return JDS->Header; + } + } + + auto H = dlopenInitialize(Path, Mode); + if (!H) { + DLFcnError = toString(H.takeError()); + return nullptr; + } + + return *H; +} + +int MachOPlatformRuntimeState::dlclose(void *DSOHandle) { + runAtExits(DSOHandle); + return 0; +} + +void *MachOPlatformRuntimeState::dlsym(void *DSOHandle, string_view Symbol) { + auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol); + if (!Addr) { + DLFcnError = toString(Addr.takeError()); + return 0; + } + + return Addr->toPtr<void *>(); +} + +int MachOPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg, + void *DSOHandle) { + // FIXME: Handle out-of-memory errors, returning -1 if OOM. + std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); + auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle); + assert(JDS && "JITDylib state not initialized"); + JDS->AtExits.push_back({F, Arg}); + return 0; +} + +void MachOPlatformRuntimeState::runAtExits(void *DSOHandle) { + // FIXME: Should atexits be allowed to run concurrently with access to + // JDState? + AtExitsVector V; + { + std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); + auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle); + assert(JDS && "JITDlybi state not initialized"); + std::swap(V, JDS->AtExits); + } + + while (!V.empty()) { + auto &AE = V.back(); + AE.Func(AE.Arg); + V.pop_back(); + } +} + +Expected<std::pair<const char *, size_t>> +MachOPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) { + std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex); + auto I = ThreadDataSections.upper_bound(ThreadData); + // Check that we have a valid entry covering this address. + if (I == ThreadDataSections.begin()) + return make_error<StringError>("No thread local data section for key"); + I = std::prev(I); + if (ThreadData >= I->first + I->second) + return make_error<StringError>("No thread local data section for key"); + return *I; +} + +MachOPlatformRuntimeState::PerJITDylibState * +MachOPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) { + auto I = JDStates.find(DSOHandle); + if (I == JDStates.end()) + return nullptr; + return &I->second; +} + +MachOPlatformRuntimeState::PerJITDylibState * +MachOPlatformRuntimeState::getJITDylibStateByName(string_view Name) { + // FIXME: Avoid creating string copy here. + auto I = JDNameToHeader.find(std::string(Name.data(), Name.size())); + if (I == JDNameToHeader.end()) + return nullptr; + void *H = I->second; + auto J = JDStates.find(H); + assert(J != JDStates.end() && + "JITDylib has name map entry but no header map entry"); + return &J->second; +} + +MachOPlatformRuntimeState::PerJITDylibState & +MachOPlatformRuntimeState::getOrCreateJITDylibState( + MachOJITDylibInitializers &MOJDIs) { + void *Header = MOJDIs.MachOHeaderAddress.toPtr<void *>(); + + auto &JDS = JDStates[Header]; + + // If this entry hasn't been created yet. + if (!JDS.Header) { + assert(!JDNameToHeader.count(MOJDIs.Name) && + "JITDylib has header map entry but no name map entry"); + JDNameToHeader[MOJDIs.Name] = Header; + JDS.Header = Header; + } + + return JDS; +} + +Error MachOPlatformRuntimeState::registerThreadDataSection( + span<const char> ThreadDataSection) { + std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex); + auto I = ThreadDataSections.upper_bound(ThreadDataSection.data()); + if (I != ThreadDataSections.begin()) { + auto J = std::prev(I); + if (J->first + J->second > ThreadDataSection.data()) + return make_error<StringError>("Overlapping __thread_data sections"); + } + ThreadDataSections.insert( + I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size())); + return Error::success(); +} + +Expected<ExecutorAddress> +MachOPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle, + string_view Sym) { + Expected<ExecutorAddress> Result((ExecutorAddress())); + if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddress>( + SPSExecutorAddress, + SPSString)>::call(&__orc_rt_macho_symbol_lookup_tag, Result, + ExecutorAddress::fromPtr(DSOHandle), Sym)) + return std::move(Err); + return Result; +} + +Expected<MachOJITDylibInitializerSequence> +MachOPlatformRuntimeState::getJITDylibInitializersByName(string_view Path) { + Expected<MachOJITDylibInitializerSequence> Result( + (MachOJITDylibInitializerSequence())); + std::string PathStr(Path.data(), Path.size()); + if (auto Err = + WrapperFunction<SPSExpected<SPSMachOJITDylibInitializerSequence>( + SPSString)>::call(&__orc_rt_macho_get_initializers_tag, Result, + Path)) + return std::move(Err); + return Result; +} + +Expected<void *> MachOPlatformRuntimeState::dlopenInitialize(string_view Path, + int Mode) { + // Either our JITDylib wasn't loaded, or it or one of its dependencies allows + // reinitialization. We need to call in to the JIT to see if there's any new + // work pending. + auto InitSeq = getJITDylibInitializersByName(Path); + if (!InitSeq) + return InitSeq.takeError(); + + // Init sequences should be non-empty. + if (InitSeq->empty()) + return make_error<StringError>( + "__orc_rt_macho_get_initializers returned an " + "empty init sequence"); + + // Otherwise register and run initializers for each JITDylib. + for (auto &MOJDIs : *InitSeq) + if (auto Err = initializeJITDylib(MOJDIs)) + return std::move(Err); + + // Return the header for the last item in the list. + auto *JDS = getJITDylibStateByHeaderAddr( + InitSeq->back().MachOHeaderAddress.toPtr<void *>()); + assert(JDS && "Missing state entry for JD"); + return JDS->Header; +} + +Error MachOPlatformRuntimeState::initializeJITDylib( + MachOJITDylibInitializers &MOJDIs) { + + auto &JDS = getOrCreateJITDylibState(MOJDIs); + ++JDS.RefCount; + + for (auto &KV : InitSections) { + const auto &Name = KV.first; + const auto &Handler = KV.second; + auto I = MOJDIs.InitSections.find(Name); + if (I != MOJDIs.InitSections.end()) { + if (auto Err = Handler(I->second, MOJDIs)) + return Err; + } + } + + return Error::success(); +} + +class MachOPlatformRuntimeTLVManager { +public: + void *getInstance(const char *ThreadData); + +private: + std::unordered_map<const char *, char *> Instances; + std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections; +}; + +void *MachOPlatformRuntimeTLVManager::getInstance(const char *ThreadData) { + auto I = Instances.find(ThreadData); + if (I != Instances.end()) + return I->second; + + auto TDS = + MachOPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData); + if (!TDS) { + __orc_rt_log_error(toString(TDS.takeError()).c_str()); + return nullptr; + } + + auto &Allocated = AllocatedSections[TDS->first]; + if (!Allocated) { + Allocated = std::make_unique<char[]>(TDS->second); + memcpy(Allocated.get(), TDS->first, TDS->second); + } + + size_t ThreadDataDelta = ThreadData - TDS->first; + assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds"); + + char *Instance = Allocated.get() + ThreadDataDelta; + Instances[ThreadData] = Instance; + return Instance; +} + +void destroyMachOTLVMgr(void *MachOTLVMgr) { + delete static_cast<MachOPlatformRuntimeTLVManager *>(MachOTLVMgr); +} + +} // end anonymous namespace + +//------------------------------------------------------------------------------ +// JIT entry points +//------------------------------------------------------------------------------ + +ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult +__orc_rt_macho_platform_bootstrap(char *ArgData, size_t ArgSize) { + MachOPlatformRuntimeState::initialize(); + return WrapperFunctionResult().release(); +} + +ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult +__orc_rt_macho_platform_shutdown(char *ArgData, size_t ArgSize) { + MachOPlatformRuntimeState::destroy(); + return WrapperFunctionResult().release(); +} + +/// Wrapper function for registering metadata on a per-object basis. +ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult +__orc_rt_macho_register_object_sections(char *ArgData, size_t ArgSize) { + return WrapperFunction<SPSError(SPSMachOPerObjectSectionsToRegister)>::handle( + ArgData, ArgSize, + [](MachOPerObjectSectionsToRegister &POSR) { + return MachOPlatformRuntimeState::get().registerObjectSections( + std::move(POSR)); + }) + .release(); +} + +/// Wrapper for releasing per-object metadat. +ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult +__orc_rt_macho_deregister_object_sections(char *ArgData, size_t ArgSize) { + return WrapperFunction<SPSError(SPSMachOPerObjectSectionsToRegister)>::handle( + ArgData, ArgSize, + [](MachOPerObjectSectionsToRegister &POSR) { + return MachOPlatformRuntimeState::get().deregisterObjectSections( + std::move(POSR)); + }) + .release(); +} + +//------------------------------------------------------------------------------ +// TLV support +//------------------------------------------------------------------------------ + +ORC_RT_INTERFACE void *__orc_rt_macho_tlv_get_addr_impl(TLVDescriptor *D) { + auto *TLVMgr = static_cast<MachOPlatformRuntimeTLVManager *>( + pthread_getspecific(D->Key)); + if (!TLVMgr) { + TLVMgr = new MachOPlatformRuntimeTLVManager(); + if (pthread_setspecific(D->Key, TLVMgr)) { + __orc_rt_log_error("Call to pthread_setspecific failed"); + return nullptr; + } + } + + return TLVMgr->getInstance( + reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress))); +} + +ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult +__orc_rt_macho_create_pthread_key(char *ArgData, size_t ArgSize) { + return WrapperFunction<SPSExpected<uint64_t>(void)>::handle( + ArgData, ArgSize, + []() -> Expected<uint64_t> { + pthread_key_t Key; + if (int Err = pthread_key_create(&Key, destroyMachOTLVMgr)) { + __orc_rt_log_error("Call to pthread_key_create failed"); + return make_error<StringError>(strerror(Err)); + } + return static_cast<uint64_t>(Key); + }) + .release(); +} + +//------------------------------------------------------------------------------ +// cxa_atexit support +//------------------------------------------------------------------------------ + +int __orc_rt_macho_cxa_atexit(void (*func)(void *), void *arg, + void *dso_handle) { + return MachOPlatformRuntimeState::get().registerAtExit(func, arg, dso_handle); +} + +void __orc_rt_macho_cxa_finalize(void *dso_handle) { + MachOPlatformRuntimeState::get().runAtExits(dso_handle); +} + +//------------------------------------------------------------------------------ +// JIT'd dlfcn alternatives. +//------------------------------------------------------------------------------ + +const char *__orc_rt_macho_jit_dlerror() { + return MachOPlatformRuntimeState::get().dlerror(); +} + +void *__orc_rt_macho_jit_dlopen(const char *path, int mode) { + return MachOPlatformRuntimeState::get().dlopen(path, mode); +} + +int __orc_rt_macho_jit_dlclose(void *dso_handle) { + return MachOPlatformRuntimeState::get().dlclose(dso_handle); +} + +void *__orc_rt_macho_jit_dlsym(void *dso_handle, const char *symbol) { + return MachOPlatformRuntimeState::get().dlsym(dso_handle, symbol); +} + +//------------------------------------------------------------------------------ +// MachO Run Program +//------------------------------------------------------------------------------ + +ORC_RT_INTERFACE int64_t __orc_rt_macho_run_program(const char *JITDylibName, + const char *EntrySymbolName, + int argc, char *argv[]) { + using MainTy = int (*)(int, char *[]); + + void *H = __orc_rt_macho_jit_dlopen(JITDylibName, + __orc_rt::macho::ORC_RT_RTLD_LAZY); + if (!H) { + __orc_rt_log_error(__orc_rt_macho_jit_dlerror()); + return -1; + } + + auto *Main = + reinterpret_cast<MainTy>(__orc_rt_macho_jit_dlsym(H, EntrySymbolName)); + + if (!Main) { + __orc_rt_log_error(__orc_rt_macho_jit_dlerror()); + return -1; + } + + int Result = Main(argc, argv); + + if (__orc_rt_macho_jit_dlclose(H) == -1) + __orc_rt_log_error(__orc_rt_macho_jit_dlerror()); + + return Result; +} diff --git a/compiler-rt/lib/orc/macho_platform.h b/compiler-rt/lib/orc/macho_platform.h new file mode 100644 index 000000000000..6c05e844b0cd --- /dev/null +++ b/compiler-rt/lib/orc/macho_platform.h @@ -0,0 +1,135 @@ +//===- macho_platform.h -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ORC Runtime support for Darwin dynamic loading features. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_MACHO_PLATFORM_H +#define ORC_RT_MACHO_PLATFORM_H + +#include "common.h" +#include "executor_address.h" + +// Atexit functions. +ORC_RT_INTERFACE int __orc_rt_macho_cxa_atexit(void (*func)(void *), void *arg, + void *dso_handle); +ORC_RT_INTERFACE void __orc_rt_macho_cxa_finalize(void *dso_handle); + +// dlfcn functions. +ORC_RT_INTERFACE const char *__orc_rt_macho_jit_dlerror(); +ORC_RT_INTERFACE void *__orc_rt_macho_jit_dlopen(const char *path, int mode); +ORC_RT_INTERFACE int __orc_rt_macho_jit_dlclose(void *dso_handle); +ORC_RT_INTERFACE void *__orc_rt_macho_jit_dlsym(void *dso_handle, + const char *symbol); + +namespace __orc_rt { +namespace macho { + +struct MachOPerObjectSectionsToRegister { + ExecutorAddressRange EHFrameSection; + ExecutorAddressRange ThreadDataSection; +}; + +struct MachOJITDylibInitializers { + using SectionList = std::vector<ExecutorAddressRange>; + + MachOJITDylibInitializers() = default; + MachOJITDylibInitializers(std::string Name, + ExecutorAddress MachOHeaderAddress) + : Name(std::move(Name)), + MachOHeaderAddress(std::move(MachOHeaderAddress)) {} + + std::string Name; + ExecutorAddress MachOHeaderAddress; + ExecutorAddress ObjCImageInfoAddress; + + std::unordered_map<std::string, SectionList> InitSections; +}; + +class MachOJITDylibDeinitializers {}; + +using MachOJITDylibInitializerSequence = std::vector<MachOJITDylibInitializers>; + +using MachOJITDylibDeinitializerSequence = + std::vector<MachOJITDylibDeinitializers>; + +enum dlopen_mode : int { + ORC_RT_RTLD_LAZY = 0x1, + ORC_RT_RTLD_NOW = 0x2, + ORC_RT_RTLD_LOCAL = 0x4, + ORC_RT_RTLD_GLOBAL = 0x8 +}; + +} // end namespace macho + +using SPSMachOPerObjectSectionsToRegister = + SPSTuple<SPSExecutorAddressRange, SPSExecutorAddressRange>; + +template <> +class SPSSerializationTraits<SPSMachOPerObjectSectionsToRegister, + macho::MachOPerObjectSectionsToRegister> { + +public: + static size_t size(const macho::MachOPerObjectSectionsToRegister &MOPOSR) { + return SPSMachOPerObjectSectionsToRegister::AsArgList::size( + MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection); + } + + static bool serialize(SPSOutputBuffer &OB, + const macho::MachOPerObjectSectionsToRegister &MOPOSR) { + return SPSMachOPerObjectSectionsToRegister::AsArgList::serialize( + OB, MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection); + } + + static bool deserialize(SPSInputBuffer &IB, + macho::MachOPerObjectSectionsToRegister &MOPOSR) { + return SPSMachOPerObjectSectionsToRegister::AsArgList::deserialize( + IB, MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection); + } +}; + +using SPSNamedExecutorAddressRangeSequenceMap = + SPSSequence<SPSTuple<SPSString, SPSExecutorAddressRangeSequence>>; + +using SPSMachOJITDylibInitializers = + SPSTuple<SPSString, SPSExecutorAddress, SPSExecutorAddress, + SPSNamedExecutorAddressRangeSequenceMap>; + +using SPSMachOJITDylibInitializerSequence = + SPSSequence<SPSMachOJITDylibInitializers>; + +/// Serialization traits for MachOJITDylibInitializers. +template <> +class SPSSerializationTraits<SPSMachOJITDylibInitializers, + macho::MachOJITDylibInitializers> { +public: + static size_t size(const macho::MachOJITDylibInitializers &MOJDIs) { + return SPSMachOJITDylibInitializers::AsArgList::size( + MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.ObjCImageInfoAddress, + MOJDIs.InitSections); + } + + static bool serialize(SPSOutputBuffer &OB, + const macho::MachOJITDylibInitializers &MOJDIs) { + return SPSMachOJITDylibInitializers::AsArgList::serialize( + OB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.ObjCImageInfoAddress, + MOJDIs.InitSections); + } + + static bool deserialize(SPSInputBuffer &IB, + macho::MachOJITDylibInitializers &MOJDIs) { + return SPSMachOJITDylibInitializers::AsArgList::deserialize( + IB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.ObjCImageInfoAddress, + MOJDIs.InitSections); + } +}; + +} // end namespace __orc_rt + +#endif // ORC_RT_MACHO_PLATFORM_H diff --git a/compiler-rt/lib/orc/macho_tlv.x86-64.S b/compiler-rt/lib/orc/macho_tlv.x86-64.S new file mode 100644 index 000000000000..0affe403eec2 --- /dev/null +++ b/compiler-rt/lib/orc/macho_tlv.x86-64.S @@ -0,0 +1,68 @@ +//===-- orc_rt_macho_tlv.x86-64.s -------------------------------*- ASM -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +#define REGISTER_SAVE_SPACE_SIZE 512 + + .text + + // returns address of TLV in %rax, all other registers preserved + .globl ___orc_rt_macho_tlv_get_addr +___orc_rt_macho_tlv_get_addr: + pushq %rbp + movq %rsp, %rbp + subq $REGISTER_SAVE_SPACE_SIZE, %rsp + movq %rbx, -8(%rbp) + movq %rcx, -16(%rbp) + movq %rdx, -24(%rbp) + movq %rsi, -32(%rbp) + movq %rdi, -40(%rbp) + movq %r8, -48(%rbp) + movq %r9, -56(%rbp) + movq %r10, -64(%rbp) + movq %r11, -72(%rbp) + movq %r12, -80(%rbp) + movq %r13, -88(%rbp) + movq %r14, -96(%rbp) + movq %r15, -104(%rbp) + movdqa %xmm0, -128(%rbp) + movdqa %xmm1, -144(%rbp) + movdqa %xmm2, -160(%rbp) + movdqa %xmm3, -176(%rbp) + movdqa %xmm4, -192(%rbp) + movdqa %xmm5, -208(%rbp) + movdqa %xmm6, -224(%rbp) + movdqa %xmm7, -240(%rbp) + call ___orc_rt_macho_tlv_get_addr_impl + movq -8(%rbp), %rbx + movq -16(%rbp), %rcx + movq -24(%rbp), %rdx + movq -32(%rbp), %rsi + movq -40(%rbp), %rdi + movq -48(%rbp), %r8 + movq -56(%rbp), %r9 + movq -64(%rbp), %r10 + movq -72(%rbp), %r11 + movq -80(%rbp), %r12 + movq -88(%rbp), %r13 + movq -96(%rbp), %r14 + movq -104(%rbp), %r15 + movdqa -128(%rbp), %xmm0 + movdqa -144(%rbp), %xmm1 + movdqa -160(%rbp), %xmm2 + movdqa -176(%rbp), %xmm3 + movdqa -192(%rbp), %xmm4 + movdqa -208(%rbp), %xmm5 + movdqa -224(%rbp), %xmm6 + movdqa -240(%rbp), %xmm7 + addq $REGISTER_SAVE_SPACE_SIZE, %rsp + popq %rbp + ret diff --git a/compiler-rt/lib/orc/run_program_wrapper.cpp b/compiler-rt/lib/orc/run_program_wrapper.cpp new file mode 100644 index 000000000000..d0f88534aa9c --- /dev/null +++ b/compiler-rt/lib/orc/run_program_wrapper.cpp @@ -0,0 +1,51 @@ +//===- run_program_wrapper.cpp --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +#include "adt.h" +#include "common.h" +#include "wrapper_function_utils.h" + +#include <vector> + +using namespace __orc_rt; + +extern "C" int64_t __orc_rt_run_program(const char *JITDylibName, + const char *EntrySymbolName, int argc, + char *argv[]); + +ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult +__orc_rt_run_program_wrapper(const char *ArgData, size_t ArgSize) { + return WrapperFunction<int64_t(SPSString, SPSString, + SPSSequence<SPSString>)>:: + handle(ArgData, ArgSize, + [](const std::string &JITDylibName, + const std::string &EntrySymbolName, + const std::vector<string_view> &Args) { + std::vector<std::unique_ptr<char[]>> ArgVStorage; + ArgVStorage.reserve(Args.size()); + for (auto &Arg : Args) { + ArgVStorage.push_back( + std::make_unique<char[]>(Arg.size() + 1)); + memcpy(ArgVStorage.back().get(), Arg.data(), Arg.size()); + ArgVStorage.back()[Arg.size()] = '\0'; + } + std::vector<char *> ArgV; + ArgV.reserve(ArgVStorage.size() + 1); + for (auto &ArgStorage : ArgVStorage) + ArgV.push_back(ArgStorage.get()); + ArgV.push_back(nullptr); + return __orc_rt_run_program(JITDylibName.c_str(), + EntrySymbolName.c_str(), + ArgV.size() - 1, ArgV.data()); + }) + .release(); +} diff --git a/compiler-rt/lib/orc/simple_packed_serialization.h b/compiler-rt/lib/orc/simple_packed_serialization.h new file mode 100644 index 000000000000..b561a19d8f04 --- /dev/null +++ b/compiler-rt/lib/orc/simple_packed_serialization.h @@ -0,0 +1,579 @@ +//===--- simple_packed_serialization.h - simple serialization ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +// The behavior of the utilities in this header must be synchronized with the +// behavior of the utilities in +// llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h. +// +// The Simple Packed Serialization (SPS) utilities are used to generate +// argument and return buffers for wrapper functions using the following +// serialization scheme: +// +// Primitives: +// bool, char, int8_t, uint8_t -- Two's complement 8-bit (0=false, 1=true) +// int16_t, uint16_t -- Two's complement 16-bit little endian +// int32_t, uint32_t -- Two's complement 32-bit little endian +// int64_t, int64_t -- Two's complement 64-bit little endian +// +// Sequence<T>: +// Serialized as the sequence length (as a uint64_t) followed by the +// serialization of each of the elements without padding. +// +// Tuple<T1, ..., TN>: +// Serialized as each of the element types from T1 to TN without padding. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_SIMPLE_PACKED_SERIALIZATION_H +#define ORC_RT_SIMPLE_PACKED_SERIALIZATION_H + +#include "adt.h" +#include "endianness.h" +#include "error.h" +#include "stl_extras.h" + +#include <string> +#include <tuple> +#include <type_traits> +#include <unordered_map> +#include <utility> +#include <vector> + +namespace __orc_rt { + +/// Output char buffer with overflow check. +class SPSOutputBuffer { +public: + SPSOutputBuffer(char *Buffer, size_t Remaining) + : Buffer(Buffer), Remaining(Remaining) {} + bool write(const char *Data, size_t Size) { + if (Size > Remaining) + return false; + memcpy(Buffer, Data, Size); + Buffer += Size; + Remaining -= Size; + return true; + } + +private: + char *Buffer = nullptr; + size_t Remaining = 0; +}; + +/// Input char buffer with underflow check. +class SPSInputBuffer { +public: + SPSInputBuffer() = default; + SPSInputBuffer(const char *Buffer, size_t Remaining) + : Buffer(Buffer), Remaining(Remaining) {} + bool read(char *Data, size_t Size) { + if (Size > Remaining) + return false; + memcpy(Data, Buffer, Size); + Buffer += Size; + Remaining -= Size; + return true; + } + + const char *data() const { return Buffer; } + bool skip(size_t Size) { + if (Size > Remaining) + return false; + Buffer += Size; + Remaining -= Size; + return true; + } + +private: + const char *Buffer = nullptr; + size_t Remaining = 0; +}; + +/// Specialize to describe how to serialize/deserialize to/from the given +/// concrete type. +template <typename SPSTagT, typename ConcreteT, typename _ = void> +class SPSSerializationTraits; + +/// A utility class for serializing to a blob from a variadic list. +template <typename... ArgTs> class SPSArgList; + +// Empty list specialization for SPSArgList. +template <> class SPSArgList<> { +public: + static size_t size() { return 0; } + + static bool serialize(SPSOutputBuffer &OB) { return true; } + static bool deserialize(SPSInputBuffer &IB) { return true; } +}; + +// Non-empty list specialization for SPSArgList. +template <typename SPSTagT, typename... SPSTagTs> +class SPSArgList<SPSTagT, SPSTagTs...> { +public: + template <typename ArgT, typename... ArgTs> + static size_t size(const ArgT &Arg, const ArgTs &...Args) { + return SPSSerializationTraits<SPSTagT, ArgT>::size(Arg) + + SPSArgList<SPSTagTs...>::size(Args...); + } + + template <typename ArgT, typename... ArgTs> + static bool serialize(SPSOutputBuffer &OB, const ArgT &Arg, + const ArgTs &...Args) { + return SPSSerializationTraits<SPSTagT, ArgT>::serialize(OB, Arg) && + SPSArgList<SPSTagTs...>::serialize(OB, Args...); + } + + template <typename ArgT, typename... ArgTs> + static bool deserialize(SPSInputBuffer &IB, ArgT &Arg, ArgTs &...Args) { + return SPSSerializationTraits<SPSTagT, ArgT>::deserialize(IB, Arg) && + SPSArgList<SPSTagTs...>::deserialize(IB, Args...); + } +}; + +/// SPS serialization for integral types, bool, and char. +template <typename SPSTagT> +class SPSSerializationTraits< + SPSTagT, SPSTagT, + std::enable_if_t<std::is_same<SPSTagT, bool>::value || + std::is_same<SPSTagT, char>::value || + std::is_same<SPSTagT, int8_t>::value || + std::is_same<SPSTagT, int16_t>::value || + std::is_same<SPSTagT, int32_t>::value || + std::is_same<SPSTagT, int64_t>::value || + std::is_same<SPSTagT, uint8_t>::value || + std::is_same<SPSTagT, uint16_t>::value || + std::is_same<SPSTagT, uint32_t>::value || + std::is_same<SPSTagT, uint64_t>::value>> { +public: + static size_t size(const SPSTagT &Value) { return sizeof(SPSTagT); } + + static bool serialize(SPSOutputBuffer &OB, const SPSTagT &Value) { + SPSTagT Tmp = Value; + if (IsBigEndianHost) + swapByteOrder(Tmp); + return OB.write(reinterpret_cast<const char *>(&Tmp), sizeof(Tmp)); + } + + static bool deserialize(SPSInputBuffer &IB, SPSTagT &Value) { + SPSTagT Tmp; + if (!IB.read(reinterpret_cast<char *>(&Tmp), sizeof(Tmp))) + return false; + if (IsBigEndianHost) + swapByteOrder(Tmp); + Value = Tmp; + return true; + } +}; + +/// Any empty placeholder suitable as a substitute for void when deserializing +class SPSEmpty {}; + +/// Represents an address in the executor. +class SPSExecutorAddress {}; + +/// SPS tag type for tuples. +/// +/// A blob tuple should be serialized by serializing each of the elements in +/// sequence. +template <typename... SPSTagTs> class SPSTuple { +public: + /// Convenience typedef of the corresponding arg list. + typedef SPSArgList<SPSTagTs...> AsArgList; +}; + +/// SPS tag type for sequences. +/// +/// SPSSequences should be serialized as a uint64_t sequence length, +/// followed by the serialization of each of the elements. +template <typename SPSElementTagT> class SPSSequence; + +/// SPS tag type for strings, which are equivalent to sequences of chars. +using SPSString = SPSSequence<char>; + +/// SPS tag type for maps. +/// +/// SPS maps are just sequences of (Key, Value) tuples. +template <typename SPSTagT1, typename SPSTagT2> +using SPSMap = SPSSequence<SPSTuple<SPSTagT1, SPSTagT2>>; + +/// Serialization for SPSEmpty type. +template <> class SPSSerializationTraits<SPSEmpty, SPSEmpty> { +public: + static size_t size(const SPSEmpty &EP) { return 0; } + static bool serialize(SPSOutputBuffer &OB, const SPSEmpty &BE) { + return true; + } + static bool deserialize(SPSInputBuffer &IB, SPSEmpty &BE) { return true; } +}; + +/// Specialize this to implement 'trivial' sequence serialization for +/// a concrete sequence type. +/// +/// Trivial sequence serialization uses the sequence's 'size' member to get the +/// length of the sequence, and uses a range-based for loop to iterate over the +/// elements. +/// +/// Specializing this template class means that you do not need to provide a +/// specialization of SPSSerializationTraits for your type. +template <typename SPSElementTagT, typename ConcreteSequenceT> +class TrivialSPSSequenceSerialization { +public: + static constexpr bool available = false; +}; + +/// Specialize this to implement 'trivial' sequence deserialization for +/// a concrete sequence type. +/// +/// Trivial deserialization calls a static 'reserve(SequenceT&)' method on your +/// specialization (you must implement this) to reserve space, and then calls +/// a static 'append(SequenceT&, ElementT&) method to append each of the +/// deserialized elements. +/// +/// Specializing this template class means that you do not need to provide a +/// specialization of SPSSerializationTraits for your type. +template <typename SPSElementTagT, typename ConcreteSequenceT> +class TrivialSPSSequenceDeserialization { +public: + static constexpr bool available = false; +}; + +/// Trivial std::string -> SPSSequence<char> serialization. +template <> class TrivialSPSSequenceSerialization<char, std::string> { +public: + static constexpr bool available = true; +}; + +/// Trivial SPSSequence<char> -> std::string deserialization. +template <> class TrivialSPSSequenceDeserialization<char, std::string> { +public: + static constexpr bool available = true; + + using element_type = char; + + static void reserve(std::string &S, uint64_t Size) { S.reserve(Size); } + static bool append(std::string &S, char C) { + S.push_back(C); + return true; + } +}; + +/// Trivial std::vector<T> -> SPSSequence<SPSElementTagT> serialization. +template <typename SPSElementTagT, typename T> +class TrivialSPSSequenceSerialization<SPSElementTagT, std::vector<T>> { +public: + static constexpr bool available = true; +}; + +/// Trivial SPSSequence<SPSElementTagT> -> std::vector<T> deserialization. +template <typename SPSElementTagT, typename T> +class TrivialSPSSequenceDeserialization<SPSElementTagT, std::vector<T>> { +public: + static constexpr bool available = true; + + using element_type = typename std::vector<T>::value_type; + + static void reserve(std::vector<T> &V, uint64_t Size) { V.reserve(Size); } + static bool append(std::vector<T> &V, T E) { + V.push_back(std::move(E)); + return true; + } +}; + +/// Trivial std::unordered_map<K, V> -> SPSSequence<SPSTuple<SPSKey, SPSValue>> +/// serialization. +template <typename SPSKeyTagT, typename SPSValueTagT, typename K, typename V> +class TrivialSPSSequenceSerialization<SPSTuple<SPSKeyTagT, SPSValueTagT>, + std::unordered_map<K, V>> { +public: + static constexpr bool available = true; +}; + +/// Trivial SPSSequence<SPSTuple<SPSKey, SPSValue>> -> std::unordered_map<K, V> +/// deserialization. +template <typename SPSKeyTagT, typename SPSValueTagT, typename K, typename V> +class TrivialSPSSequenceDeserialization<SPSTuple<SPSKeyTagT, SPSValueTagT>, + std::unordered_map<K, V>> { +public: + static constexpr bool available = true; + + using element_type = std::pair<K, V>; + + static void reserve(std::unordered_map<K, V> &M, uint64_t Size) { + M.reserve(Size); + } + static bool append(std::unordered_map<K, V> &M, element_type E) { + return M.insert(std::move(E)).second; + } +}; + +/// 'Trivial' sequence serialization: Sequence is serialized as a uint64_t size +/// followed by a for-earch loop over the elements of the sequence to serialize +/// each of them. +template <typename SPSElementTagT, typename SequenceT> +class SPSSerializationTraits<SPSSequence<SPSElementTagT>, SequenceT, + std::enable_if_t<TrivialSPSSequenceSerialization< + SPSElementTagT, SequenceT>::available>> { +public: + static size_t size(const SequenceT &S) { + size_t Size = SPSArgList<uint64_t>::size(static_cast<uint64_t>(S.size())); + for (const auto &E : S) + Size += SPSArgList<SPSElementTagT>::size(E); + return Size; + } + + static bool serialize(SPSOutputBuffer &OB, const SequenceT &S) { + if (!SPSArgList<uint64_t>::serialize(OB, static_cast<uint64_t>(S.size()))) + return false; + for (const auto &E : S) + if (!SPSArgList<SPSElementTagT>::serialize(OB, E)) + return false; + return true; + } + + static bool deserialize(SPSInputBuffer &IB, SequenceT &S) { + using TBSD = TrivialSPSSequenceDeserialization<SPSElementTagT, SequenceT>; + uint64_t Size; + if (!SPSArgList<uint64_t>::deserialize(IB, Size)) + return false; + TBSD::reserve(S, Size); + for (size_t I = 0; I != Size; ++I) { + typename TBSD::element_type E; + if (!SPSArgList<SPSElementTagT>::deserialize(IB, E)) + return false; + if (!TBSD::append(S, std::move(E))) + return false; + } + return true; + } +}; + +/// SPSTuple serialization for std::pair. +template <typename SPSTagT1, typename SPSTagT2, typename T1, typename T2> +class SPSSerializationTraits<SPSTuple<SPSTagT1, SPSTagT2>, std::pair<T1, T2>> { +public: + static size_t size(const std::pair<T1, T2> &P) { + return SPSArgList<SPSTagT1>::size(P.first) + + SPSArgList<SPSTagT2>::size(P.second); + } + + static bool serialize(SPSOutputBuffer &OB, const std::pair<T1, T2> &P) { + return SPSArgList<SPSTagT1>::serialize(OB, P.first) && + SPSArgList<SPSTagT2>::serialize(OB, P.second); + } + + static bool deserialize(SPSInputBuffer &IB, std::pair<T1, T2> &P) { + return SPSArgList<SPSTagT1>::deserialize(IB, P.first) && + SPSArgList<SPSTagT2>::deserialize(IB, P.second); + } +}; + +/// Serialization for string_views. +/// +/// Serialization is as for regular strings. Deserialization points directly +/// into the blob. +template <> class SPSSerializationTraits<SPSString, __orc_rt::string_view> { +public: + static size_t size(const __orc_rt::string_view &S) { + return SPSArgList<uint64_t>::size(static_cast<uint64_t>(S.size())) + + S.size(); + } + + static bool serialize(SPSOutputBuffer &OB, const __orc_rt::string_view &S) { + if (!SPSArgList<uint64_t>::serialize(OB, static_cast<uint64_t>(S.size()))) + return false; + return OB.write(S.data(), S.size()); + } + + static bool deserialize(SPSInputBuffer &IB, __orc_rt::string_view &S) { + const char *Data = nullptr; + uint64_t Size; + if (!SPSArgList<uint64_t>::deserialize(IB, Size)) + return false; + Data = IB.data(); + if (!IB.skip(Size)) + return false; + S = {Data, Size}; + return true; + } +}; + +/// SPS tag type for errors. +class SPSError; + +/// SPS tag type for expecteds, which are either a T or a string representing +/// an error. +template <typename SPSTagT> class SPSExpected; + +namespace detail { + +/// Helper type for serializing Errors. +/// +/// llvm::Errors are move-only, and not inspectable except by consuming them. +/// This makes them unsuitable for direct serialization via +/// SPSSerializationTraits, which needs to inspect values twice (once to +/// determine the amount of space to reserve, and then again to serialize). +/// +/// The SPSSerializableError type is a helper that can be +/// constructed from an llvm::Error, but inspected more than once. +struct SPSSerializableError { + bool HasError = false; + std::string ErrMsg; +}; + +/// Helper type for serializing Expected<T>s. +/// +/// See SPSSerializableError for more details. +/// +// FIXME: Use std::variant for storage once we have c++17. +template <typename T> struct SPSSerializableExpected { + bool HasValue = false; + T Value{}; + std::string ErrMsg; +}; + +inline SPSSerializableError toSPSSerializable(Error Err) { + if (Err) + return {true, toString(std::move(Err))}; + return {false, {}}; +} + +inline Error fromSPSSerializable(SPSSerializableError BSE) { + if (BSE.HasError) + return make_error<StringError>(BSE.ErrMsg); + return Error::success(); +} + +template <typename T> +SPSSerializableExpected<T> toSPSSerializable(Expected<T> E) { + if (E) + return {true, std::move(*E), {}}; + else + return {false, {}, toString(E.takeError())}; +} + +template <typename T> +Expected<T> fromSPSSerializable(SPSSerializableExpected<T> BSE) { + if (BSE.HasValue) + return std::move(BSE.Value); + else + return make_error<StringError>(BSE.ErrMsg); +} + +} // end namespace detail + +/// Serialize to a SPSError from a detail::SPSSerializableError. +template <> +class SPSSerializationTraits<SPSError, detail::SPSSerializableError> { +public: + static size_t size(const detail::SPSSerializableError &BSE) { + size_t Size = SPSArgList<bool>::size(BSE.HasError); + if (BSE.HasError) + Size += SPSArgList<SPSString>::size(BSE.ErrMsg); + return Size; + } + + static bool serialize(SPSOutputBuffer &OB, + const detail::SPSSerializableError &BSE) { + if (!SPSArgList<bool>::serialize(OB, BSE.HasError)) + return false; + if (BSE.HasError) + if (!SPSArgList<SPSString>::serialize(OB, BSE.ErrMsg)) + return false; + return true; + } + + static bool deserialize(SPSInputBuffer &IB, + detail::SPSSerializableError &BSE) { + if (!SPSArgList<bool>::deserialize(IB, BSE.HasError)) + return false; + + if (!BSE.HasError) + return true; + + return SPSArgList<SPSString>::deserialize(IB, BSE.ErrMsg); + } +}; + +/// Serialize to a SPSExpected<SPSTagT> from a +/// detail::SPSSerializableExpected<T>. +template <typename SPSTagT, typename T> +class SPSSerializationTraits<SPSExpected<SPSTagT>, + detail::SPSSerializableExpected<T>> { +public: + static size_t size(const detail::SPSSerializableExpected<T> &BSE) { + size_t Size = SPSArgList<bool>::size(BSE.HasValue); + if (BSE.HasValue) + Size += SPSArgList<SPSTagT>::size(BSE.Value); + else + Size += SPSArgList<SPSString>::size(BSE.ErrMsg); + return Size; + } + + static bool serialize(SPSOutputBuffer &OB, + const detail::SPSSerializableExpected<T> &BSE) { + if (!SPSArgList<bool>::serialize(OB, BSE.HasValue)) + return false; + + if (BSE.HasValue) + return SPSArgList<SPSTagT>::serialize(OB, BSE.Value); + + return SPSArgList<SPSString>::serialize(OB, BSE.ErrMsg); + } + + static bool deserialize(SPSInputBuffer &IB, + detail::SPSSerializableExpected<T> &BSE) { + if (!SPSArgList<bool>::deserialize(IB, BSE.HasValue)) + return false; + + if (BSE.HasValue) + return SPSArgList<SPSTagT>::deserialize(IB, BSE.Value); + + return SPSArgList<SPSString>::deserialize(IB, BSE.ErrMsg); + } +}; + +/// Serialize to a SPSExpected<SPSTagT> from a detail::SPSSerializableError. +template <typename SPSTagT> +class SPSSerializationTraits<SPSExpected<SPSTagT>, + detail::SPSSerializableError> { +public: + static size_t size(const detail::SPSSerializableError &BSE) { + assert(BSE.HasError && "Cannot serialize expected from a success value"); + return SPSArgList<bool>::size(false) + + SPSArgList<SPSString>::size(BSE.ErrMsg); + } + + static bool serialize(SPSOutputBuffer &OB, + const detail::SPSSerializableError &BSE) { + assert(BSE.HasError && "Cannot serialize expected from a success value"); + if (!SPSArgList<bool>::serialize(OB, false)) + return false; + return SPSArgList<SPSString>::serialize(OB, BSE.ErrMsg); + } +}; + +/// Serialize to a SPSExpected<SPSTagT> from a T. +template <typename SPSTagT, typename T> +class SPSSerializationTraits<SPSExpected<SPSTagT>, T> { +public: + static size_t size(const T &Value) { + return SPSArgList<bool>::size(true) + SPSArgList<SPSTagT>::size(Value); + } + + static bool serialize(SPSOutputBuffer &OB, const T &Value) { + if (!SPSArgList<bool>::serialize(OB, true)) + return false; + return SPSArgList<SPSTagT>::serialize(Value); + } +}; + +} // end namespace __orc_rt + +#endif // ORC_RT_SIMPLE_PACKED_SERIALIZATION_H diff --git a/compiler-rt/lib/orc/stl_extras.h b/compiler-rt/lib/orc/stl_extras.h new file mode 100644 index 000000000000..ad7286e87ae3 --- /dev/null +++ b/compiler-rt/lib/orc/stl_extras.h @@ -0,0 +1,46 @@ +//===-------- stl_extras.h - Useful STL related functions-------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_STL_EXTRAS_H +#define ORC_RT_STL_EXTRAS_H + +#include <utility> +#include <tuple> + +namespace __orc_rt { + +namespace detail { + +template <typename F, typename Tuple, std::size_t... I> +decltype(auto) apply_tuple_impl(F &&f, Tuple &&t, std::index_sequence<I...>) { + return std::forward<F>(f)(std::get<I>(std::forward<Tuple>(t))...); +} + +} // end namespace detail + +/// Given an input tuple (a1, a2, ..., an), pass the arguments of the +/// tuple variadically to f as if by calling f(a1, a2, ..., an) and +/// return the result. +/// +/// FIXME: Switch to std::apply once we can use c++17. +template <typename F, typename Tuple> +decltype(auto) apply_tuple(F &&f, Tuple &&t) { + using Indices = std::make_index_sequence< + std::tuple_size<typename std::decay<Tuple>::type>::value>; + + return detail::apply_tuple_impl(std::forward<F>(f), std::forward<Tuple>(t), + Indices{}); +} + +} // namespace __orc_rt + +#endif // ORC_RT_STL_EXTRAS diff --git a/compiler-rt/lib/orc/wrapper_function_utils.h b/compiler-rt/lib/orc/wrapper_function_utils.h new file mode 100644 index 000000000000..49faa03e5eb8 --- /dev/null +++ b/compiler-rt/lib/orc/wrapper_function_utils.h @@ -0,0 +1,367 @@ +//===-- wrapper_function_utils.h - Utilities for wrapper funcs --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_WRAPPER_FUNCTION_UTILS_H +#define ORC_RT_WRAPPER_FUNCTION_UTILS_H + +#include "c_api.h" +#include "common.h" +#include "error.h" +#include "simple_packed_serialization.h" +#include <type_traits> + +namespace __orc_rt { + +/// C++ wrapper function result: Same as CWrapperFunctionResult but +/// auto-releases memory. +class WrapperFunctionResult { +public: + /// Create a default WrapperFunctionResult. + WrapperFunctionResult() { __orc_rt_CWrapperFunctionResultInit(&R); } + + /// Create a WrapperFunctionResult from a CWrapperFunctionResult. This + /// instance takes ownership of the result object and will automatically + /// call dispose on the result upon destruction. + WrapperFunctionResult(__orc_rt_CWrapperFunctionResult R) : R(R) {} + + WrapperFunctionResult(const WrapperFunctionResult &) = delete; + WrapperFunctionResult &operator=(const WrapperFunctionResult &) = delete; + + WrapperFunctionResult(WrapperFunctionResult &&Other) { + __orc_rt_CWrapperFunctionResultInit(&R); + std::swap(R, Other.R); + } + + WrapperFunctionResult &operator=(WrapperFunctionResult &&Other) { + __orc_rt_CWrapperFunctionResult Tmp; + __orc_rt_CWrapperFunctionResultInit(&Tmp); + std::swap(Tmp, Other.R); + std::swap(R, Tmp); + return *this; + } + + ~WrapperFunctionResult() { __orc_rt_DisposeCWrapperFunctionResult(&R); } + + /// Relinquish ownership of and return the + /// __orc_rt_CWrapperFunctionResult. + __orc_rt_CWrapperFunctionResult release() { + __orc_rt_CWrapperFunctionResult Tmp; + __orc_rt_CWrapperFunctionResultInit(&Tmp); + std::swap(R, Tmp); + return Tmp; + } + + /// Get a pointer to the data contained in this instance. + const char *data() const { return __orc_rt_CWrapperFunctionResultData(&R); } + + /// Returns the size of the data contained in this instance. + size_t size() const { return __orc_rt_CWrapperFunctionResultSize(&R); } + + /// Returns true if this value is equivalent to a default-constructed + /// WrapperFunctionResult. + bool empty() const { return __orc_rt_CWrapperFunctionResultEmpty(&R); } + + /// Create a WrapperFunctionResult with the given size and return a pointer + /// to the underlying memory. + static char *allocate(WrapperFunctionResult &R, size_t Size) { + __orc_rt_DisposeCWrapperFunctionResult(&R.R); + __orc_rt_CWrapperFunctionResultInit(&R.R); + return __orc_rt_CWrapperFunctionResultAllocate(&R.R, Size); + } + + /// Copy from the given char range. + static WrapperFunctionResult copyFrom(const char *Source, size_t Size) { + return __orc_rt_CreateCWrapperFunctionResultFromRange(Source, Size); + } + + /// Copy from the given null-terminated string (includes the null-terminator). + static WrapperFunctionResult copyFrom(const char *Source) { + return __orc_rt_CreateCWrapperFunctionResultFromString(Source); + } + + /// Copy from the given std::string (includes the null terminator). + static WrapperFunctionResult copyFrom(const std::string &Source) { + return copyFrom(Source.c_str()); + } + + /// Create an out-of-band error by copying the given string. + static WrapperFunctionResult createOutOfBandError(const char *Msg) { + return __orc_rt_CreateCWrapperFunctionResultFromOutOfBandError(Msg); + } + + /// Create an out-of-band error by copying the given string. + static WrapperFunctionResult createOutOfBandError(const std::string &Msg) { + return createOutOfBandError(Msg.c_str()); + } + + /// If this value is an out-of-band error then this returns the error message, + /// otherwise returns nullptr. + const char *getOutOfBandError() const { + return __orc_rt_CWrapperFunctionResultGetOutOfBandError(&R); + } + +private: + __orc_rt_CWrapperFunctionResult R; +}; + +namespace detail { + +template <typename SPSArgListT, typename... ArgTs> +Expected<WrapperFunctionResult> +serializeViaSPSToWrapperFunctionResult(const ArgTs &...Args) { + WrapperFunctionResult Result; + char *DataPtr = + WrapperFunctionResult::allocate(Result, SPSArgListT::size(Args...)); + SPSOutputBuffer OB(DataPtr, Result.size()); + if (!SPSArgListT::serialize(OB, Args...)) + return make_error<StringError>( + "Error serializing arguments to blob in call"); + return std::move(Result); +} + +template <typename RetT> class WrapperFunctionHandlerCaller { +public: + template <typename HandlerT, typename ArgTupleT, std::size_t... I> + static decltype(auto) call(HandlerT &&H, ArgTupleT &Args, + std::index_sequence<I...>) { + return std::forward<HandlerT>(H)(std::get<I>(Args)...); + } +}; + +template <> class WrapperFunctionHandlerCaller<void> { +public: + template <typename HandlerT, typename ArgTupleT, std::size_t... I> + static SPSEmpty call(HandlerT &&H, ArgTupleT &Args, + std::index_sequence<I...>) { + std::forward<HandlerT>(H)(std::get<I>(Args)...); + return SPSEmpty(); + } +}; + +template <typename WrapperFunctionImplT, + template <typename> class ResultSerializer, typename... SPSTagTs> +class WrapperFunctionHandlerHelper + : public WrapperFunctionHandlerHelper< + decltype(&std::remove_reference_t<WrapperFunctionImplT>::operator()), + ResultSerializer, SPSTagTs...> {}; + +template <typename RetT, typename... ArgTs, + template <typename> class ResultSerializer, typename... SPSTagTs> +class WrapperFunctionHandlerHelper<RetT(ArgTs...), ResultSerializer, + SPSTagTs...> { +public: + using ArgTuple = std::tuple<std::decay_t<ArgTs>...>; + using ArgIndices = std::make_index_sequence<std::tuple_size<ArgTuple>::value>; + + template <typename HandlerT> + static WrapperFunctionResult apply(HandlerT &&H, const char *ArgData, + size_t ArgSize) { + ArgTuple Args; + if (!deserialize(ArgData, ArgSize, Args, ArgIndices{})) + return WrapperFunctionResult::createOutOfBandError( + "Could not deserialize arguments for wrapper function call"); + + auto HandlerResult = WrapperFunctionHandlerCaller<RetT>::call( + std::forward<HandlerT>(H), Args, ArgIndices{}); + + if (auto Result = ResultSerializer<decltype(HandlerResult)>::serialize( + std::move(HandlerResult))) + return std::move(*Result); + else + return WrapperFunctionResult::createOutOfBandError( + toString(Result.takeError())); + } + +private: + template <std::size_t... I> + static bool deserialize(const char *ArgData, size_t ArgSize, ArgTuple &Args, + std::index_sequence<I...>) { + SPSInputBuffer IB(ArgData, ArgSize); + return SPSArgList<SPSTagTs...>::deserialize(IB, std::get<I>(Args)...); + } + +}; + +// Map function references to function types. +template <typename RetT, typename... ArgTs, + template <typename> class ResultSerializer, typename... SPSTagTs> +class WrapperFunctionHandlerHelper<RetT (&)(ArgTs...), ResultSerializer, + SPSTagTs...> + : public WrapperFunctionHandlerHelper<RetT(ArgTs...), ResultSerializer, + SPSTagTs...> {}; + +// Map non-const member function types to function types. +template <typename ClassT, typename RetT, typename... ArgTs, + template <typename> class ResultSerializer, typename... SPSTagTs> +class WrapperFunctionHandlerHelper<RetT (ClassT::*)(ArgTs...), ResultSerializer, + SPSTagTs...> + : public WrapperFunctionHandlerHelper<RetT(ArgTs...), ResultSerializer, + SPSTagTs...> {}; + +// Map const member function types to function types. +template <typename ClassT, typename RetT, typename... ArgTs, + template <typename> class ResultSerializer, typename... SPSTagTs> +class WrapperFunctionHandlerHelper<RetT (ClassT::*)(ArgTs...) const, + ResultSerializer, SPSTagTs...> + : public WrapperFunctionHandlerHelper<RetT(ArgTs...), ResultSerializer, + SPSTagTs...> {}; + +template <typename SPSRetTagT, typename RetT> class ResultSerializer { +public: + static Expected<WrapperFunctionResult> serialize(RetT Result) { + return serializeViaSPSToWrapperFunctionResult<SPSArgList<SPSRetTagT>>( + Result); + } +}; + +template <typename SPSRetTagT> class ResultSerializer<SPSRetTagT, Error> { +public: + static Expected<WrapperFunctionResult> serialize(Error Err) { + return serializeViaSPSToWrapperFunctionResult<SPSArgList<SPSRetTagT>>( + toSPSSerializable(std::move(Err))); + } +}; + +template <typename SPSRetTagT, typename T> +class ResultSerializer<SPSRetTagT, Expected<T>> { +public: + static Expected<WrapperFunctionResult> serialize(Expected<T> E) { + return serializeViaSPSToWrapperFunctionResult<SPSArgList<SPSRetTagT>>( + toSPSSerializable(std::move(E))); + } +}; + +template <typename SPSRetTagT, typename RetT> class ResultDeserializer { +public: + static void makeSafe(RetT &Result) {} + + static Error deserialize(RetT &Result, const char *ArgData, size_t ArgSize) { + SPSInputBuffer IB(ArgData, ArgSize); + if (!SPSArgList<SPSRetTagT>::deserialize(IB, Result)) + return make_error<StringError>( + "Error deserializing return value from blob in call"); + return Error::success(); + } +}; + +template <> class ResultDeserializer<SPSError, Error> { +public: + static void makeSafe(Error &Err) { cantFail(std::move(Err)); } + + static Error deserialize(Error &Err, const char *ArgData, size_t ArgSize) { + SPSInputBuffer IB(ArgData, ArgSize); + SPSSerializableError BSE; + if (!SPSArgList<SPSError>::deserialize(IB, BSE)) + return make_error<StringError>( + "Error deserializing return value from blob in call"); + Err = fromSPSSerializable(std::move(BSE)); + return Error::success(); + } +}; + +template <typename SPSTagT, typename T> +class ResultDeserializer<SPSExpected<SPSTagT>, Expected<T>> { +public: + static void makeSafe(Expected<T> &E) { cantFail(E.takeError()); } + + static Error deserialize(Expected<T> &E, const char *ArgData, + size_t ArgSize) { + SPSInputBuffer IB(ArgData, ArgSize); + SPSSerializableExpected<T> BSE; + if (!SPSArgList<SPSExpected<SPSTagT>>::deserialize(IB, BSE)) + return make_error<StringError>( + "Error deserializing return value from blob in call"); + E = fromSPSSerializable(std::move(BSE)); + return Error::success(); + } +}; + +} // end namespace detail + +template <typename SPSSignature> class WrapperFunction; + +template <typename SPSRetTagT, typename... SPSTagTs> +class WrapperFunction<SPSRetTagT(SPSTagTs...)> { +private: + template <typename RetT> + using ResultSerializer = detail::ResultSerializer<SPSRetTagT, RetT>; + +public: + template <typename RetT, typename... ArgTs> + static Error call(const void *FnTag, RetT &Result, const ArgTs &...Args) { + + // RetT might be an Error or Expected value. Set the checked flag now: + // we don't want the user to have to check the unused result if this + // operation fails. + detail::ResultDeserializer<SPSRetTagT, RetT>::makeSafe(Result); + + if (ORC_RT_UNLIKELY(!&__orc_rt_jit_dispatch_ctx)) + return make_error<StringError>("__orc_rt_jit_dispatch_ctx not set"); + if (ORC_RT_UNLIKELY(!&__orc_rt_jit_dispatch)) + return make_error<StringError>("__orc_rt_jit_dispatch not set"); + + auto ArgBuffer = + detail::serializeViaSPSToWrapperFunctionResult<SPSArgList<SPSTagTs...>>( + Args...); + if (!ArgBuffer) + return ArgBuffer.takeError(); + + WrapperFunctionResult ResultBuffer = + __orc_rt_jit_dispatch(&__orc_rt_jit_dispatch_ctx, FnTag, + ArgBuffer->data(), ArgBuffer->size()); + if (auto ErrMsg = ResultBuffer.getOutOfBandError()) + return make_error<StringError>(ErrMsg); + + return detail::ResultDeserializer<SPSRetTagT, RetT>::deserialize( + Result, ResultBuffer.data(), ResultBuffer.size()); + } + + template <typename HandlerT> + static WrapperFunctionResult handle(const char *ArgData, size_t ArgSize, + HandlerT &&Handler) { + using WFHH = + detail::WrapperFunctionHandlerHelper<HandlerT, ResultSerializer, + SPSTagTs...>; + return WFHH::apply(std::forward<HandlerT>(Handler), ArgData, ArgSize); + } + +private: + template <typename T> static const T &makeSerializable(const T &Value) { + return Value; + } + + static detail::SPSSerializableError makeSerializable(Error Err) { + return detail::toSPSSerializable(std::move(Err)); + } + + template <typename T> + static detail::SPSSerializableExpected<T> makeSerializable(Expected<T> E) { + return detail::toSPSSerializable(std::move(E)); + } +}; + +template <typename... SPSTagTs> +class WrapperFunction<void(SPSTagTs...)> + : private WrapperFunction<SPSEmpty(SPSTagTs...)> { +public: + template <typename... ArgTs> + static Error call(const void *FnTag, const ArgTs &...Args) { + SPSEmpty BE; + return WrapperFunction<SPSEmpty(SPSTagTs...)>::call(FnTag, BE, Args...); + } + + using WrapperFunction<SPSEmpty(SPSTagTs...)>::handle; +}; + +} // end namespace __orc_rt + +#endif // ORC_RT_WRAPPER_FUNCTION_UTILS_H |