diff options
Diffstat (limited to '3rdparty/asmjit/src/asmjit/core/virtmem.cpp')
-rw-r--r-- | 3rdparty/asmjit/src/asmjit/core/virtmem.cpp | 1179 |
1 files changed, 898 insertions, 281 deletions
diff --git a/3rdparty/asmjit/src/asmjit/core/virtmem.cpp b/3rdparty/asmjit/src/asmjit/core/virtmem.cpp index 0606748ef7f..743847746af 100644 --- a/3rdparty/asmjit/src/asmjit/core/virtmem.cpp +++ b/3rdparty/asmjit/src/asmjit/core/virtmem.cpp @@ -1,30 +1,12 @@ -// AsmJit - Machine code generation for C++ +// This file is part of AsmJit project <https://asmjit.com> // -// * Official AsmJit Home Page: https://asmjit.com -// * Official Github Repository: https://github.com/asmjit/asmjit -// -// Copyright (c) 2008-2020 The AsmJit Authors -// -// This software is provided 'as-is', without any express or implied -// warranty. In no event will the authors be held liable for any damages -// arising from the use of this software. -// -// Permission is granted to anyone to use this software for any purpose, -// including commercial applications, and to alter it and redistribute it -// freely, subject to the following restrictions: -// -// 1. The origin of this software must not be misrepresented; you must not -// claim that you wrote the original software. If you use this software -// in a product, an acknowledgment in the product documentation would be -// appreciated but is not required. -// 2. Altered source versions must be plainly marked as such, and must not be -// misrepresented as being the original software. -// 3. This notice may not be removed or altered from any source distribution. +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" #ifndef ASMJIT_NO_JIT -#include "../core/osutils.h" +#include "../core/osutils_p.h" #include "../core/string.h" #include "../core/support.h" #include "../core/virtmem.h" @@ -37,16 +19,51 @@ #include <sys/types.h> #include <unistd.h> + #if !ASMJIT_ARCH_X86 + #include <sys/time.h> // required by gettimeofday() + #endif + // Linux has a `memfd_create` syscall that we would like to use, if available. #if defined(__linux__) #include <sys/syscall.h> + #include <sys/utsname.h> + + #ifndef MAP_HUGETLB + #define MAP_HUGETLB 0x40000 + #endif // MAP_HUGETLB + + #ifndef MAP_HUGE_SHIFT + #define MAP_HUGE_SHIFT 26 + #endif // MAP_HUGE_SHIFT + + #if !defined(MFD_CLOEXEC) + #define MFD_CLOEXEC 0x0001u + #endif // MFD_CLOEXEC + + #if !defined(MFD_NOEXEC_SEAL) + #define MFD_NOEXEC_SEAL 0x0008u + #endif // MFD_NOEXEC_SEAL + + #if !defined(MFD_EXEC) + #define MFD_EXEC 0x0010u + #endif // MFD_EXEC + + #ifndef MFD_HUGETLB + #define MFD_HUGETLB 0x0004 + #endif // MFD_HUGETLB + + #ifndef MFD_HUGE_SHIFT + #define MFD_HUGE_SHIFT 26 + #endif // MFD_HUGE_SHIFT #endif // Apple recently introduced MAP_JIT flag, which we want to use. #if defined(__APPLE__) + #include <pthread.h> #include <TargetConditionals.h> #if TARGET_OS_OSX #include <sys/utsname.h> + #include <libkern/OSCacheControl.h> // sys_icache_invalidate(). #endif // Older SDK doesn't define `MAP_JIT`. #ifndef MAP_JIT @@ -54,36 +71,92 @@ #endif #endif - // BSD/OSX: `MAP_ANONYMOUS` is not defined, `MAP_ANON` is. + // BSD/MAC: `MAP_ANONYMOUS` is not defined, `MAP_ANON` is. #if !defined(MAP_ANONYMOUS) #define MAP_ANONYMOUS MAP_ANON #endif + + // Android NDK doesn't provide `shm_open()` and `shm_unlink()`. + #if !defined(__BIONIC__) && !defined(ASMJIT_NO_SHM_OPEN) + #define ASMJIT_HAS_SHM_OPEN + #endif + + #if defined(__APPLE__) || defined(__BIONIC__) || !defined(ASMJIT_HAS_SHM_OPEN) + #define ASMJIT_VM_SHM_DETECT 0 + #else + #define ASMJIT_VM_SHM_DETECT 1 + #endif + + #if defined(__APPLE__) && TARGET_OS_OSX + #if ASMJIT_ARCH_X86 != 0 + #define ASMJIT_ANONYMOUS_MEMORY_USE_MACH_VM_REMAP + #endif + #if ASMJIT_ARCH_ARM >= 64 + #define ASMJIT_HAS_PTHREAD_JIT_WRITE_PROTECT_NP + #endif + #endif + + #if defined(__APPLE__) && ASMJIT_ARCH_X86 == 0 + #define ASMJIT_NO_DUAL_MAPPING + #endif + + #if defined(__NetBSD__) && defined(MAP_REMAPDUP) && defined(PROT_MPROTECT) + #define ASMJIT_ANONYMOUS_MEMORY_USE_REMAPDUP + #endif + + #if !defined(ASMJIT_ANONYMOUS_MEMORY_USE_REMAPDUP) && \ + !defined(ASMJIT_ANONYMOUS_MEMORY_USE_MACH_VM_REMAP) && \ + !defined(ASMJIT_NO_DUAL_MAPPING) + #define ASMJIT_ANONYMOUS_MEMORY_USE_FD + #endif #endif #include <atomic> -#if defined(__APPLE__) - #define ASMJIT_VM_SHM_DETECT 0 +#if defined(ASMJIT_ANONYMOUS_MEMORY_USE_MACH_VM_REMAP) +#include <mach/mach.h> +#include <mach/mach_time.h> + +extern "C" { + +#ifdef mig_external +mig_external #else - #define ASMJIT_VM_SHM_DETECT 1 +extern +#endif +kern_return_t mach_vm_remap( + vm_map_t target_task, + mach_vm_address_t *target_address, + mach_vm_size_t size, + mach_vm_offset_t mask, + int flags, + vm_map_t src_task, + mach_vm_address_t src_address, + boolean_t copy, + vm_prot_t *cur_protection, + vm_prot_t *max_protection, + vm_inherit_t inheritance +); + +} // {extern "C"} #endif -ASMJIT_BEGIN_NAMESPACE +ASMJIT_BEGIN_SUB_NAMESPACE(VirtMem) -// ============================================================================ -// [asmjit::VirtMem - Utilities] -// ============================================================================ +// Virtual Memory Utilities +// ======================== -static const uint32_t VirtMem_dualMappingFilter[2] = { - VirtMem::kAccessWrite, - VirtMem::kAccessExecute +ASMJIT_MAYBE_UNUSED +static const constexpr MemoryFlags dualMappingFilter[2] = { + MemoryFlags::kAccessWrite | MemoryFlags::kMMapMaxAccessWrite, + MemoryFlags::kAccessExecute | MemoryFlags::kMMapMaxAccessExecute }; -// ============================================================================ -// [asmjit::VirtMem - Virtual Memory [Windows]] -// ============================================================================ +// Virtual Memory [Windows] +// ======================== #if defined(_WIN32) + struct ScopedHandle { inline ScopedHandle() noexcept : value(nullptr) {} @@ -96,7 +169,7 @@ struct ScopedHandle { HANDLE value; }; -static void VirtMem_getInfo(VirtMem::Info& vmInfo) noexcept { +static void detectVMInfo(Info& vmInfo) noexcept { SYSTEM_INFO systemInfo; ::GetSystemInfo(&systemInfo); @@ -104,15 +177,24 @@ static void VirtMem_getInfo(VirtMem::Info& vmInfo) noexcept { vmInfo.pageGranularity = systemInfo.dwAllocationGranularity; } -// Windows specific implementation that uses `VirtualAlloc` and `VirtualFree`. -static DWORD VirtMem_accessToWinProtectFlags(uint32_t flags) noexcept { +static size_t detectLargePageSize() noexcept { + return ::GetLargePageMinimum(); +} + +static bool hasDualMappingSupport() noexcept { + // TODO: This assumption works on X86 platforms, this may not work on AArch64. + return true; +} + +// Returns windows-specific protectFlags from \ref MemoryFlags. +static DWORD protectFlagsFromMemoryFlags(MemoryFlags memoryFlags) noexcept { DWORD protectFlags; // READ|WRITE|EXECUTE. - if (flags & VirtMem::kAccessExecute) - protectFlags = (flags & VirtMem::kAccessWrite) ? PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ; - else if (flags & VirtMem::kAccessReadWrite) - protectFlags = (flags & VirtMem::kAccessWrite) ? PAGE_READWRITE : PAGE_READONLY; + if (Support::test(memoryFlags, MemoryFlags::kAccessExecute)) + protectFlags = Support::test(memoryFlags, MemoryFlags::kAccessWrite) ? PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ; + else if (Support::test(memoryFlags, MemoryFlags::kAccessRW)) + protectFlags = Support::test(memoryFlags, MemoryFlags::kAccessWrite) ? PAGE_READWRITE : PAGE_READONLY; else protectFlags = PAGE_NOACCESS; @@ -120,21 +202,44 @@ static DWORD VirtMem_accessToWinProtectFlags(uint32_t flags) noexcept { return protectFlags; } -static DWORD VirtMem_accessToWinDesiredAccess(uint32_t flags) noexcept { - DWORD access = (flags & VirtMem::kAccessWrite) ? FILE_MAP_WRITE : FILE_MAP_READ; - if (flags & VirtMem::kAccessExecute) +static DWORD desiredAccessFromMemoryFlags(MemoryFlags memoryFlags) noexcept { + DWORD access = Support::test(memoryFlags, MemoryFlags::kAccessWrite) ? FILE_MAP_WRITE : FILE_MAP_READ; + if (Support::test(memoryFlags, MemoryFlags::kAccessExecute)) access |= FILE_MAP_EXECUTE; return access; } -Error VirtMem::alloc(void** p, size_t size, uint32_t flags) noexcept { +static HardenedRuntimeFlags getHardenedRuntimeFlags() noexcept { + HardenedRuntimeFlags flags = HardenedRuntimeFlags::kNone; + + if (hasDualMappingSupport()) + flags |= HardenedRuntimeFlags::kDualMapping; + + return flags; +} + +Error alloc(void** p, size_t size, MemoryFlags memoryFlags) noexcept { *p = nullptr; if (size == 0) return DebugUtils::errored(kErrorInvalidArgument); - DWORD protectFlags = VirtMem_accessToWinProtectFlags(flags); - void* result = ::VirtualAlloc(nullptr, size, MEM_COMMIT | MEM_RESERVE, protectFlags); + DWORD allocationType = MEM_COMMIT | MEM_RESERVE; + DWORD protectFlags = protectFlagsFromMemoryFlags(memoryFlags); + + if (Support::test(memoryFlags, MemoryFlags::kMMapLargePages)) { + size_t lpSize = largePageSize(); + + // Does it make sense to call VirtualAlloc() if we failed to query large page size? + if (lpSize == 0) + return DebugUtils::errored(kErrorFeatureNotEnabled); + + if (!Support::isAligned(size, lpSize)) + return DebugUtils::errored(kErrorInvalidArgument); + allocationType |= MEM_LARGE_PAGES; + } + + void* result = ::VirtualAlloc(nullptr, size, allocationType, protectFlags); if (!result) return DebugUtils::errored(kErrorOutOfMemory); @@ -142,15 +247,17 @@ Error VirtMem::alloc(void** p, size_t size, uint32_t flags) noexcept { return kErrorOk; } -Error VirtMem::release(void* p, size_t size) noexcept { +Error release(void* p, size_t size) noexcept { DebugUtils::unused(size); - if (ASMJIT_UNLIKELY(!::VirtualFree(p, 0, MEM_RELEASE))) + // NOTE: If the `dwFreeType` parameter is MEM_RELEASE, `size` parameter must be zero. + constexpr DWORD dwFreeType = MEM_RELEASE; + if (ASMJIT_UNLIKELY(!::VirtualFree(p, 0, dwFreeType))) return DebugUtils::errored(kErrorInvalidArgument); return kErrorOk; } -Error VirtMem::protect(void* p, size_t size, uint32_t flags) noexcept { - DWORD protectFlags = VirtMem_accessToWinProtectFlags(flags); +Error protect(void* p, size_t size, MemoryFlags memoryFlags) noexcept { + DWORD protectFlags = protectFlagsFromMemoryFlags(memoryFlags); DWORD oldFlags; if (::VirtualProtect(p, size, protectFlags, &oldFlags)) @@ -159,8 +266,8 @@ Error VirtMem::protect(void* p, size_t size, uint32_t flags) noexcept { return DebugUtils::errored(kErrorInvalidArgument); } -Error VirtMem::allocDualMapping(DualMapping* dm, size_t size, uint32_t flags) noexcept { - dm->ro = nullptr; +Error allocDualMapping(DualMapping* dm, size_t size, MemoryFlags memoryFlags) noexcept { + dm->rx = nullptr; dm->rw = nullptr; if (size == 0) @@ -180,87 +287,89 @@ Error VirtMem::allocDualMapping(DualMapping* dm, size_t size, uint32_t flags) no void* ptr[2]; for (uint32_t i = 0; i < 2; i++) { - DWORD desiredAccess = VirtMem_accessToWinDesiredAccess(flags & ~VirtMem_dualMappingFilter[i]); + MemoryFlags accessFlags = memoryFlags & ~dualMappingFilter[i]; + DWORD desiredAccess = desiredAccessFromMemoryFlags(accessFlags); ptr[i] = ::MapViewOfFile(handle.value, desiredAccess, 0, 0, size); if (ptr[i] == nullptr) { - if (i == 0) + if (i == 1u) ::UnmapViewOfFile(ptr[0]); return DebugUtils::errored(kErrorOutOfMemory); } } - dm->ro = ptr[0]; + dm->rx = ptr[0]; dm->rw = ptr[1]; return kErrorOk; } -Error VirtMem::releaseDualMapping(DualMapping* dm, size_t size) noexcept { +Error releaseDualMapping(DualMapping* dm, size_t size) noexcept { DebugUtils::unused(size); bool failed = false; - if (!::UnmapViewOfFile(dm->ro)) + if (!::UnmapViewOfFile(dm->rx)) failed = true; - if (dm->ro != dm->rw && !UnmapViewOfFile(dm->rw)) + if (dm->rx != dm->rw && !UnmapViewOfFile(dm->rw)) failed = true; if (failed) return DebugUtils::errored(kErrorInvalidArgument); - dm->ro = nullptr; + dm->rx = nullptr; dm->rw = nullptr; return kErrorOk; } + #endif -// ============================================================================ -// [asmjit::VirtMem - Virtual Memory [Posix]] -// ============================================================================ +// Virtual Memory [Unix] +// ===================== #if !defined(_WIN32) -struct ScopedFD { - inline ScopedFD() noexcept - : value(-1) {} - inline ~ScopedFD() noexcept { - if (value != -1) - close(value); - } +// Virtual Memory [Unix] - Utilities +// ================================= + +#if defined(__linux__) || (defined(__APPLE__) && TARGET_OS_OSX) +struct KernelVersion { + long ver[2]; - int value; + inline long major() const noexcept { return ver[0]; } + inline long minor() const noexcept { return ver[1]; } + + inline bool eq(long major, long minor) const noexcept { return ver[0] == major && ver[1] == minor; } + inline bool ge(long major, long minor) const noexcept { return ver[0] > major || (ver[0] == major && ver[1] >= minor); } }; -static void VirtMem_getInfo(VirtMem::Info& vmInfo) noexcept { - uint32_t pageSize = uint32_t(::getpagesize()); +ASMJIT_MAYBE_UNUSED +static KernelVersion getKernelVersion() noexcept { + KernelVersion out {}; + struct utsname buf {}; - vmInfo.pageSize = pageSize; - vmInfo.pageGranularity = Support::max<uint32_t>(pageSize, 65536); -} + uname(&buf); -// Some operating systems don't allow /dev/shm to be executable. On Linux this -// happens when /dev/shm is mounted with 'noexec', which is enforced by systemd. -// Other operating systems like OSX also restrict executable permissions regarding -// /dev/shm, so we use a runtime detection before trying to allocate the requested -// memory by the user. Sometimes we don't need the detection as we know it would -// always result in 'kShmStrategyTmpDir'. -enum ShmStrategy : uint32_t { - kShmStrategyUnknown = 0, - kShmStrategyDevShm = 1, - kShmStrategyTmpDir = 2 -}; + size_t i = 0; + char* p = buf.release; -// Posix specific implementation that uses `mmap()` and `munmap()`. -static int VirtMem_accessToPosixProtection(uint32_t flags) noexcept { - int protection = 0; - if (flags & VirtMem::kAccessRead ) protection |= PROT_READ; - if (flags & VirtMem::kAccessWrite ) protection |= PROT_READ | PROT_WRITE; - if (flags & VirtMem::kAccessExecute) protection |= PROT_READ | PROT_EXEC; - return protection; + while (*p && i < 2u) { + uint8_t c = uint8_t(*p); + if (c >= uint8_t('0') && c <= uint8_t('9')) { + out.ver[i] = strtol(p, &p, 10); + i++; + continue; + } + + p++; + } + + return out; } +#endif // getKernelVersion // Translates libc errors specific to VirtualMemory mapping to `asmjit::Error`. -static Error VirtMem_makeErrorFromErrno(int e) noexcept { +ASMJIT_MAYBE_UNUSED +static Error asmjitErrorFromErrno(int e) noexcept { switch (e) { case EACCES: case EAGAIN: @@ -282,300 +391,745 @@ static Error VirtMem_makeErrorFromErrno(int e) noexcept { } } -#if defined(__APPLE__) -// Detects whether the current process is hardened, which means that pages that -// have WRITE and EXECUTABLE flags cannot be allocated without MAP_JIT flag. -static ASMJIT_INLINE bool VirtMem_isHardened() noexcept { - static volatile uint32_t globalHardenedFlag; +ASMJIT_MAYBE_UNUSED +static MemoryFlags maxAccessFlagsToRegularAccessFlags(MemoryFlags memoryFlags) noexcept { + static constexpr uint32_t kMaxProtShift = Support::ConstCTZ<uint32_t(MemoryFlags::kMMapMaxAccessRead)>::value; + return MemoryFlags(uint32_t(memoryFlags & MemoryFlags::kMMapMaxAccessRWX) >> kMaxProtShift); +} - enum HardenedFlag : uint32_t { - kHardenedFlagUnknown = 0, - kHardenedFlagDisabled = 1, - kHardenedFlagEnabled = 2 - }; +ASMJIT_MAYBE_UNUSED +static MemoryFlags regularAccessFlagsToMaxAccessFlags(MemoryFlags memoryFlags) noexcept { + static constexpr uint32_t kMaxProtShift = Support::ConstCTZ<uint32_t(MemoryFlags::kMMapMaxAccessRead)>::value; + return MemoryFlags(uint32_t(memoryFlags & MemoryFlags::kAccessRWX) << kMaxProtShift); +} - uint32_t flag = globalHardenedFlag; - if (flag == kHardenedFlagUnknown) { - VirtMem::Info memInfo; - VirtMem_getInfo(memInfo); +// Returns `mmap()` protection flags from \ref MemoryFlags. +ASMJIT_MAYBE_UNUSED +static int mmProtFromMemoryFlags(MemoryFlags memoryFlags) noexcept { + int protection = 0; + if (Support::test(memoryFlags, MemoryFlags::kAccessRead)) protection |= PROT_READ; + if (Support::test(memoryFlags, MemoryFlags::kAccessWrite)) protection |= PROT_READ | PROT_WRITE; + if (Support::test(memoryFlags, MemoryFlags::kAccessExecute)) protection |= PROT_READ | PROT_EXEC; + return protection; +} - void* ptr = mmap(nullptr, memInfo.pageSize, PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (ptr == MAP_FAILED) { - flag = kHardenedFlagEnabled; - } - else { - flag = kHardenedFlagDisabled; - munmap(ptr, memInfo.pageSize); - } - globalHardenedFlag = flag; +// Returns maximum protection flags from `memoryFlags`. +// +// Uses: +// - `PROT_MPROTECT()` on NetBSD. +// - `PROT_MAX()` when available on other BSDs. +ASMJIT_MAYBE_UNUSED +static inline int mmMaxProtFromMemoryFlags(MemoryFlags memoryFlags) noexcept { + MemoryFlags acc = maxAccessFlagsToRegularAccessFlags(memoryFlags); + if (acc != MemoryFlags::kNone) { +#if defined(__NetBSD__) && defined(PROT_MPROTECT) + return PROT_MPROTECT(mmProtFromMemoryFlags(acc)); +#elif defined(PROT_MAX) + return PROT_MAX(mmProtFromMemoryFlags(acc)); +#else + return 0; +#endif } - return flag == kHardenedFlagEnabled; + return 0; } -// MAP_JIT flag required to run unsigned JIT code is only supported by kernel -// version 10.14+ (Mojave) and IOS. -static ASMJIT_INLINE bool VirtMem_hasMapJitSupport() noexcept { -#if TARGET_OS_OSX - static volatile int globalVersion; +static void detectVMInfo(Info& vmInfo) noexcept { + uint32_t pageSize = uint32_t(::getpagesize()); - int ver = globalVersion; - if (!ver) { - struct utsname osname; - uname(&osname); - ver = atoi(osname.release); - globalVersion = ver; - } - return ver >= 18; -#else - // Assume it's available. - return true; -#endif + vmInfo.pageSize = pageSize; + vmInfo.pageGranularity = Support::max<uint32_t>(pageSize, 65536); } -static ASMJIT_INLINE int VirtMem_appleSpecificMMapFlags(uint32_t flags) { - // Always use MAP_JIT flag if user asked for it (could be used for testing - // on non-hardened processes) and detect whether it must be used when the - // process is actually hardened (in that case it doesn't make sense to rely - // on user `flags`). - bool useMapJit = ((flags & VirtMem::kMMapEnableMapJit) != 0) || VirtMem_isHardened(); - if (useMapJit) - return VirtMem_hasMapJitSupport() ? int(MAP_JIT) : 0; +static size_t detectLargePageSize() noexcept { +#if defined(__APPLE__) && defined(VM_FLAGS_SUPERPAGE_SIZE_2MB) && ASMJIT_ARCH_X86 + return 2u * 1024u * 1024u; +#elif defined(__FreeBSD__) + Support::Array<size_t, 2> pageSize; + // TODO: Does it return unsigned? + return (getpagesizes(pageSize.data(), 2) < 2) ? 0 : uint32_t(pageSize[1]); +#elif defined(__linux__) + StringTmp<128> storage; + if (OSUtils::readFile("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", storage, 16) != kErrorOk || storage.empty()) + return 0u; + + // The first value should be the size of the page (hpage_pmd_size). + size_t largePageSize = 0; + + const char* buf = storage.data(); + size_t bufSize = storage.size(); + + for (size_t i = 0; i < bufSize; i++) { + uint32_t digit = uint32_t(uint8_t(buf[i]) - uint8_t('0')); + if (digit >= 10u) + break; + largePageSize = largePageSize * 10 + digit; + } + + if (Support::isPowerOf2(largePageSize)) + return largePageSize; else - return 0; -} + return 0u; #else -static ASMJIT_INLINE int VirtMem_appleSpecificMMapFlags(uint32_t flags) { - DebugUtils::unused(flags); - return 0; -} + return 0u; #endif +} + +// Virtual Memory [Posix] - Anonymous Memory +// ========================================= + +#if defined(ASMJIT_ANONYMOUS_MEMORY_USE_FD) + +// Some operating systems don't allow /dev/shm to be executable. On Linux this happens when /dev/shm is mounted with +// 'noexec', which is enforced by systemd. Other operating systems like MacOS also restrict executable permissions +// regarding /dev/shm, so we use a runtime detection before attempting to allocate executable memory. Sometimes we +// don't need the detection as we know it would always result in `AnonymousMemoryStrategy::kTmpDir`. +enum class AnonymousMemoryStrategy : uint32_t { + kUnknown = 0, + kDevShm = 1, + kTmpDir = 2 +}; -static const char* VirtMem_getTmpDir() noexcept { +#if !defined(SHM_ANON) +static const char* getTmpDir() noexcept { const char* tmpDir = getenv("TMPDIR"); return tmpDir ? tmpDir : "/tmp"; } +#endif -static Error VirtMem_openAnonymousMemory(int* fd, bool preferTmpOverDevShm) noexcept { -#if defined(SYS_memfd_create) - // Linux specific 'memfd_create' - if the syscall returns `ENOSYS` it means - // it's not available and we will never call it again (would be pointless). +#if defined(__linux__) && defined(__NR_memfd_create) +static uint32_t getMfdExecFlag() noexcept { + static std::atomic<uint32_t> cachedMfdExecSupported; + uint32_t val = cachedMfdExecSupported.load(); - // Zero initialized, if ever changed to '1' that would mean the syscall is not - // available and we must use `shm_open()` and `shm_unlink()`. - static volatile uint32_t memfd_create_not_supported; + if (val == 0u) { + KernelVersion ver = getKernelVersion(); + val = uint32_t(ver.ge(6, 3)) + 1u; + cachedMfdExecSupported.store(val); + } - if (!memfd_create_not_supported) { - *fd = (int)syscall(SYS_memfd_create, "vmem", 0); - if (ASMJIT_LIKELY(*fd >= 0)) - return kErrorOk; + return val == 2u ? uint32_t(MFD_EXEC) : uint32_t(0u); +} +#endif // __linux__ && __NR_memfd_create - int e = errno; - if (e == ENOSYS) - memfd_create_not_supported = 1; - else - return DebugUtils::errored(VirtMem_makeErrorFromErrno(e)); + +// It's not fully random, just to avoid collisions when opening TMP or SHM file. +ASMJIT_MAYBE_UNUSED +static uint64_t generateRandomBits(uintptr_t stackPtr, uint32_t attempt) noexcept { + static std::atomic<uint32_t> internalCounter; + +#if defined(__GNUC__) && ASMJIT_ARCH_X86 + // Use RDTSC instruction to avoid gettimeofday() as we just need some "random" bits. + uint64_t mix = __builtin_ia32_rdtsc(); +#else + struct timeval tm {}; + uint64_t mix = 1; // only used when gettimeofday() fails, which is unlikely. + if (gettimeofday(&tm, nullptr) == 0) { + mix = uint64_t(tm.tv_usec) ^ uint64_t(tm.tv_sec); } #endif -#if defined(SHM_ANON) - // Originally FreeBSD extension, apparently works in other BSDs too. - DebugUtils::unused(preferTmpOverDevShm); - *fd = shm_open(SHM_ANON, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR); + uint64_t bits = (uint64_t(stackPtr) & 0x1010505000055590u) - mix * 773703683; + bits = (bits >> 33) ^ (bits << 7) ^ (attempt * 87178291199); + return bits + uint64_t(++internalCounter) * 10619863; +} - if (ASMJIT_LIKELY(*fd >= 0)) - return kErrorOk; - else - return DebugUtils::errored(VirtMem_makeErrorFromErrno(errno)); -#else - // POSIX API. We have to generate somehow a unique name. This is nothing - // cryptographic, just using a bit from the stack address to always have - // a different base for different threads (as threads have their own stack) - // and retries for avoiding collisions. We use `shm_open()` with flags that - // require creation of the file so we never open an existing shared memory. - static std::atomic<uint32_t> internalCounter; +class AnonymousMemory { +public: + enum FileType : uint32_t { + kFileTypeNone, + kFileTypeShm, + kFileTypeTmp + }; + + int _fd; + FileType _fileType; + StringTmp<128> _tmpName; + + inline AnonymousMemory() noexcept + : _fd(-1), + _fileType(kFileTypeNone), + _tmpName() {} + + inline ~AnonymousMemory() noexcept { + unlink(); + close(); + } - StringTmp<128> uniqueName; - const char* kShmFormat = "/shm-id-%08llX"; + inline int fd() const noexcept { return _fd; } - uint32_t kRetryCount = 100; - uint64_t bits = ((uintptr_t)(void*)&uniqueName) & 0x55555555u; + Error open(bool preferTmpOverDevShm) noexcept { +#if defined(__linux__) && defined(__NR_memfd_create) + // Linux specific 'memfd_create' - if the syscall returns `ENOSYS` it means + // it's not available and we will never call it again (would be pointless). + // + // NOTE: There is also memfd_create() libc function in FreeBSD, but it internally + // uses `shm_open(SHM_ANON, ...)` so it's not needed to add support for it (it's + // not a syscall as in Linux). - for (uint32_t i = 0; i < kRetryCount; i++) { - bits -= uint64_t(OSUtils::getTickCount()) * 773703683; - bits = ((bits >> 14) ^ (bits << 6)) + uint64_t(++internalCounter) * 10619863; + // Zero initialized, if ever changed to '1' that would mean the syscall is not + // available and we must use `shm_open()` and `shm_unlink()` (or regular `open()`). + static volatile uint32_t memfd_create_not_supported; - if (!ASMJIT_VM_SHM_DETECT || preferTmpOverDevShm) { - uniqueName.assign(VirtMem_getTmpDir()); - uniqueName.appendFormat(kShmFormat, (unsigned long long)bits); - *fd = open(uniqueName.data(), O_RDWR | O_CREAT | O_EXCL, 0); - if (ASMJIT_LIKELY(*fd >= 0)) { - unlink(uniqueName.data()); + if (!memfd_create_not_supported) { + _fd = (int)syscall(__NR_memfd_create, "vmem", MFD_CLOEXEC | getMfdExecFlag()); + if (ASMJIT_LIKELY(_fd >= 0)) return kErrorOk; - } + + int e = errno; + if (e == ENOSYS) + memfd_create_not_supported = 1; + else + return DebugUtils::errored(asmjitErrorFromErrno(e)); } - else { - uniqueName.assignFormat(kShmFormat, (unsigned long long)bits); - *fd = shm_open(uniqueName.data(), O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR); - if (ASMJIT_LIKELY(*fd >= 0)) { - shm_unlink(uniqueName.data()); - return kErrorOk; +#endif // __linux__ && __NR_memfd_create + +#if defined(ASMJIT_HAS_SHM_OPEN) && defined(SHM_ANON) + // Originally FreeBSD extension, apparently works in other BSDs too. + DebugUtils::unused(preferTmpOverDevShm); + _fd = ::shm_open(SHM_ANON, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR); + + if (ASMJIT_LIKELY(_fd >= 0)) + return kErrorOk; + else + return DebugUtils::errored(asmjitErrorFromErrno(errno)); +#else + // POSIX API. We have to generate somehow a unique name, so use `generateRandomBits()` helper. To prevent + // having file collisions we use `shm_open()` with flags that require creation of the file so we never open + // an existing shared memory. + static const char kShmFormat[] = "/shm-id-%016llX"; + uint32_t kRetryCount = 100; + + for (uint32_t i = 0; i < kRetryCount; i++) { + bool useTmp = !ASMJIT_VM_SHM_DETECT || preferTmpOverDevShm; + uint64_t bits = generateRandomBits((uintptr_t)this, i); + + if (useTmp) { + _tmpName.assign(getTmpDir()); + _tmpName.appendFormat(kShmFormat, (unsigned long long)bits); + _fd = ASMJIT_FILE64_API(::open)(_tmpName.data(), O_RDWR | O_CREAT | O_EXCL, 0); + if (ASMJIT_LIKELY(_fd >= 0)) { + _fileType = kFileTypeTmp; + return kErrorOk; + } + } +#if defined(ASMJIT_HAS_SHM_OPEN) + else { + _tmpName.assignFormat(kShmFormat, (unsigned long long)bits); + _fd = ::shm_open(_tmpName.data(), O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR); + if (ASMJIT_LIKELY(_fd >= 0)) { + _fileType = kFileTypeShm; + return kErrorOk; + } } +#endif + + int e = errno; + if (e != EEXIST) + return DebugUtils::errored(asmjitErrorFromErrno(e)); } - int e = errno; - if (e == EEXIST) - continue; - else - return DebugUtils::errored(VirtMem_makeErrorFromErrno(e)); + return DebugUtils::errored(kErrorFailedToOpenAnonymousMemory); +#endif } - return kErrorOk; + + void unlink() noexcept { + FileType type = _fileType; + _fileType = kFileTypeNone; + +#ifdef ASMJIT_HAS_SHM_OPEN + if (type == kFileTypeShm) { + ::shm_unlink(_tmpName.data()); + return; + } #endif -} + + if (type == kFileTypeTmp) { + ::unlink(_tmpName.data()); + return; + } + } + + void close() noexcept { + if (_fd >= 0) { + ::close(_fd); + _fd = -1; + } + } + + Error allocate(size_t size) noexcept { + // TODO: Improve this by using `posix_fallocate()` when available. + if (ASMJIT_FILE64_API(ftruncate)(_fd, off_t(size)) != 0) + return DebugUtils::errored(asmjitErrorFromErrno(errno)); + + return kErrorOk; + } +}; #if ASMJIT_VM_SHM_DETECT -static Error VirtMem_detectShmStrategy(uint32_t* strategyOut) noexcept { - ScopedFD fd; - VirtMem::Info vmInfo = VirtMem::info(); +static Error detectAnonymousMemoryStrategy(AnonymousMemoryStrategy* strategyOut) noexcept { + AnonymousMemory anonMem; + Info vmInfo = info(); - ASMJIT_PROPAGATE(VirtMem_openAnonymousMemory(&fd.value, false)); - if (ftruncate(fd.value, off_t(vmInfo.pageSize)) != 0) - return DebugUtils::errored(VirtMem_makeErrorFromErrno(errno)); + ASMJIT_PROPAGATE(anonMem.open(false)); + ASMJIT_PROPAGATE(anonMem.allocate(vmInfo.pageSize)); - void* ptr = mmap(nullptr, vmInfo.pageSize, PROT_READ | PROT_EXEC, MAP_SHARED, fd.value, 0); + void* ptr = mmap(nullptr, vmInfo.pageSize, PROT_READ | PROT_EXEC, MAP_SHARED, anonMem.fd(), 0); if (ptr == MAP_FAILED) { int e = errno; if (e == EINVAL) { - *strategyOut = kShmStrategyTmpDir; + *strategyOut = AnonymousMemoryStrategy::kTmpDir; return kErrorOk; } - return DebugUtils::errored(VirtMem_makeErrorFromErrno(e)); + return DebugUtils::errored(asmjitErrorFromErrno(e)); } else { munmap(ptr, vmInfo.pageSize); - *strategyOut = kShmStrategyDevShm; + *strategyOut = AnonymousMemoryStrategy::kDevShm; return kErrorOk; } } #endif +static Error getAnonymousMemoryStrategy(AnonymousMemoryStrategy* strategyOut) noexcept { #if ASMJIT_VM_SHM_DETECT -static Error VirtMem_getShmStrategy(uint32_t* strategyOut) noexcept { - // Initially don't assume anything. It has to be tested whether - // '/dev/shm' was mounted with 'noexec' flag or not. - static volatile uint32_t globalShmStrategy = kShmStrategyUnknown; - - uint32_t strategy = globalShmStrategy; - if (strategy == kShmStrategyUnknown) { - ASMJIT_PROPAGATE(VirtMem_detectShmStrategy(&strategy)); - globalShmStrategy = strategy; + // Initially don't assume anything. It has to be tested whether '/dev/shm' was mounted with 'noexec' flag or not. + static std::atomic<uint32_t> cachedStrategy; + + AnonymousMemoryStrategy strategy = static_cast<AnonymousMemoryStrategy>(cachedStrategy.load()); + if (strategy == AnonymousMemoryStrategy::kUnknown) { + ASMJIT_PROPAGATE(detectAnonymousMemoryStrategy(&strategy)); + cachedStrategy.store(static_cast<uint32_t>(strategy)); } *strategyOut = strategy; return kErrorOk; -} #else -static Error VirtMem_getShmStrategy(uint32_t* strategyOut) noexcept { - *strategyOut = kShmStrategyTmpDir; + *strategyOut = AnonymousMemoryStrategy::kTmpDir; return kErrorOk; +#endif } + +#endif // ASMJIT_ANONYMOUS_MEMORY_USE_FD + +// Virtual Memory [Posix] - Hardened Runtime & MAP_JIT +// =================================================== + +// Detects whether the current process is hardened, which means that pages that have WRITE and EXECUTABLE flags +// cannot be normally allocated. On OSX + AArch64 such allocation requires MAP_JIT flag, other platforms don't +// support this combination. +static bool hasHardenedRuntime() noexcept { +#if defined(__APPLE__) && TARGET_OS_OSX && ASMJIT_ARCH_ARM >= 64 + // OSX on AArch64 has always hardened runtime enabled. + return true; +#else + static std::atomic<uint32_t> cachedHardenedFlag; + + enum HardenedFlag : uint32_t { + kHardenedFlagUnknown = 0, + kHardenedFlagDisabled = 1, + kHardenedFlagEnabled = 2 + }; + + uint32_t flag = cachedHardenedFlag.load(); + if (flag == kHardenedFlagUnknown) { + size_t pageSize = size_t(::getpagesize()); + void* ptr = mmap(nullptr, pageSize, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + + if (ptr == MAP_FAILED) { + flag = kHardenedFlagEnabled; + } + else { + flag = kHardenedFlagDisabled; + munmap(ptr, pageSize); + } + + cachedHardenedFlag.store(flag); + } + + return flag == kHardenedFlagEnabled; #endif +} -Error VirtMem::alloc(void** p, size_t size, uint32_t flags) noexcept { - *p = nullptr; +// Detects whether MAP_JIT is available. +static inline bool hasMapJitSupport() noexcept { +#if defined(__APPLE__) && TARGET_OS_OSX && ASMJIT_ARCH_X86 == 0 + // Apple platforms always use hardened runtime + MAP_JIT on non-x86 hardware: + // - https://developer.apple.com/documentation/apple_silicon/porting_just-in-time_compilers_to_apple_silicon + return true; +#elif defined(__APPLE__) && TARGET_OS_OSX + // MAP_JIT flag required to run unsigned JIT code is only supported by kernel version 10.14+ (Mojave). + static std::atomic<uint32_t> cachedMapJitSupport; + uint32_t val = cachedMapJitSupport.load(); + + if (val == 0u) { + KernelVersion ver = getKernelVersion(); + val = uint32_t(ver.ge(18, 0)) + 1u; + cachedMapJitSupport.store(val); + } + + return val == 2u; +#else + // MAP_JIT is not available (it's only available on OSX). + return false; +#endif +} +// Returns either MAP_JIT or 0 based on `memoryFlags` and the host operating system. +static inline int mmMapJitFromMemoryFlags(MemoryFlags memoryFlags) noexcept { +#if defined(__APPLE__) + // Always use MAP_JIT flag if user asked for it (could be used for testing on non-hardened processes) and detect + // whether it must be used when the process is actually hardened (in that case it doesn't make sense to rely on + // user `memoryFlags`). + // + // MAP_JIT is not required when dual-mapping memory and is incompatible with MAP_SHARED, so it will not be + // added when the latter is enabled. + bool useMapJit = (Support::test(memoryFlags, MemoryFlags::kMMapEnableMapJit) || hasHardenedRuntime()) + && !Support::test(memoryFlags, MemoryFlags::kMapShared); + if (useMapJit) + return hasMapJitSupport() ? int(MAP_JIT) : 0; + else + return 0; +#else + DebugUtils::unused(memoryFlags); + return 0; +#endif +} + +static inline bool hasDualMappingSupport() noexcept { +#if defined(ASMJIT_NO_DUAL_MAPPING) + return false; +#else + return true; +#endif +} + +static HardenedRuntimeFlags getHardenedRuntimeFlags() noexcept { + HardenedRuntimeFlags flags = HardenedRuntimeFlags::kNone; + + if (hasHardenedRuntime()) + flags |= HardenedRuntimeFlags::kEnabled; + + if (hasMapJitSupport()) + flags |= HardenedRuntimeFlags::kMapJit; + + if (hasDualMappingSupport()) + flags |= HardenedRuntimeFlags::kDualMapping; + + return flags; +} + +static Error mapMemory(void** p, size_t size, MemoryFlags memoryFlags, int fd = -1, off_t offset = 0) noexcept { + *p = nullptr; if (size == 0) return DebugUtils::errored(kErrorInvalidArgument); - int protection = VirtMem_accessToPosixProtection(flags); - int mmFlags = MAP_PRIVATE | MAP_ANONYMOUS | VirtMem_appleSpecificMMapFlags(flags); - void* ptr = mmap(nullptr, size, protection, mmFlags, -1, 0); + int protection = mmProtFromMemoryFlags(memoryFlags) | mmMaxProtFromMemoryFlags(memoryFlags); + int mmFlags = mmMapJitFromMemoryFlags(memoryFlags); + + mmFlags |= Support::test(memoryFlags, MemoryFlags::kMapShared) ? MAP_SHARED : MAP_PRIVATE; + if (fd == -1) + mmFlags |= MAP_ANONYMOUS; + + bool useLargePages = Support::test(memoryFlags, VirtMem::MemoryFlags::kMMapLargePages); + if (useLargePages) { +#if defined(__linux__) + size_t lpSize = largePageSize(); + if (lpSize == 0) + return DebugUtils::errored(kErrorFeatureNotEnabled); + + if (!Support::isAligned(size, lpSize)) + return DebugUtils::errored(kErrorInvalidArgument); + + unsigned lpSizeLog2 = Support::ctz(lpSize); + mmFlags |= int(unsigned(MAP_HUGETLB) | (lpSizeLog2 << MAP_HUGE_SHIFT)); +#else + return DebugUtils::errored(kErrorFeatureNotEnabled); +#endif // __linux__ + } + + void* ptr = mmap(nullptr, size, protection, mmFlags, fd, offset); if (ptr == MAP_FAILED) - return DebugUtils::errored(kErrorOutOfMemory); + return DebugUtils::errored(asmjitErrorFromErrno(errno)); + +#if defined(MADV_HUGEPAGE) + if (useLargePages) { + madvise(ptr, size, MADV_HUGEPAGE); + } +#endif *p = ptr; return kErrorOk; } -Error VirtMem::release(void* p, size_t size) noexcept { +static Error unmapMemory(void* p, size_t size) noexcept { if (ASMJIT_UNLIKELY(munmap(p, size) != 0)) - return DebugUtils::errored(kErrorInvalidArgument); + return DebugUtils::errored(asmjitErrorFromErrno(errno)); return kErrorOk; } +Error alloc(void** p, size_t size, MemoryFlags memoryFlags) noexcept { + return mapMemory(p, size, memoryFlags); +} + +Error release(void* p, size_t size) noexcept { + return unmapMemory(p, size); +} -Error VirtMem::protect(void* p, size_t size, uint32_t flags) noexcept { - int protection = VirtMem_accessToPosixProtection(flags); +Error protect(void* p, size_t size, MemoryFlags memoryFlags) noexcept { + int protection = mmProtFromMemoryFlags(memoryFlags); if (mprotect(p, size, protection) == 0) return kErrorOk; - return DebugUtils::errored(kErrorInvalidArgument); + return DebugUtils::errored(asmjitErrorFromErrno(errno)); } -Error VirtMem::allocDualMapping(DualMapping* dm, size_t size, uint32_t flags) noexcept { - dm->ro = nullptr; +// Virtual Memory [Posix] - Dual Mapping +// ===================================== + +#if !defined(ASMJIT_NO_DUAL_MAPPING) +static Error unmapDualMapping(DualMapping* dm, size_t size) noexcept { + Error err1 = unmapMemory(dm->rx, size); + Error err2 = kErrorOk; + + if (dm->rx != dm->rw) + err2 = unmapMemory(dm->rw, size); + + // We can report only one error, so report the first... + if (err1 || err2) + return DebugUtils::errored(err1 ? err1 : err2); + + dm->rx = nullptr; dm->rw = nullptr; + return kErrorOk; +} +#endif // !ASMJIT_NO_DUAL_MAPPING - if (off_t(size) <= 0) - return DebugUtils::errored(size == 0 ? kErrorInvalidArgument : kErrorTooLarge); +#if defined(ASMJIT_ANONYMOUS_MEMORY_USE_REMAPDUP) +static Error allocDualMappingUsingRemapdup(DualMapping* dmOut, size_t size, MemoryFlags memoryFlags) noexcept { + MemoryFlags maxAccessFlags = regularAccessFlagsToMaxAccessFlags(memoryFlags); + MemoryFlags finalFlags = memoryFlags | maxAccessFlags | MemoryFlags::kMapShared; + + MemoryFlags rxFlags = finalFlags & ~(MemoryFlags::kAccessWrite | MemoryFlags::kMMapMaxAccessWrite); + MemoryFlags rwFlags = finalFlags & ~(MemoryFlags::kAccessExecute); + + // Allocate RW mapping. + DualMapping dm {}; + ASMJIT_PROPAGATE(mapMemory(&dm.rw, size, rwFlags)); + + // Allocate RX mapping. + dm.rx = mremap(dm.rw, size, nullptr, size, MAP_REMAPDUP); + if (dm.rx == MAP_FAILED) { + int e = errno; + munmap(dm.rw, size); + return DebugUtils::errored(asmjitErrorFromErrno(e)); + } + + if (mprotect(dm.rx, size, mmProtFromMemoryFlags(rxFlags)) != 0) { + int e = errno; + unmapDualMapping(&dm, size); + return DebugUtils::errored(asmjitErrorFromErrno(e)); + } + + *dmOut = dm; + return kErrorOk; +} +#endif + +#if defined(ASMJIT_ANONYMOUS_MEMORY_USE_MACH_VM_REMAP) +static Error asmjitErrorFromKernResult(kern_return_t result) noexcept { + switch (result) { + case KERN_PROTECTION_FAILURE: + return DebugUtils::errored(kErrorProtectionFailure); + case KERN_NO_SPACE: + return DebugUtils::errored(kErrorOutOfMemory); + case KERN_INVALID_ARGUMENT: + return DebugUtils::errored(kErrorInvalidArgument); + default: + return DebugUtils::errored(kErrorInvalidState); + } +} - bool preferTmpOverDevShm = (flags & kMappingPreferTmp) != 0; +static Error allocDualMappingUsingMachVmRemap(DualMapping* dmOut, size_t size, MemoryFlags memoryFlags) noexcept { + DualMapping dm {}; + + MemoryFlags mmapFlags = MemoryFlags::kAccessReadWrite | (memoryFlags & MemoryFlags::kMapShared); + ASMJIT_PROPAGATE(mapMemory(&dm.rx, size, mmapFlags)); + + vm_prot_t curProt; + vm_prot_t maxProt; + + int rwProtectFlags = VM_PROT_READ | VM_PROT_WRITE; + int rxProtectFlags = VM_PROT_READ; + + if (Support::test(memoryFlags, MemoryFlags::kAccessExecute)) + rxProtectFlags |= VM_PROT_EXECUTE; + + kern_return_t result {}; + do { + vm_map_t task = mach_task_self(); + mach_vm_address_t remappedAddr {}; + +#if defined(VM_FLAGS_RANDOM_ADDR) + int remapFlags = VM_FLAGS_ANYWHERE | VM_FLAGS_RANDOM_ADDR; +#else + int remapFlags = VM_FLAGS_ANYWHERE; +#endif + + // Try to remap the existing memory into a different address. + result = mach_vm_remap( + task, // target_task + &remappedAddr, // target_address + size, // size + 0, // mask + remapFlags, // flags + task, // src_task + (mach_vm_address_t)dm.rx, // src_address + false, // copy + &curProt, // cur_protection + &maxProt, // max_protection + VM_INHERIT_DEFAULT); // inheritance + + if (result != KERN_SUCCESS) + break; + + dm.rw = (void*)remappedAddr; + + // Now, try to change permissions of both map regions into RW and RX. The vm_protect() + // API is used twice as we also want to set maximum permissions, so nobody would be + // allowed to change the RX region back to RW or RWX (if RWX is allowed). + uint32_t i; + for (i = 0; i < 2; i++) { + bool setMaximum = (i == 0); + + result = vm_protect( + task, // target_task + (vm_address_t)dm.rx, // address + size, // size + setMaximum, // set_maximum + rxProtectFlags); // new_protection + + if (result != KERN_SUCCESS) + break; + + result = vm_protect(task, // target_task + (vm_address_t)dm.rw, // address + size, // size + setMaximum, // set_maximum + rwProtectFlags); // new_protection + + if (result != KERN_SUCCESS) + break; + } + } while (0); + + if (result != KERN_SUCCESS) { + unmapDualMapping(&dm, size); + return DebugUtils::errored(asmjitErrorFromKernResult(result)); + } + + *dmOut = dm; + return kErrorOk; +} +#endif // ASMJIT_ANONYMOUS_MEMORY_USE_MACH_VM_REMAP + +#if defined(ASMJIT_ANONYMOUS_MEMORY_USE_FD) +static Error allocDualMappingUsingFile(DualMapping* dm, size_t size, MemoryFlags memoryFlags) noexcept { + bool preferTmpOverDevShm = Support::test(memoryFlags, MemoryFlags::kMappingPreferTmp); if (!preferTmpOverDevShm) { - uint32_t strategy; - ASMJIT_PROPAGATE(VirtMem_getShmStrategy(&strategy)); - preferTmpOverDevShm = (strategy == kShmStrategyTmpDir); + AnonymousMemoryStrategy strategy; + ASMJIT_PROPAGATE(getAnonymousMemoryStrategy(&strategy)); + preferTmpOverDevShm = (strategy == AnonymousMemoryStrategy::kTmpDir); } - // ScopedFD will automatically close the file descriptor in its destructor. - ScopedFD fd; - ASMJIT_PROPAGATE(VirtMem_openAnonymousMemory(&fd.value, preferTmpOverDevShm)); - if (ftruncate(fd.value, off_t(size)) != 0) - return DebugUtils::errored(VirtMem_makeErrorFromErrno(errno)); + AnonymousMemory anonMem; + ASMJIT_PROPAGATE(anonMem.open(preferTmpOverDevShm)); + ASMJIT_PROPAGATE(anonMem.allocate(size)); void* ptr[2]; for (uint32_t i = 0; i < 2; i++) { - ptr[i] = mmap(nullptr, size, VirtMem_accessToPosixProtection(flags & ~VirtMem_dualMappingFilter[i]), MAP_SHARED, fd.value, 0); - if (ptr[i] == MAP_FAILED) { - // Get the error now before `munmap` has a chance to clobber it. - int e = errno; + MemoryFlags restrictedMemoryFlags = memoryFlags & ~dualMappingFilter[i]; + Error err = mapMemory(&ptr[i], size, restrictedMemoryFlags | MemoryFlags::kMapShared, anonMem.fd(), 0); + if (err != kErrorOk) { if (i == 1) - munmap(ptr[0], size); - return DebugUtils::errored(VirtMem_makeErrorFromErrno(e)); + unmapMemory(ptr[0], size); + return err; } } - dm->ro = ptr[0]; + dm->rx = ptr[0]; dm->rw = ptr[1]; return kErrorOk; } +#endif // ASMJIT_ANONYMOUS_MEMORY_USE_FD -Error VirtMem::releaseDualMapping(DualMapping* dm, size_t size) noexcept { - Error err = release(dm->ro, size); - if (dm->ro != dm->rw) - err |= release(dm->rw, size); +Error allocDualMapping(DualMapping* dm, size_t size, MemoryFlags memoryFlags) noexcept { + dm->rx = nullptr; + dm->rw = nullptr; - if (err) - return DebugUtils::errored(kErrorInvalidArgument); +#if defined(ASMJIT_NO_DUAL_MAPPING) + DebugUtils::unused(size, memoryFlags); + return DebugUtils::errored(kErrorFeatureNotEnabled); +#else + if (off_t(size) <= 0) + return DebugUtils::errored(size == 0 ? kErrorInvalidArgument : kErrorTooLarge); - dm->ro = nullptr; - dm->rw = nullptr; - return kErrorOk; +#if defined(ASMJIT_ANONYMOUS_MEMORY_USE_REMAPDUP) + return allocDualMappingUsingRemapdup(dm, size, memoryFlags); +#elif defined(ASMJIT_ANONYMOUS_MEMORY_USE_MACH_VM_REMAP) + return allocDualMappingUsingMachVmRemap(dm, size, memoryFlags); +#elif defined(ASMJIT_ANONYMOUS_MEMORY_USE_FD) + return allocDualMappingUsingFile(dm, size, memoryFlags); +#else + #error "[asmjit] VirtMem::allocDualMapping() doesn't have implementation for the target OS or architecture" +#endif +#endif // ASMJIT_NO_DUAL_MAPPING } + +Error releaseDualMapping(DualMapping* dm, size_t size) noexcept { +#if defined(ASMJIT_NO_DUAL_MAPPING) + DebugUtils::unused(dm, size); + return DebugUtils::errored(kErrorFeatureNotEnabled); +#else + return unmapDualMapping(dm, size); +#endif // ASMJIT_NO_DUAL_MAPPING +} +#endif + +// Virtual Memory - Flush Instruction Cache +// ======================================== + +void flushInstructionCache(void* p, size_t size) noexcept { +#if ASMJIT_ARCH_X86 || defined(__EMSCRIPTEN__) + // X86/X86_64 architecture doesn't require to do anything to flush instruction cache. + DebugUtils::unused(p, size); +#elif defined(__APPLE__) + sys_icache_invalidate(p, size); +#elif defined(_WIN32) + // Windows has a built-in support in `kernel32.dll`. + FlushInstructionCache(GetCurrentProcess(), p, size); +#elif defined(__GNUC__) + char* start = static_cast<char*>(p); + char* end = start + size; + __builtin___clear_cache(start, end); +#else + #pragma message("[asmjit] VirtMem::flushInstructionCache() doesn't have implementation for the target OS and compiler") + DebugUtils::unused(p, size); #endif +} -// ============================================================================ -// [asmjit::VirtMem - Virtual Memory [Memory Info]] -// ============================================================================ +// Virtual Memory - Memory Info +// ============================ -VirtMem::Info VirtMem::info() noexcept { - static VirtMem::Info vmInfo; +Info info() noexcept { static std::atomic<uint32_t> vmInfoInitialized; + static Info vmInfo; if (!vmInfoInitialized.load()) { - VirtMem::Info localMemInfo; - VirtMem_getInfo(localMemInfo); + Info localMemInfo; + detectVMInfo(localMemInfo); vmInfo = localMemInfo; vmInfoInitialized.store(1u); @@ -584,6 +1138,69 @@ VirtMem::Info VirtMem::info() noexcept { return vmInfo; } -ASMJIT_END_NAMESPACE +size_t largePageSize() noexcept { + static std::atomic<size_t> largePageSize; + static constexpr size_t kNotAvailable = 1; + + size_t size = largePageSize.load(); + if (ASMJIT_LIKELY(size > kNotAvailable)) + return size; + + if (size == kNotAvailable) + return 0; + + size = detectLargePageSize(); + largePageSize.store(size != 0 ? size : kNotAvailable); + return size; +} + +// Virtual Memory - Hardened Runtime Info +// ====================================== +HardenedRuntimeInfo hardenedRuntimeInfo() noexcept { + return HardenedRuntimeInfo { getHardenedRuntimeFlags() }; +} + +// Virtual Memory - Project JIT Memory +// =================================== + +void protectJitMemory(ProtectJitAccess access) noexcept { +#if defined(ASMJIT_HAS_PTHREAD_JIT_WRITE_PROTECT_NP) + pthread_jit_write_protect_np(static_cast<int>(access)); +#else + DebugUtils::unused(access); #endif +} + +ASMJIT_END_SUB_NAMESPACE + +// Virtual Memory - Tests +// ====================== + +#if defined(ASMJIT_TEST) +ASMJIT_BEGIN_NAMESPACE + +UNIT(virt_mem) { + VirtMem::Info vmInfo = VirtMem::info(); + + INFO("VirtMem::info():"); + INFO(" pageSize: %zu", size_t(vmInfo.pageSize)); + INFO(" pageGranularity: %zu", size_t(vmInfo.pageGranularity)); + + INFO("VirtMem::largePageSize():"); + INFO(" largePageSize: %zu", size_t(VirtMem::largePageSize())); + + VirtMem::HardenedRuntimeInfo hardenedRtInfo = VirtMem::hardenedRuntimeInfo(); + VirtMem::HardenedRuntimeFlags hardenedFlags = hardenedRtInfo.flags; + + INFO("VirtMem::hardenedRuntimeInfo():"); + INFO(" flags:"); + INFO(" kEnabled: %s" , Support::test(hardenedFlags, VirtMem::HardenedRuntimeFlags::kEnabled ) ? "true" : "false"); + INFO(" kMapJit: %s" , Support::test(hardenedFlags, VirtMem::HardenedRuntimeFlags::kMapJit ) ? "true" : "false"); + INFO(" kDualMapping: %s", Support::test(hardenedFlags, VirtMem::HardenedRuntimeFlags::kDualMapping) ? "true" : "false"); +} + +ASMJIT_END_NAMESPACE +#endif // ASMJIT_TEST + +#endif // !ASMJIT_NO_JIT |