summaryrefslogtreecommitdiffstatshomepage
path: root/3rdparty/asmjit/src/asmjit/core/cpuinfo.cpp
diff options
context:
space:
mode:
Diffstat (limited to '3rdparty/asmjit/src/asmjit/core/cpuinfo.cpp')
-rw-r--r--3rdparty/asmjit/src/asmjit/core/cpuinfo.cpp1997
1 files changed, 1950 insertions, 47 deletions
diff --git a/3rdparty/asmjit/src/asmjit/core/cpuinfo.cpp b/3rdparty/asmjit/src/asmjit/core/cpuinfo.cpp
index edc7d172227..c6d35459389 100644
--- a/3rdparty/asmjit/src/asmjit/core/cpuinfo.cpp
+++ b/3rdparty/asmjit/src/asmjit/core/cpuinfo.cpp
@@ -1,40 +1,94 @@
-// AsmJit - Machine code generation for C++
+// This file is part of AsmJit project <https://asmjit.com>
//
-// * Official AsmJit Home Page: https://asmjit.com
-// * Official Github Repository: https://github.com/asmjit/asmjit
-//
-// Copyright (c) 2008-2020 The AsmJit Authors
-//
-// This software is provided 'as-is', without any express or implied
-// warranty. In no event will the authors be held liable for any damages
-// arising from the use of this software.
-//
-// Permission is granted to anyone to use this software for any purpose,
-// including commercial applications, and to alter it and redistribute it
-// freely, subject to the following restrictions:
-//
-// 1. The origin of this software must not be misrepresented; you must not
-// claim that you wrote the original software. If you use this software
-// in a product, an acknowledgment in the product documentation would be
-// appreciated but is not required.
-// 2. Altered source versions must be plainly marked as such, and must not be
-// misrepresented as being the original software.
-// 3. This notice may not be removed or altered from any source distribution.
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
#include "../core/api-build_p.h"
#include "../core/cpuinfo.h"
+#include "../core/support.h"
+
+#include <atomic>
+
+// Required by `__cpuidex()` and `_xgetbv()`.
+#if ASMJIT_ARCH_X86
+ #if defined(_MSC_VER)
+ #include <intrin.h>
+ #endif
+#endif // ASMJIT_ARCH_X86
#if !defined(_WIN32)
- #include <errno.h>
- #include <sys/utsname.h>
#include <unistd.h>
#endif
+#if ASMJIT_ARCH_ARM
+ // Required by various utilities that are required by features detection.
+ #if !defined(_WIN32)
+ #include <errno.h>
+ #include <sys/utsname.h>
+ #endif
+
+ //! Required to detect CPU and features on Apple platforms.
+ #if defined(__APPLE__)
+ #include <mach/machine.h>
+ #include <sys/types.h>
+ #include <sys/sysctl.h>
+ #endif
+
+ #if (defined(__linux__) || defined(__FreeBSD__))
+ // Required by `getauxval()` on Linux and FreeBSD.
+ #include <sys/auxv.h>
+ #define ASMJIT_ARM_DETECT_VIA_HWCAPS
+ #endif
+
+ #if ASMJIT_ARCH_ARM >= 64 && defined(__GNUC__) && defined(__linux__) && 0
+ // This feature is disabled at the moment - it works, but it seems linux supports ARM features
+ // via HWCAPS pretty well and the most recent features need to access more registers that were
+ // not originally accessible, which would break on some systems.
+ #define ASMJIT_ARM_DETECT_VIA_CPUID
+ #endif
+
+ #if ASMJIT_ARCH_ARM >= 64 && defined(__OpenBSD__)
+ #include <sys/sysctl.h>
+ #include <machine/cpu.h>
+ #endif
+#endif // ASMJIT_ARCH_ARM
+
ASMJIT_BEGIN_NAMESPACE
-// ============================================================================
-// [asmjit::CpuInfo - Detect - CPU NumThreads]
-// ============================================================================
+// CpuInfo - Detect - Compatibility
+// ================================
+
+// CPU features detection is a minefield on non-X86 platforms. The following list describes which
+// operating systems and architectures are supported and the status of the implementation:
+//
+// * X86, X86_64:
+// - All OSes supported
+// - Detection is based on using a CPUID instruction, which is a user-space instruction, so there
+// is no need to use any OS specific APIs or syscalls to detect all features provided by the CPU.
+//
+// * ARM32:
+// - Linux - HWCAPS based detection.
+// - FreeBSD - HWCAPS based detection (shared with Linux code).
+// - NetBSD - NOT IMPLEMENTED!
+// - OpenBSD - NOT IMPLEMENTED!
+// - Apple - sysctlbyname() based detection (this architecture is deprecated on Apple HW).
+// - Windows - IsProcessorFeaturePresent() based detection (only detects a subset of features).
+// - Others - NOT IMPLEMENTED!
+//
+// * ARM64:
+// - Linux - HWCAPS and CPUID based detection.
+// - FreeBSD - HWCAPS and CPUID based detection (shared with Linux code).
+// - NetBSD - NOT IMPLEMENTED!
+// - OpenBSD - CPUID based detection (reading CPUID via sysctl's CTL_MACHDEP).
+// - Apple - sysctlbyname() based detection with FamilyId matrix (record for each family id).
+// - Windows - IsProcessorFeaturePresent() based detection (only detects a subset of features).
+// - Others - NOT IMPLEMENTED!
+//
+// * Others
+// - NOT IMPLEMENTED!
+
+// CpuInfo - Detect - HW-Thread Count
+// ==================================
#if defined(_WIN32)
static inline uint32_t detectHWThreadCount() noexcept {
@@ -53,42 +107,1891 @@ static inline uint32_t detectHWThreadCount() noexcept {
}
#endif
-// ============================================================================
-// [asmjit::CpuInfo - Detect - CPU Features]
-// ============================================================================
+// CpuInfo - Detect - X86
+// ======================
+
+// X86 and X86_64 detection is based on CPUID.
+
+#if ASMJIT_ARCH_X86
+
+namespace x86 {
+
+typedef CpuFeatures::X86 Ext;
+
+struct cpuid_t { uint32_t eax, ebx, ecx, edx; };
+struct xgetbv_t { uint32_t eax, edx; };
+
+// Executes `cpuid` instruction.
+static inline void cpuidQuery(cpuid_t* out, uint32_t inEax, uint32_t inEcx = 0) noexcept {
+#if defined(_MSC_VER)
+ __cpuidex(reinterpret_cast<int*>(out), inEax, inEcx);
+#elif defined(__GNUC__) && ASMJIT_ARCH_X86 == 32
+ __asm__ __volatile__(
+ "mov %%ebx, %%edi\n"
+ "cpuid\n"
+ "xchg %%edi, %%ebx\n" : "=a"(out->eax), "=D"(out->ebx), "=c"(out->ecx), "=d"(out->edx) : "a"(inEax), "c"(inEcx));
+#elif defined(__GNUC__) && ASMJIT_ARCH_X86 == 64
+ __asm__ __volatile__(
+ "mov %%rbx, %%rdi\n"
+ "cpuid\n"
+ "xchg %%rdi, %%rbx\n" : "=a"(out->eax), "=D"(out->ebx), "=c"(out->ecx), "=d"(out->edx) : "a"(inEax), "c"(inEcx));
+#else
+ #error "[asmjit] x86::cpuidQuery() - Unsupported compiler."
+#endif
+}
+
+// Executes 'xgetbv' instruction.
+static inline void xgetbvQuery(xgetbv_t* out, uint32_t inEcx) noexcept {
+#if defined(_MSC_VER)
+ uint64_t value = _xgetbv(inEcx);
+ out->eax = uint32_t(value & 0xFFFFFFFFu);
+ out->edx = uint32_t(value >> 32);
+#elif defined(__GNUC__)
+ uint32_t outEax;
+ uint32_t outEdx;
+
+ // Replaced, because the world is not perfect:
+ // __asm__ __volatile__("xgetbv" : "=a"(outEax), "=d"(outEdx) : "c"(inEcx));
+ __asm__ __volatile__(".byte 0x0F, 0x01, 0xD0" : "=a"(outEax), "=d"(outEdx) : "c"(inEcx));
+
+ out->eax = outEax;
+ out->edx = outEdx;
+#else
+ out->eax = 0;
+ out->edx = 0;
+#endif
+}
+
+// Map a 12-byte vendor string returned by `cpuid` into a `CpuInfo::Vendor` ID.
+static inline void simplifyCpuVendor(CpuInfo& cpu, uint32_t d0, uint32_t d1, uint32_t d2) noexcept {
+ struct Vendor {
+ char normalized[8];
+ union { char text[12]; uint32_t d[3]; };
+ };
+
+ static const Vendor table[] = {
+ { { 'A', 'M', 'D' }, {{ 'A', 'u', 't', 'h', 'e', 'n', 't', 'i', 'c', 'A', 'M', 'D' }} },
+ { { 'I', 'N', 'T', 'E', 'L' }, {{ 'G', 'e', 'n', 'u', 'i', 'n', 'e', 'I', 'n', 't', 'e', 'l' }} },
+ { { 'V', 'I', 'A' }, {{ 'C', 'e', 'n', 't', 'a', 'u', 'r', 'H', 'a', 'u', 'l', 's' }} },
+ { { 'V', 'I', 'A' }, {{ 'V', 'I', 'A', 0 , 'V', 'I', 'A', 0 , 'V', 'I', 'A', 0 }} },
+ { { 'U', 'N', 'K', 'N', 'O', 'W', 'N' }, {{ 0 }} }
+ };
+
+ uint32_t i;
+ for (i = 0; i < ASMJIT_ARRAY_SIZE(table) - 1; i++)
+ if (table[i].d[0] == d0 && table[i].d[1] == d1 && table[i].d[2] == d2)
+ break;
+ memcpy(cpu._vendor.str, table[i].normalized, 8);
+}
+
+static ASMJIT_FAVOR_SIZE void simplifyCpuBrand(char* s) noexcept {
+ char* d = s;
+
+ char c = s[0];
+ char prev = 0;
+
+ // Used to always clear the current character to ensure that the result
+ // doesn't contain garbage after a new null terminator is placed at the end.
+ s[0] = '\0';
+
+ for (;;) {
+ if (!c)
+ break;
+
+ if (!(c == ' ' && (prev == '@' || s[1] == ' ' || s[1] == '@'))) {
+ *d++ = c;
+ prev = c;
+ }
+
+ c = *++s;
+ s[0] = '\0';
+ }
+
+ d[0] = '\0';
+}
+
+static ASMJIT_FAVOR_SIZE void detectX86Cpu(CpuInfo& cpu) noexcept {
+ using Support::bitTest;
+
+ cpuid_t regs;
+ xgetbv_t xcr0 { 0, 0 };
+ CpuFeatures::X86& features = cpu.features().x86();
+
+ cpu._wasDetected = true;
+ cpu._maxLogicalProcessors = 1;
+
+ // We are gonna execute CPUID, which was introduced by I486, so it's the requirement.
+ features.add(Ext::kI486);
+
+ // CPUID EAX=0
+ // -----------
+
+ // Get vendor string/id.
+ cpuidQuery(&regs, 0x0);
+
+ uint32_t maxId = regs.eax;
+ uint32_t maxSubLeafId_0x7 = 0;
+
+ simplifyCpuVendor(cpu, regs.ebx, regs.edx, regs.ecx);
+
+ // CPUID EAX=1
+ // -----------
+
+ if (maxId >= 0x1) {
+ // Get feature flags in ECX/EDX and family/model in EAX.
+ cpuidQuery(&regs, 0x1);
+
+ // Fill family and model fields.
+ uint32_t modelId = (regs.eax >> 4) & 0x0F;
+ uint32_t familyId = (regs.eax >> 8) & 0x0F;
+
+ // Use extended family and model fields.
+ if (familyId == 0x06u || familyId == 0x0Fu)
+ modelId += (((regs.eax >> 16) & 0x0Fu) << 4);
+
+ if (familyId == 0x0Fu)
+ familyId += ((regs.eax >> 20) & 0xFFu);
+
+ cpu._modelId = modelId;
+ cpu._familyId = familyId;
+ cpu._brandId = (regs.ebx) & 0xFF;
+ cpu._processorType = (regs.eax >> 12) & 0x03;
+ cpu._maxLogicalProcessors = (regs.ebx >> 16) & 0xFF;
+ cpu._stepping = (regs.eax) & 0x0F;
+ cpu._cacheLineSize = ((regs.ebx >> 8) & 0xFF) * 8;
+
+ features.addIf(bitTest(regs.ecx, 0), Ext::kSSE3);
+ features.addIf(bitTest(regs.ecx, 1), Ext::kPCLMULQDQ);
+ features.addIf(bitTest(regs.ecx, 3), Ext::kMONITOR);
+ features.addIf(bitTest(regs.ecx, 5), Ext::kVMX);
+ features.addIf(bitTest(regs.ecx, 6), Ext::kSMX);
+ features.addIf(bitTest(regs.ecx, 9), Ext::kSSSE3);
+ features.addIf(bitTest(regs.ecx, 13), Ext::kCMPXCHG16B);
+ features.addIf(bitTest(regs.ecx, 19), Ext::kSSE4_1);
+ features.addIf(bitTest(regs.ecx, 20), Ext::kSSE4_2);
+ features.addIf(bitTest(regs.ecx, 22), Ext::kMOVBE);
+ features.addIf(bitTest(regs.ecx, 23), Ext::kPOPCNT);
+ features.addIf(bitTest(regs.ecx, 25), Ext::kAESNI);
+ features.addIf(bitTest(regs.ecx, 26), Ext::kXSAVE);
+ features.addIf(bitTest(regs.ecx, 27), Ext::kOSXSAVE);
+ features.addIf(bitTest(regs.ecx, 30), Ext::kRDRAND);
+ features.addIf(bitTest(regs.edx, 0), Ext::kFPU);
+ features.addIf(bitTest(regs.edx, 4), Ext::kRDTSC);
+ features.addIf(bitTest(regs.edx, 5), Ext::kMSR);
+ features.addIf(bitTest(regs.edx, 8), Ext::kCMPXCHG8B);
+ features.addIf(bitTest(regs.edx, 15), Ext::kCMOV);
+ features.addIf(bitTest(regs.edx, 19), Ext::kCLFLUSH);
+ features.addIf(bitTest(regs.edx, 23), Ext::kMMX);
+ features.addIf(bitTest(regs.edx, 24), Ext::kFXSR);
+ features.addIf(bitTest(regs.edx, 25), Ext::kSSE, Ext::kMMX2);
+ features.addIf(bitTest(regs.edx, 26), Ext::kSSE2, Ext::kSSE);
+ features.addIf(bitTest(regs.edx, 28), Ext::kMT);
+
+ // Get the content of XCR0 if supported by the CPU and enabled by the OS.
+ if (features.hasXSAVE() && features.hasOSXSAVE()) {
+ xgetbvQuery(&xcr0, 0);
+ }
+
+ // Detect AVX+.
+ if (bitTest(regs.ecx, 28)) {
+ // - XCR0[2:1] == 11b
+ // XMM & YMM states need to be enabled by OS.
+ if ((xcr0.eax & 0x00000006u) == 0x00000006u) {
+ features.add(Ext::kAVX);
+ features.addIf(bitTest(regs.ecx, 12), Ext::kFMA);
+ features.addIf(bitTest(regs.ecx, 29), Ext::kF16C);
+ }
+ }
+ }
+
+ constexpr uint32_t kXCR0_AMX_Bits = 0x3u << 17;
+ bool amxEnabledByOS = (xcr0.eax & kXCR0_AMX_Bits) == kXCR0_AMX_Bits;
+
+#if defined(__APPLE__)
+ // Apple platform provides on-demand AVX512 support. When an AVX512 instruction is used the first time it results
+ // in #UD, which would cause the thread being promoted to use AVX512 support by the OS in addition to enabling the
+ // necessary bits in XCR0 register.
+ bool avx512EnabledByOS = true;
+#else
+ // - XCR0[2:1] == 11b - XMM/YMM states need to be enabled by OS.
+ // - XCR0[7:5] == 111b - Upper 256-bit of ZMM0-XMM15 and ZMM16-ZMM31 need to be enabled by OS.
+ constexpr uint32_t kXCR0_AVX512_Bits = (0x3u << 1) | (0x7u << 5);
+ bool avx512EnabledByOS = (xcr0.eax & kXCR0_AVX512_Bits) == kXCR0_AVX512_Bits;
+#endif
+
+ // CPUID EAX=7 ECX=0
+ // -----------------
+
+ // Detect new features if the processor supports CPUID-07.
+ bool maybeMPX = false;
+
+ if (maxId >= 0x7) {
+ cpuidQuery(&regs, 0x7);
+
+ maybeMPX = bitTest(regs.ebx, 14);
+ maxSubLeafId_0x7 = regs.eax;
+
+ features.addIf(bitTest(regs.ebx, 0), Ext::kFSGSBASE);
+ features.addIf(bitTest(regs.ebx, 3), Ext::kBMI);
+ features.addIf(bitTest(regs.ebx, 4), Ext::kHLE);
+ features.addIf(bitTest(regs.ebx, 7), Ext::kSMEP);
+ features.addIf(bitTest(regs.ebx, 8), Ext::kBMI2);
+ features.addIf(bitTest(regs.ebx, 9), Ext::kERMS);
+ features.addIf(bitTest(regs.ebx, 11), Ext::kRTM);
+ features.addIf(bitTest(regs.ebx, 18), Ext::kRDSEED);
+ features.addIf(bitTest(regs.ebx, 19), Ext::kADX);
+ features.addIf(bitTest(regs.ebx, 20), Ext::kSMAP);
+ features.addIf(bitTest(regs.ebx, 23), Ext::kCLFLUSHOPT);
+ features.addIf(bitTest(regs.ebx, 24), Ext::kCLWB);
+ features.addIf(bitTest(regs.ebx, 29), Ext::kSHA);
+ features.addIf(bitTest(regs.ecx, 0), Ext::kPREFETCHWT1);
+ features.addIf(bitTest(regs.ecx, 4), Ext::kOSPKE);
+ features.addIf(bitTest(regs.ecx, 5), Ext::kWAITPKG);
+ features.addIf(bitTest(regs.ecx, 7), Ext::kCET_SS);
+ features.addIf(bitTest(regs.ecx, 8), Ext::kGFNI);
+ features.addIf(bitTest(regs.ecx, 9), Ext::kVAES);
+ features.addIf(bitTest(regs.ecx, 10), Ext::kVPCLMULQDQ);
+ features.addIf(bitTest(regs.ecx, 22), Ext::kRDPID);
+ features.addIf(bitTest(regs.ecx, 25), Ext::kCLDEMOTE);
+ features.addIf(bitTest(regs.ecx, 27), Ext::kMOVDIRI);
+ features.addIf(bitTest(regs.ecx, 28), Ext::kMOVDIR64B);
+ features.addIf(bitTest(regs.ecx, 29), Ext::kENQCMD);
+ features.addIf(bitTest(regs.edx, 4), Ext::kFSRM);
+ features.addIf(bitTest(regs.edx, 5), Ext::kUINTR);
+ features.addIf(bitTest(regs.edx, 14), Ext::kSERIALIZE);
+ features.addIf(bitTest(regs.edx, 16), Ext::kTSXLDTRK);
+ features.addIf(bitTest(regs.edx, 18), Ext::kPCONFIG);
+ features.addIf(bitTest(regs.edx, 20), Ext::kCET_IBT);
+
+ // Detect 'TSX' - Requires at least one of `HLE` and `RTM` features.
+ if (features.hasHLE() || features.hasRTM()) {
+ features.add(Ext::kTSX);
+ }
+
+ if (bitTest(regs.ebx, 5) && features.hasAVX()) {
+ features.add(Ext::kAVX2);
+ }
+
+ if (avx512EnabledByOS && bitTest(regs.ebx, 16)) {
+ features.add(Ext::kAVX512_F);
+
+ features.addIf(bitTest(regs.ebx, 17), Ext::kAVX512_DQ);
+ features.addIf(bitTest(regs.ebx, 21), Ext::kAVX512_IFMA);
+ features.addIf(bitTest(regs.ebx, 26), Ext::kAVX512_PF);
+ features.addIf(bitTest(regs.ebx, 27), Ext::kAVX512_ER);
+ features.addIf(bitTest(regs.ebx, 28), Ext::kAVX512_CD);
+ features.addIf(bitTest(regs.ebx, 30), Ext::kAVX512_BW);
+ features.addIf(bitTest(regs.ebx, 31), Ext::kAVX512_VL);
+ features.addIf(bitTest(regs.ecx, 1), Ext::kAVX512_VBMI);
+ features.addIf(bitTest(regs.ecx, 6), Ext::kAVX512_VBMI2);
+ features.addIf(bitTest(regs.ecx, 11), Ext::kAVX512_VNNI);
+ features.addIf(bitTest(regs.ecx, 12), Ext::kAVX512_BITALG);
+ features.addIf(bitTest(regs.ecx, 14), Ext::kAVX512_VPOPCNTDQ);
+ features.addIf(bitTest(regs.edx, 2), Ext::kAVX512_4VNNIW);
+ features.addIf(bitTest(regs.edx, 3), Ext::kAVX512_4FMAPS);
+ features.addIf(bitTest(regs.edx, 8), Ext::kAVX512_VP2INTERSECT);
+ features.addIf(bitTest(regs.edx, 23), Ext::kAVX512_FP16);
+ }
+
+ if (amxEnabledByOS) {
+ features.addIf(bitTest(regs.edx, 22), Ext::kAMX_BF16);
+ features.addIf(bitTest(regs.edx, 24), Ext::kAMX_TILE);
+ features.addIf(bitTest(regs.edx, 25), Ext::kAMX_INT8);
+ }
+ }
+
+ // CPUID EAX=7 ECX=1
+ // -----------------
+
+ if (maxSubLeafId_0x7 >= 1) {
+ cpuidQuery(&regs, 0x7, 1);
+
+ features.addIf(bitTest(regs.eax, 0), Ext::kSHA512);
+ features.addIf(bitTest(regs.eax, 1), Ext::kSM3);
+ features.addIf(bitTest(regs.eax, 2), Ext::kSM4);
+ features.addIf(bitTest(regs.eax, 3), Ext::kRAO_INT);
+ features.addIf(bitTest(regs.eax, 7), Ext::kCMPCCXADD);
+ features.addIf(bitTest(regs.eax, 10), Ext::kFZRM);
+ features.addIf(bitTest(regs.eax, 11), Ext::kFSRS);
+ features.addIf(bitTest(regs.eax, 12), Ext::kFSRC);
+ features.addIf(bitTest(regs.eax, 19), Ext::kWRMSRNS);
+ features.addIf(bitTest(regs.eax, 22), Ext::kHRESET);
+ features.addIf(bitTest(regs.eax, 26), Ext::kLAM);
+ features.addIf(bitTest(regs.eax, 27), Ext::kMSRLIST);
+ features.addIf(bitTest(regs.ebx, 1), Ext::kTSE);
+ features.addIf(bitTest(regs.edx, 14), Ext::kPREFETCHI);
+ features.addIf(bitTest(regs.edx, 18), Ext::kCET_SSS);
+ features.addIf(bitTest(regs.edx, 21), Ext::kAPX_F);
+
+ if (features.hasAVX2()) {
+ features.addIf(bitTest(regs.eax, 4), Ext::kAVX_VNNI);
+ features.addIf(bitTest(regs.eax, 23), Ext::kAVX_IFMA);
+ features.addIf(bitTest(regs.edx, 4), Ext::kAVX_VNNI_INT8);
+ features.addIf(bitTest(regs.edx, 5), Ext::kAVX_NE_CONVERT);
+ features.addIf(bitTest(regs.edx, 10), Ext::kAVX_VNNI_INT16);
+ }
+
+ if (features.hasAVX512_F()) {
+ features.addIf(bitTest(regs.eax, 5), Ext::kAVX512_BF16);
+ }
+
+ if (amxEnabledByOS) {
+ features.addIf(bitTest(regs.eax, 21), Ext::kAMX_FP16);
+ features.addIf(bitTest(regs.edx, 8), Ext::kAMX_COMPLEX);
+ }
+ }
+
+ // CPUID EAX=13 ECX=0
+ // ------------------
+
+ if (maxId >= 0xD) {
+ cpuidQuery(&regs, 0xD, 0);
+
+ // Both CPUID result and XCR0 has to be enabled to have support for MPX.
+ if (((regs.eax & xcr0.eax) & 0x00000018u) == 0x00000018u && maybeMPX)
+ features.add(Ext::kMPX);
+
+ cpuidQuery(&regs, 0xD, 1);
+
+ features.addIf(bitTest(regs.eax, 0), Ext::kXSAVEOPT);
+ features.addIf(bitTest(regs.eax, 1), Ext::kXSAVEC);
+ features.addIf(bitTest(regs.eax, 3), Ext::kXSAVES);
+ }
+
+ // CPUID EAX=14 ECX=0
+ // ------------------
+
+ if (maxId >= 0xE) {
+ cpuidQuery(&regs, 0xE, 0);
+
+ features.addIf(bitTest(regs.ebx, 4), Ext::kPTWRITE);
+ }
+
+ // CPUID EAX=0x80000000...maxId
+ // ----------------------------
+
+ maxId = 0x80000000u;
+ uint32_t i = maxId;
+
+ // The highest EAX that we understand.
+ constexpr uint32_t kHighestProcessedEAX = 0x8000001Fu;
+
+ // Several CPUID calls are required to get the whole branc string. It's easier
+ // to copy one DWORD at a time instead of copying the string a byte by byte.
+ uint32_t* brand = cpu._brand.u32;
+ do {
+ cpuidQuery(&regs, i);
+ switch (i) {
+ case 0x80000000u:
+ maxId = Support::min<uint32_t>(regs.eax, kHighestProcessedEAX);
+ break;
+
+ case 0x80000001u:
+ features.addIf(bitTest(regs.ecx, 0), Ext::kLAHFSAHF);
+ features.addIf(bitTest(regs.ecx, 2), Ext::kSVM);
+ features.addIf(bitTest(regs.ecx, 5), Ext::kLZCNT);
+ features.addIf(bitTest(regs.ecx, 6), Ext::kSSE4A);
+ features.addIf(bitTest(regs.ecx, 7), Ext::kMSSE);
+ features.addIf(bitTest(regs.ecx, 8), Ext::kPREFETCHW);
+ features.addIf(bitTest(regs.ecx, 12), Ext::kSKINIT);
+ features.addIf(bitTest(regs.ecx, 15), Ext::kLWP);
+ features.addIf(bitTest(regs.ecx, 21), Ext::kTBM);
+ features.addIf(bitTest(regs.ecx, 29), Ext::kMONITORX);
+ features.addIf(bitTest(regs.edx, 20), Ext::kNX);
+ features.addIf(bitTest(regs.edx, 21), Ext::kFXSROPT);
+ features.addIf(bitTest(regs.edx, 22), Ext::kMMX2);
+ features.addIf(bitTest(regs.edx, 27), Ext::kRDTSCP);
+ features.addIf(bitTest(regs.edx, 29), Ext::kPREFETCHW);
+ features.addIf(bitTest(regs.edx, 30), Ext::k3DNOW2, Ext::kMMX2);
+ features.addIf(bitTest(regs.edx, 31), Ext::kPREFETCHW);
+
+ if (features.hasAVX()) {
+ features.addIf(bitTest(regs.ecx, 11), Ext::kXOP);
+ features.addIf(bitTest(regs.ecx, 16), Ext::kFMA4);
+ }
+
+ // This feature seems to be only supported by AMD.
+ if (cpu.isVendor("AMD")) {
+ features.addIf(bitTest(regs.ecx, 4), Ext::kALTMOVCR8);
+ }
+ break;
+
+ case 0x80000002u:
+ case 0x80000003u:
+ case 0x80000004u:
+ *brand++ = regs.eax;
+ *brand++ = regs.ebx;
+ *brand++ = regs.ecx;
+ *brand++ = regs.edx;
+
+ // Go directly to the next one we are interested in.
+ if (i == 0x80000004u)
+ i = 0x80000008u - 1;
+ break;
+
+ case 0x80000008u:
+ features.addIf(bitTest(regs.ebx, 0), Ext::kCLZERO);
+ features.addIf(bitTest(regs.ebx, 0), Ext::kRDPRU);
+ features.addIf(bitTest(regs.ebx, 8), Ext::kMCOMMIT);
+ features.addIf(bitTest(regs.ebx, 9), Ext::kWBNOINVD);
+
+ // Go directly to the next one we are interested in.
+ i = 0x8000001Fu - 1;
+ break;
+
+ case 0x8000001Fu:
+ features.addIf(bitTest(regs.eax, 0), Ext::kSME);
+ features.addIf(bitTest(regs.eax, 1), Ext::kSEV);
+ features.addIf(bitTest(regs.eax, 3), Ext::kSEV_ES);
+ features.addIf(bitTest(regs.eax, 4), Ext::kSEV_SNP);
+ features.addIf(bitTest(regs.eax, 6), Ext::kRMPQUERY);
+ break;
+ }
+ } while (++i <= maxId);
+
+ // Simplify CPU brand string a bit by removing some unnecessary spaces.
+ simplifyCpuBrand(cpu._brand.str);
+}
+
+} // {x86}
+
+#endif // ASMJIT_ARCH_X86
+
+// CpuInfo - Detect - ARM
+// ======================
+
+// Implement the most code outside the platform specific #ifdefs to minimize breaking the detection on
+// platforms that don't run on our CI infrastructure. The problem with the detection is that every OS
+// requires a specific implementation as ARM features cannot be detected in user-mode without OS enablement.
+
+// The most relevant and accurate information can be found here:
+// https://github.com/llvm-project/llvm/blob/master/lib/Target/AArch64/AArch64.td
+// https://github.com/apple/llvm-project/blob/apple/main/llvm/lib/Target/AArch64/AArch64.td (Apple fork)
+//
+// Other resources:
+// https://en.wikipedia.org/wiki/AArch64
+// https://en.wikipedia.org/wiki/Apple_silicon#List_of_Apple_processors
+// https://developer.arm.com/downloads/-/exploration-tools/feature-names-for-a-profile
+// https://developer.arm.com/architectures/learn-the-architecture/understanding-the-armv8-x-extensions/single-page
+
+#if ASMJIT_ARCH_ARM
+
+namespace arm {
+
+// ARM commonly refers to CPU features using FEAT_ prefix, we use Ext:: to make it compatible with other parts.
+typedef CpuFeatures::ARM Ext;
+
+// CpuInfo - Detect - ARM - OS Kernel Version
+// ==========================================
+
+#if defined(__linux__)
+struct UNameKernelVersion {
+ int parts[3];
+
+ inline bool atLeast(int major, int minor, int patch = 0) const noexcept {
+ if (parts[0] >= major) {
+ if (parts[0] > major)
+ return true;
+
+ if (parts[1] >= minor) {
+ if (parts[1] > minor)
+ return true;
+
+ return parts[2] >= patch;
+ }
+ }
+
+ return false;
+ }
+};
+
+ASMJIT_MAYBE_UNUSED
+static UNameKernelVersion getUNameKernelVersion() noexcept {
+ UNameKernelVersion ver{};
+ ver.parts[0] = -1;
+
+ utsname buffer;
+ if (uname(&buffer) != 0)
+ return ver;
+
+ size_t count = 0;
+ char* p = buffer.release;
+ while (*p) {
+ uint32_t c = uint8_t(*p);
+ if (c >= uint32_t('0') && c <= uint32_t('9')) {
+ ver.parts[count] = int(strtol(p, &p, 10));
+ if (++count == 3)
+ break;
+ }
+ else if (c == '.' || c == '-') {
+ p++;
+ }
+ else {
+ break;
+ }
+ }
+
+ return ver;
+}
+#endif // __linux__
+
+// CpuInfo - Detect - ARM - Baseline Features of ARM Architectures
+// ===============================================================
+
+ASMJIT_MAYBE_UNUSED
+static inline void populateBaseAArch32Features(CpuFeatures::ARM& features) noexcept {
+ // No baseline flags at the moment.
+ DebugUtils::unused(features);
+}
+
+ASMJIT_MAYBE_UNUSED
+static inline void populateBaseAArch64Features(CpuFeatures::ARM& features) noexcept {
+ // AArch64 is based on ARMv8.0 and later.
+ features.add(Ext::kARMv6);
+ features.add(Ext::kARMv7);
+ features.add(Ext::kARMv8a);
+
+ // AArch64 comes with these features by default.
+ features.add(Ext::kASIMD);
+ features.add(Ext::kFP);
+ features.add(Ext::kIDIVA);
+}
+
+static inline void populateBaseARMFeatures(CpuInfo& cpu) noexcept {
+#if ASMJIT_ARCH_ARM == 32
+ populateBaseAArch32Features(cpu.features().arm());
+#else
+ populateBaseAArch64Features(cpu.features().arm());
+#endif
+}
+
+// CpuInfo - Detect - ARM - Mandatory Features of ARM Architectures
+// ================================================================
+
+// Populates mandatory ARMv8.[v]A features.
+ASMJIT_MAYBE_UNUSED
+static ASMJIT_FAVOR_SIZE void populateARMv8AFeatures(CpuFeatures::ARM& features, uint32_t v) noexcept {
+ switch (v) {
+ default:
+ ASMJIT_FALLTHROUGH;
+ case 9: // ARMv8.9
+ features.add(Ext::kCLRBHB, Ext::kCSSC, Ext::kPRFMSLC, Ext::kSPECRES2, Ext::kRAS2);
+ ASMJIT_FALLTHROUGH;
+ case 8: // ARMv8.8
+ features.add(Ext::kHBC, Ext::kMOPS, Ext::kNMI);
+ ASMJIT_FALLTHROUGH;
+ case 7: // ARMv8.7
+ features.add(Ext::kHCX, Ext::kPAN3, Ext::kWFXT, Ext::kXS);
+ ASMJIT_FALLTHROUGH;
+ case 6: // ARMv8.6
+ features.add(Ext::kAMU1_1, Ext::kBF16, Ext::kECV, Ext::kFGT, Ext::kI8MM);
+ ASMJIT_FALLTHROUGH;
+ case 5: // ARMv8.5
+ features.add(Ext::kBTI, Ext::kCSV2, Ext::kDPB2, Ext::kFLAGM2, Ext::kFRINTTS, Ext::kSB, Ext::kSPECRES, Ext::kSSBS);
+ ASMJIT_FALLTHROUGH;
+ case 4: // ARMv8.4
+ features.add(Ext::kAMU1, Ext::kDIT, Ext::kDOTPROD, Ext::kFLAGM,
+ Ext::kLRCPC2, Ext::kLSE2, Ext::kMPAM, Ext::kNV,
+ Ext::kSEL2, Ext::kTLBIOS, Ext::kTLBIRANGE, Ext::kTRF);
+ ASMJIT_FALLTHROUGH;
+ case 3: // ARMv8.3
+ features.add(Ext::kCCIDX, Ext::kFCMA, Ext::kJSCVT, Ext::kLRCPC, Ext::kPAUTH);
+ ASMJIT_FALLTHROUGH;
+ case 2: // ARMv8.2
+ features.add(Ext::kDPB, Ext::kPAN2, Ext::kRAS, Ext::kUAO);
+ ASMJIT_FALLTHROUGH;
+ case 1: // ARMv8.1
+ features.add(Ext::kCRC32, Ext::kLOR, Ext::kLSE, Ext::kPAN, Ext::kRDM, Ext::kVHE);
+ ASMJIT_FALLTHROUGH;
+ case 0: // ARMv8.0
+ features.add(Ext::kASIMD, Ext::kFP, Ext::kIDIVA, Ext::kVFP_D32);
+ break;
+ }
+}
+
+// Populates mandatory ARMv9.[v] features.
+ASMJIT_MAYBE_UNUSED
+static ASMJIT_FAVOR_SIZE void populateARMv9AFeatures(CpuFeatures::ARM& features, uint32_t v) noexcept {
+ populateARMv8AFeatures(features, v <= 4u ? 5u + v : 9u);
+
+ switch (v) {
+ default:
+ ASMJIT_FALLTHROUGH;
+ case 4: // ARMv9.4 - based on ARMv8.9.
+ ASMJIT_FALLTHROUGH;
+ case 3: // ARMv9.3 - based on ARMv8.8.
+ ASMJIT_FALLTHROUGH;
+ case 2: // ARMv9.2 - based on ARMv8.7.
+ ASMJIT_FALLTHROUGH;
+ case 1: // ARMv9.1 - based on ARMv8.6.
+ ASMJIT_FALLTHROUGH;
+ case 0: // ARMv9.0 - based on ARMv8.5.
+ features.add(Ext::kRME, Ext::kSVE, Ext::kSVE2);
+ break;
+ }
+}
+
+// CpuInfo - Detect - ARM - CPUID Based Features
+// =============================================
+
+// This implements detection based on the content of CPUID registers. The following code doesn't actually read any
+// of the registers so it's an implementation that can theoretically be tested / used in mocks.
+
+// Merges a feature that contains 0b1111 when it doesn't exist and starts at 0b0000 when it does.
+ASMJIT_MAYBE_UNUSED
+static ASMJIT_FORCE_INLINE void mergeAArch64CPUIDFeatureNA(CpuFeatures::ARM& features, uint64_t regBits, uint32_t offset,
+ Ext::Id f0,
+ Ext::Id f1 = Ext::kNone,
+ Ext::Id f2 = Ext::kNone,
+ Ext::Id f3 = Ext::kNone) noexcept {
+
+ uint32_t val = uint32_t((regBits >> offset) & 0xFu);
+
+ // If val == 0b1111 then the feature is not implemented in this case (some early extensions).
+ if (val == 0xFu)
+ return;
+
+ if (f0 != Ext::kNone) features.add(f0);
+ if (f1 != Ext::kNone) features.addIf(val >= 1, f1);
+ if (f2 != Ext::kNone) features.addIf(val >= 2, f2);
+ if (f3 != Ext::kNone) features.addIf(val >= 3, f3);
+}
+
+// Merges a feature identified by a single bit at `offset`.
+ASMJIT_MAYBE_UNUSED
+static ASMJIT_FORCE_INLINE void mergeAArch64CPUIDFeature1B(CpuFeatures::ARM& features, uint64_t regBits, uint32_t offset, Ext::Id f1) noexcept {
+ features.addIf((regBits & (uint64_t(1) << offset)) != 0, f1);
+}
+
+// Merges a feature-list starting from 0b01 when it does (0b00 means feature not supported).
+ASMJIT_MAYBE_UNUSED
+static ASMJIT_FORCE_INLINE void mergeAArch64CPUIDFeature2B(CpuFeatures::ARM& features, uint64_t regBits, uint32_t offset, Ext::Id f1, Ext::Id f2, Ext::Id f3) noexcept {
+ uint32_t val = uint32_t((regBits >> offset) & 0x3u);
+
+ if (f1 != Ext::kNone) features.addIf(val >= 1, f1);
+ if (f2 != Ext::kNone) features.addIf(val >= 2, f2);
+ if (f3 != Ext::kNone) features.addIf(val == 3, f3);
+}
+
+// Merges a feature-list starting from 0b0001 when it does (0b0000 means feature not supported).
+ASMJIT_MAYBE_UNUSED
+static ASMJIT_FORCE_INLINE void mergeAArch64CPUIDFeature4B(CpuFeatures::ARM& features, uint64_t regBits, uint32_t offset,
+ Ext::Id f1,
+ Ext::Id f2 = Ext::kNone,
+ Ext::Id f3 = Ext::kNone,
+ Ext::Id f4 = Ext::kNone) noexcept {
+
+ uint32_t val = uint32_t((regBits >> offset) & 0xFu);
+
+ // if val == 0 it means that this feature is not supported.
+
+ if (f1 != Ext::kNone) features.addIf(val >= 1, f1);
+ if (f2 != Ext::kNone) features.addIf(val >= 2, f2);
+ if (f3 != Ext::kNone) features.addIf(val >= 3, f3);
+ if (f4 != Ext::kNone) features.addIf(val >= 4, f4);
+}
+
+// Merges a feature that is identified by an exact bit-combination of 4 bits.
+ASMJIT_MAYBE_UNUSED
+static ASMJIT_FORCE_INLINE void mergeAArch64CPUIDFeature4S(CpuFeatures::ARM& features, uint64_t regBits, uint32_t offset, uint32_t value, Ext::Id f1) noexcept {
+ features.addIf(uint32_t((regBits >> offset) & 0xFu) == value, f1);
+}
+
+#define MERGE_FEATURE_NA(identifier, reg, offset, ...) mergeAArch64CPUIDFeatureNA(cpu.features().arm(), reg, offset, __VA_ARGS__)
+#define MERGE_FEATURE_1B(identifier, reg, offset, ...) mergeAArch64CPUIDFeature1B(cpu.features().arm(), reg, offset, __VA_ARGS__)
+#define MERGE_FEATURE_2B(identifier, reg, offset, ...) mergeAArch64CPUIDFeature2B(cpu.features().arm(), reg, offset, __VA_ARGS__)
+#define MERGE_FEATURE_4B(identifier, reg, offset, ...) mergeAArch64CPUIDFeature4B(cpu.features().arm(), reg, offset, __VA_ARGS__)
+#define MERGE_FEATURE_4S(identifier, reg, offset, ...) mergeAArch64CPUIDFeature4S(cpu.features().arm(), reg, offset, __VA_ARGS__)
+
+// Detects features based on the content of ID_AA64PFR0_EL1 and ID_AA64PFR1_EL1 registers.
+ASMJIT_MAYBE_UNUSED
+static inline void detectAArch64FeaturesViaCPUID_AA64PFR0_AA64PFR1(CpuInfo& cpu, uint64_t fpr0, uint64_t fpr1) noexcept {
+ // ID_AA64PFR0_EL1
+ // ===============
+
+ // FP and AdvSIMD bits should match (i.e. if FP features FP16, ASIMD must feature it too).
+ MERGE_FEATURE_NA("FP bits [19:16]" , fpr0, 16, Ext::kFP, Ext::kFP16);
+ MERGE_FEATURE_NA("AdvSIMD bits [23:20]" , fpr0, 20, Ext::kASIMD, Ext::kFP16);
+ /*
+ MERGE_FEATURE_4B("GIC bits [27:24]" , fpr0, 24, ...);
+ */
+ MERGE_FEATURE_4B("RAS bits [31:28]" , fpr0, 28, Ext::kRAS, Ext::kRAS1_1, Ext::kRAS2);
+ MERGE_FEATURE_4B("SVE bits [35:32]" , fpr0, 32, Ext::kSVE);
+ MERGE_FEATURE_4B("SEL2 bits [39:36]" , fpr0, 36, Ext::kSEL2);
+ MERGE_FEATURE_4B("MPAM bits [43:40]" , fpr0, 40, Ext::kMPAM);
+ MERGE_FEATURE_4B("AMU bits [47:44]" , fpr0, 44, Ext::kAMU1, Ext::kAMU1_1);
+ MERGE_FEATURE_4B("DIT bits [51:48]" , fpr0, 48, Ext::kDIT);
+ MERGE_FEATURE_4B("RME bits [55:52]" , fpr0, 52, Ext::kRME);
+ MERGE_FEATURE_4B("CSV2 bits [59:56]" , fpr0, 56, Ext::kCSV2, Ext::kCSV2, Ext::kCSV2, Ext::kCSV2_3);
+ MERGE_FEATURE_4B("CSV3 bits [63:60]" , fpr0, 60, Ext::kCSV3);
+
+ // ID_AA64PFR1_EL1
+ // ===============
+
+ MERGE_FEATURE_4B("BT bits [3:0]" , fpr1, 0, Ext::kBTI);
+ MERGE_FEATURE_4B("SSBS bits [7:4]" , fpr1, 4, Ext::kSSBS, Ext::kSSBS2);
+ MERGE_FEATURE_4B("MTE bits [11:8]" , fpr1, 8, Ext::kMTE, Ext::kMTE2, Ext::kMTE3);
+ /*
+ MERGE_FEATURE_4B("RAS_frac bits [15:12]" , fpr1, 12, ...);
+ MERGE_FEATURE_4B("MPAM_frac bits [19:16]" , fpr1, 16, ...);
+ */
+ MERGE_FEATURE_4B("SME bits [27:24]" , fpr1, 24, Ext::kSME, Ext::kSME2);
+ MERGE_FEATURE_4B("RNDR_trap bits [31:28]" , fpr1, 28, Ext::kRNG_TRAP);
+ /*
+ MERGE_FEATURE_4B("CSV2_frac bits [35:32]" , fpr1, 32, ...);
+ */
+ MERGE_FEATURE_4B("NMI bits [39:36]" , fpr1, 36, Ext::kNMI);
+ /*
+ MERGE_FEATURE_4B("MTE_frac bits [43:40]" , fpr1, 40, ...);
+ */
+ MERGE_FEATURE_4B("GCS bits [47:44]" , fpr1, 44, Ext::kGCS);
+ MERGE_FEATURE_4B("THE bits [51:48]" , fpr1, 48, Ext::kTHE);
+
+ // MTEX extensions are only available when MTE3 is available.
+ if (cpu.features().arm().hasMTE3())
+ MERGE_FEATURE_4B("MTEX bits [55:52]" , fpr1, 52, Ext::kMTE4);
+
+ /*
+ MERGE_FEATURE_4B("DF2 bits [59:56]" , fpr1, 56, ...);
+ */
+ MERGE_FEATURE_4B("PFAR bits [63:60]" , fpr1, 60, Ext::kPFAR);
+
+ // ID_AA64PFR0_EL1 + ID_AA64PFR1_EL1
+ // =================================
+
+ uint32_t rasMain = uint32_t((fpr0 >> 28) & 0xFu);
+ uint32_t rasFrac = uint32_t((fpr1 >> 12) & 0xFu);
+
+ if (rasMain == 1 && rasFrac == 1) {
+ cpu.features().arm().add(Ext::kRAS1_1);
+ }
+
+ uint32_t mpamMain = uint32_t((fpr0 >> 40) & 0xFu);
+ uint32_t mpamFrac = uint32_t((fpr1 >> 16) & 0xFu);
+
+ if (mpamMain || mpamFrac)
+ cpu.features().arm().add(Ext::kMPAM);
+}
+
+// Detects features based on the content of ID_AA64ISAR0_EL1 and ID_AA64ISAR1_EL1 registers.
+ASMJIT_MAYBE_UNUSED
+static inline void detectAArch64FeaturesViaCPUID_AA64ISAR0_AA64ISAR1(CpuInfo& cpu, uint64_t isar0, uint64_t isar1) noexcept {
+ // ID_AA64ISAR0_EL1
+ // ================
+
+ MERGE_FEATURE_4B("AES bits [7:4]" , isar0, 4, Ext::kAES, Ext::kPMULL);
+ MERGE_FEATURE_4B("SHA1 bits [11:8]" , isar0, 8, Ext::kSHA1);
+ MERGE_FEATURE_4B("SHA2 bits [15:12]" , isar0, 12, Ext::kSHA256, Ext::kSHA512);
+ MERGE_FEATURE_4B("CRC32 bits [19:16]" , isar0, 16, Ext::kCRC32);
+ MERGE_FEATURE_4B("Atomic bits [23:20]" , isar0, 20, Ext::kNone, Ext::kLSE, Ext::kLSE128);
+ MERGE_FEATURE_4B("TME bits [27:24]" , isar0, 24, Ext::kTME);
+ MERGE_FEATURE_4B("RDM bits [31:28]" , isar0, 28, Ext::kRDM);
+ MERGE_FEATURE_4B("SHA3 bits [35:32]" , isar0, 32, Ext::kSHA3);
+ MERGE_FEATURE_4B("SM3 bits [39:36]" , isar0, 36, Ext::kSM3);
+ MERGE_FEATURE_4B("SM4 bits [43:40]" , isar0, 40, Ext::kSM4);
+ MERGE_FEATURE_4B("DP bits [47:44]" , isar0, 44, Ext::kDOTPROD);
+ MERGE_FEATURE_4B("FHM bits [51:48]" , isar0, 48, Ext::kFHM);
+ MERGE_FEATURE_4B("TS bits [55:52]" , isar0, 52, Ext::kFLAGM, Ext::kFLAGM2);
+ /*
+ MERGE_FEATURE_4B("TLB bits [59:56]" , isar0, 56, ...);
+ */
+ MERGE_FEATURE_4B("RNDR bits [63:60]" , isar0, 60, Ext::kFLAGM, Ext::kRNG);
+
+ // ID_AA64ISAR1_EL1
+ // ================
+
+ MERGE_FEATURE_4B("DPB bits [3:0]" , isar1, 0, Ext::kDPB, Ext::kDPB2);
+ /*
+ MERGE_FEATURE_4B("APA bits [7:4]" , isar1, 4, ...);
+ MERGE_FEATURE_4B("API bits [11:8]" , isar1, 8, ...);
+ */
+ MERGE_FEATURE_4B("JSCVT bits [15:12]" , isar1, 12, Ext::kJSCVT);
+ MERGE_FEATURE_4B("FCMA bits [19:16]" , isar1, 16, Ext::kFCMA);
+ MERGE_FEATURE_4B("LRCPC bits [23:20]" , isar1, 20, Ext::kLRCPC, Ext::kLRCPC2, Ext::kLRCPC3);
+ /*
+ MERGE_FEATURE_4B("GPA bits [27:24]" , isar1, 24, ...);
+ MERGE_FEATURE_4B("GPI bits [31:28]" , isar1, 28, ...);
+ */
+ MERGE_FEATURE_4B("FRINTTS bits [35:32]" , isar1, 32, Ext::kFRINTTS);
+ MERGE_FEATURE_4B("SB bits [39:36]" , isar1, 36, Ext::kSB);
+ MERGE_FEATURE_4B("SPECRES bits [43:40]" , isar1, 40, Ext::kSPECRES, Ext::kSPECRES2);
+ MERGE_FEATURE_4B("BF16 bits [47:44]" , isar1, 44, Ext::kBF16, Ext::kEBF16);
+ MERGE_FEATURE_4B("DGH bits [51:48]" , isar1, 48, Ext::kDGH);
+ MERGE_FEATURE_4B("I8MM bits [55:52]" , isar1, 52, Ext::kI8MM);
+ MERGE_FEATURE_4B("XS bits [59:56]" , isar1, 56, Ext::kXS);
+ MERGE_FEATURE_4B("LS64 bits [63:60]" , isar1, 60, Ext::kLS64, Ext::kLS64_V, Ext::kLS64_ACCDATA);
+}
+
+// Detects features based on the content of ID_AA64ISAR2_EL1 register.
+ASMJIT_MAYBE_UNUSED
+static inline void detectAArch64FeaturesViaCPUID_AA64ISAR2(CpuInfo& cpu, uint64_t isar2) noexcept {
+ MERGE_FEATURE_4B("WFxT bits [3:0]" , isar2, 0, Ext::kNone, Ext::kWFXT);
+ MERGE_FEATURE_4B("RPRES bits [7:4]" , isar2, 4, Ext::kRPRES);
+ /*
+ MERGE_FEATURE_4B("GPA3 bits [11:8]" , isar2, 8, ...);
+ MERGE_FEATURE_4B("APA3 bits [15:12]" , isar2, 12, ...);
+ */
+ MERGE_FEATURE_4B("MOPS bits [19:16]" , isar2, 16, Ext::kMOPS);
+ MERGE_FEATURE_4B("BC bits [23:20]" , isar2, 20, Ext::kHBC);
+ MERGE_FEATURE_4B("PAC_frac bits [27:24]" , isar2, 24, Ext::kCONSTPACFIELD);
+ MERGE_FEATURE_4B("CLRBHB bits [31:28]" , isar2, 28, Ext::kCLRBHB);
+ MERGE_FEATURE_4B("SYSREG128 bits [35:32]" , isar2, 32, Ext::kSYSREG128);
+ MERGE_FEATURE_4B("SYSINSTR128 bits [39:36]" , isar2, 36, Ext::kSYSINSTR128);
+ MERGE_FEATURE_4B("PRFMSLC bits [43:40]" , isar2, 40, Ext::kPRFMSLC);
+ MERGE_FEATURE_4B("RPRFM bits [51:48]" , isar2, 48, Ext::kRPRFM);
+ MERGE_FEATURE_4B("CSSC bits [55:52]" , isar2, 52, Ext::kCSSC);
+ MERGE_FEATURE_4B("LUT bits [59:56]" , isar2, 56, Ext::kLUT);
+}
+
+// TODO: This register is not accessed at the moment.
+#if 0
+// Detects features based on the content of ID_AA64ISAR3_EL1register.
+ASMJIT_MAYBE_UNUSED
+static inline void detectAArch64FeaturesViaCPUID_AA64ISAR3(CpuInfo& cpu, uint64_t isar3) noexcept {
+ // ID_AA64ISAR3_EL1
+ // ================
+
+ MERGE_FEATURE_4B("CPA bits [3:0]" , isar3, 0, Ext::kCPA, Ext::kCPA2);
+ MERGE_FEATURE_4B("FAMINMAX bits [7:4]" , isar3, 4, Ext::kFAMINMAX);
+ MERGE_FEATURE_4B("TLBIW bits [11:8]" , isar3, 8, Ext::kTLBIW);
+}
+#endif
+
+ASMJIT_MAYBE_UNUSED
+static inline void detectAArch64FeaturesViaCPUID_AA64MMFR0(CpuInfo& cpu, uint64_t mmfr0) noexcept {
+ // ID_AA64MMFR0_EL1
+ // ================
+
+ /*
+ MERGE_FEATURE_4B("PARange bits [3:0]" , mmfr0, 0, ...);
+ MERGE_FEATURE_4B("ASIDBits bits [7:4]" , mmfr0, 4, ...);
+ MERGE_FEATURE_4B("BigEnd bits [11:8]" , mmfr0, 8, ...);
+ MERGE_FEATURE_4B("SNSMem bits [15:12]" , mmfr0, 12, ...);
+ MERGE_FEATURE_4B("BigEndEL0 bits [19:16]" , mmfr0, 16, ...);
+ MERGE_FEATURE_4B("TGran16 bits [23:20]" , mmfr0, 20, ...);
+ MERGE_FEATURE_4B("TGran64 bits [27:24]" , mmfr0, 24, ...);
+ MERGE_FEATURE_4B("TGran4 bits [31:28]" , mmfr0, 28, ...);
+ MERGE_FEATURE_4B("TGran16_2 bits [35:32]" , mmfr0, 32, ...);
+ MERGE_FEATURE_4B("TGran64_2 bits [39:36]" , mmfr0, 36, ...);
+ MERGE_FEATURE_4B("TGran4_2 bits [43:40]" , mmfr0, 40, ...);
+ MERGE_FEATURE_4B("ExS bits [47:44]" , mmfr0, 44, ...);
+ */
+ MERGE_FEATURE_4B("FGT bits [59:56]" , mmfr0, 56, Ext::kFGT, Ext::kFGT2);
+ MERGE_FEATURE_4B("ECV bits [63:60]" , mmfr0, 60, Ext::kECV);
+}
+
+ASMJIT_MAYBE_UNUSED
+static inline void detectAArch64FeaturesViaCPUID_AA64MMFR1(CpuInfo& cpu, uint64_t mmfr1) noexcept {
+ // ID_AA64MMFR1_EL1
+ // ================
+
+ MERGE_FEATURE_4B("HAFDBS bits [3:0]" , mmfr1, 0, Ext::kHAFDBS, Ext::kNone, Ext::kHAFT, Ext::kHDBSS);
+ MERGE_FEATURE_4B("VMIDBits bits [7:4]" , mmfr1, 4, Ext::kVMID16);
+ MERGE_FEATURE_4B("VH bits [11:8]" , mmfr1, 8, Ext::kVHE);
+ MERGE_FEATURE_4B("HPDS bits [15:12]" , mmfr1, 12, Ext::kHPDS, Ext::kHPDS2);
+ MERGE_FEATURE_4B("LO bits [19:16]" , mmfr1, 16, Ext::kLOR);
+ MERGE_FEATURE_4B("PAN bits [23:20]" , mmfr1, 20, Ext::kPAN, Ext::kPAN2, Ext::kPAN3);
+ /*
+ MERGE_FEATURE_4B("SpecSEI bits [27:24]" , mmfr1, 24, ...);
+ */
+ MERGE_FEATURE_4B("XNX bits [31:28]" , mmfr1, 28, Ext::kXNX);
+ /*
+ MERGE_FEATURE_4B("TWED bits [35:32]" , mmfr1, 32, ...);
+ MERGE_FEATURE_4B("ETS bits [39:36]" , mmfr1, 36, ...);
+ */
+ MERGE_FEATURE_4B("HCX bits [43:40]" , mmfr1, 40, Ext::kHCX);
+ MERGE_FEATURE_4B("AFP bits [47:44]" , mmfr1, 44, Ext::kAFP);
+ /*
+ MERGE_FEATURE_4B("nTLBPA bits [51:48]" , mmfr1, 48, ...);
+ MERGE_FEATURE_4B("TIDCP1 bits [55:52]" , mmfr1, 52, ...);
+ */
+ MERGE_FEATURE_4B("CMOW bits [59:56]" , mmfr1, 56, Ext::kCMOW);
+ MERGE_FEATURE_4B("ECBHB bits [63:60]" , mmfr1, 60, Ext::kECBHB);
+}
+
+ASMJIT_MAYBE_UNUSED
+static inline void detectAArch64FeaturesViaCPUID_AA64MMFR2(CpuInfo& cpu, uint64_t mmfr2) noexcept {
+ // ID_AA64MMFR2_EL1
+ // ================
+
+ /*
+ MERGE_FEATURE_4B("CnP bits [3:0]" , mmfr2, 0, ...);
+ */
+ MERGE_FEATURE_4B("UAO bits [7:4]" , mmfr2, 4, Ext::kUAO);
+ /*
+ MERGE_FEATURE_4B("LSM bits [11:8]" , mmfr2, 8, ...);
+ MERGE_FEATURE_4B("IESB bits [15:12]" , mmfr2, 12, ...);
+ */
+ MERGE_FEATURE_4B("VARange bits [19:16]" , mmfr2, 16, Ext::kLVA, Ext::kLVA3);
+ MERGE_FEATURE_4B("CCIDX bits [23:20]" , mmfr2, 20, Ext::kCCIDX);
+ MERGE_FEATURE_4B("NV bits [27:24]" , mmfr2, 24, Ext::kNV, Ext::kNV2);
+ /*
+ MERGE_FEATURE_4B("ST bits [31:28]" , mmfr2, 28, ...);
+ */
+ MERGE_FEATURE_4B("AT bits [35:32]" , mmfr2, 32, Ext::kLSE2);
+ /*
+ MERGE_FEATURE_4B("IDS bits [39:36]" , mmfr2, 36, ...);
+ MERGE_FEATURE_4B("FWB bits [43:40]" , mmfr2, 40, ...);
+ MERGE_FEATURE_4B("TTL bits [51:48]" , mmfr2, 48, ...);
+ MERGE_FEATURE_4B("BBM bits [55:52]" , mmfr2, 52, ...);
+ MERGE_FEATURE_4B("EVT bits [59:56]" , mmfr2, 56, ...);
+ MERGE_FEATURE_4B("E0PD bits [63:60]" , mmfr2, 60, ...);
+ */
+}
+
+// Detects features based on the content of ID_AA64ZFR0_EL1 register.
+ASMJIT_MAYBE_UNUSED
+static inline void detectAArch64FeaturesViaCPUID_AA64ZFR0(CpuInfo& cpu, uint64_t zfr0) noexcept {
+ MERGE_FEATURE_4B("SVEver bits [3:0]" , zfr0, 0, Ext::kSVE2, Ext::kSVE2_1);
+ MERGE_FEATURE_4B("AES bits [7:4]" , zfr0, 4, Ext::kSVE_AES, Ext::kSVE_PMULL128);
+ MERGE_FEATURE_4B("BitPerm bits [19:16]" , zfr0, 16, Ext::kSVE_BITPERM);
+ MERGE_FEATURE_4B("BF16 bits [23:20]" , zfr0, 20, Ext::kSVE_BF16, Ext::kSVE_EBF16);
+ MERGE_FEATURE_4B("B16B16 bits [27:24]" , zfr0, 24, Ext::kSVE_B16B16);
+ MERGE_FEATURE_4B("SHA3 bits [35:32]" , zfr0, 32, Ext::kSVE_SHA3);
+ MERGE_FEATURE_4B("SM4 bits [43:40]" , zfr0, 40, Ext::kSVE_SM4);
+ MERGE_FEATURE_4B("I8MM bits [47:44]" , zfr0, 44, Ext::kSVE_I8MM);
+ MERGE_FEATURE_4B("F32MM bits [55:52]" , zfr0, 52, Ext::kSVE_F32MM);
+ MERGE_FEATURE_4B("F64MM bits [59:56]" , zfr0, 56, Ext::kSVE_F64MM);
+}
+
+ASMJIT_MAYBE_UNUSED
+static inline void detectAArch64FeaturesViaCPUID_AA64SMFR0(CpuInfo& cpu, uint64_t smfr0) noexcept {
+ MERGE_FEATURE_1B("SF8DP2 bit [28]" , smfr0, 29, Ext::kSSVE_FP8DOT2);
+ MERGE_FEATURE_1B("SF8DP4 bit [29]" , smfr0, 29, Ext::kSSVE_FP8DOT4);
+ MERGE_FEATURE_1B("SF8FMA bit [30]" , smfr0, 30, Ext::kSSVE_FP8FMA);
+ MERGE_FEATURE_1B("F32F32 bit [32]" , smfr0, 32, Ext::kSME_F32F32);
+ MERGE_FEATURE_1B("BI32I32 bit [33]" , smfr0, 33, Ext::kSME_BI32I32);
+ MERGE_FEATURE_1B("B16F32 bit [34]" , smfr0, 34, Ext::kSME_B16F32);
+ MERGE_FEATURE_1B("F16F32 bit [35]" , smfr0, 35, Ext::kSME_F16F32);
+ MERGE_FEATURE_4S("I8I32 bits [39:36]" , smfr0, 36, 0xF, Ext::kSME_I8I32);
+ MERGE_FEATURE_1B("F8F32 bit [40]" , smfr0, 40, Ext::kSME_F8F32);
+ MERGE_FEATURE_1B("F8F16 bit [41]" , smfr0, 41, Ext::kSME_F8F16);
+ MERGE_FEATURE_1B("F16F16 bit [42]" , smfr0, 42, Ext::kSME_F16F16);
+ MERGE_FEATURE_1B("B16B16 bit [43]" , smfr0, 43, Ext::kSME_B16B16);
+ MERGE_FEATURE_4S("I16I32 bits [47:44]" , smfr0, 44, 0x5, Ext::kSME_I16I32);
+ MERGE_FEATURE_1B("F64F64 bit [48]" , smfr0, 48, Ext::kSME_F64F64);
+ MERGE_FEATURE_4S("I16I64 bits [55:52]" , smfr0, 52, 0xF, Ext::kSME_I16I64);
+ MERGE_FEATURE_4B("SMEver bits [59:56]" , smfr0, 56, Ext::kSME2, Ext::kSME2_1);
+ MERGE_FEATURE_1B("LUTv2 bit [60]" , smfr0, 60, Ext::kSME_LUTv2);
+ MERGE_FEATURE_1B("FA64 bit [63]" , smfr0, 63, Ext::kSME_FA64);
+}
+
+#undef MERGE_FEATURE_4S
+#undef MERGE_FEATURE_4B
+#undef MERGE_FEATURE_2B
+#undef MERGE_FEATURE_1B
+#undef MERGE_FEATURE_NA
+
+// CpuInfo - Detect - ARM - CPU Vendor Features
+// ============================================
+
+// CPU features detection based on Apple family ID.
+enum class AppleFamilyId : uint32_t {
+ // Apple design.
+ kSWIFT = 0x1E2D6381u, // Apple A6/A6X (ARMv7s).
+ kCYCLONE = 0x37A09642u, // Apple A7 (ARMv8.0-A).
+ kTYPHOON = 0x2C91A47Eu, // Apple A8 (ARMv8.0-A).
+ kTWISTER = 0x92FB37C8u, // Apple A9 (ARMv8.0-A).
+ kHURRICANE = 0x67CEEE93u, // Apple A10 (ARMv8.1-A).
+ kMONSOON_MISTRAL = 0xE81E7EF6u, // Apple A11 (ARMv8.2-A).
+ kVORTEX_TEMPEST = 0x07D34B9Fu, // Apple A12 (ARMv8.3-A).
+ kLIGHTNING_THUNDER = 0x462504D2u, // Apple A13 (ARMv8.4-A).
+ kFIRESTORM_ICESTORM = 0x1B588BB3u, // Apple A14/M1 (ARMv8.5-A).
+ kAVALANCHE_BLIZZARD = 0XDA33D83Du, // Apple A15/M2.
+ kEVEREST_SAWTOOTH = 0X8765EDEAu // Apple A16.
+};
+
+ASMJIT_MAYBE_UNUSED
+static ASMJIT_FAVOR_SIZE bool detectARMFeaturesViaAppleFamilyId(CpuInfo& cpu) noexcept {
+ typedef AppleFamilyId Id;
+ CpuFeatures::ARM& features = cpu.features().arm();
+
+ switch (cpu.familyId()) {
+ // Apple A7-A9 (ARMv8.0-A).
+ case uint32_t(Id::kCYCLONE):
+ case uint32_t(Id::kTYPHOON):
+ case uint32_t(Id::kTWISTER):
+ populateARMv8AFeatures(features, 0);
+ features.add(Ext::kAES, Ext::kPMU, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256);
+ return true;
+
+ // Apple A10 (ARMv8.0-A).
+ case uint32_t(Id::kHURRICANE):
+ populateARMv8AFeatures(features, 0);
+ features.add(Ext::kAES, Ext::kCRC32, Ext::kLOR, Ext::kPAN, Ext::kPMU, Ext::kPMULL, Ext::kRDM, Ext::kSHA1,
+ Ext::kSHA256, Ext::kVHE);
+ return true;
+
+ // Apple A11 (ARMv8.2-A).
+ case uint32_t(Id::kMONSOON_MISTRAL):
+ populateARMv8AFeatures(features, 2);
+ features.add(Ext::kAES, Ext::kFP16, Ext::kFP16CONV, Ext::kPMU, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256);
+ return true;
+
+ // Apple A12 (ARMv8.3-A).
+ case uint32_t(Id::kVORTEX_TEMPEST):
+ populateARMv8AFeatures(features, 3);
+ features.add(Ext::kAES, Ext::kFP16, Ext::kFP16CONV, Ext::kPMU, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256);
+ return true;
+
+ // Apple A13 (ARMv8.4-A).
+ case uint32_t(Id::kLIGHTNING_THUNDER):
+ populateARMv8AFeatures(features, 4);
+ features.add(Ext::kAES, Ext::kFHM, Ext::kFP16, Ext::kFP16CONV, Ext::kPMU, Ext::kPMULL, Ext::kSHA1,
+ Ext::kSHA256, Ext::kSHA3, Ext::kSHA512);
+ return true;
+
+ // Apple A14/M1 (ARMv8.5-A).
+ case uint32_t(Id::kFIRESTORM_ICESTORM):
+ populateARMv8AFeatures(features, 4);
+ features.add(Ext::kAES, Ext::kCSV2, Ext::kCSV3, Ext::kDPB2, Ext::kECV, Ext::kFHM, Ext::kFLAGM2,
+ Ext::kFP16, Ext::kFP16CONV, Ext::kFRINTTS, Ext::kPMU, Ext::kPMULL, Ext::kSB,
+ Ext::kSHA1, Ext::kSHA256, Ext::kSHA3, Ext::kSHA512, Ext::kSSBS);
+ return true;
+
+ // Apple A15/M2.
+ case uint32_t(Id::kAVALANCHE_BLIZZARD):
+ populateARMv8AFeatures(features, 6);
+ features.add(Ext::kAES, Ext::kFHM, Ext::kFP16, Ext::kFP16CONV, Ext::kPMU, Ext::kPMULL, Ext::kSHA1,
+ Ext::kSHA256, Ext::kSHA3, Ext::kSHA512);
+ return true;
+
+ // Apple A16.
+ case uint32_t(Id::kEVEREST_SAWTOOTH):
+ populateARMv8AFeatures(features, 6);
+ features.add(Ext::kAES, Ext::kFHM, Ext::kFP16, Ext::kFP16CONV, Ext::kHCX, Ext::kPMU, Ext::kPMULL,
+ Ext::kSHA1, Ext::kSHA256, Ext::kSHA3, Ext::kSHA512);
+ return true;
+
+ default:
+ return false;
+ }
+}
+
+// CpuInfo - Detect - ARM - Compile Flags Features
+// ===============================================
+
+// Detects ARM version by macros defined at compile time. This means that AsmJit will report features forced at
+// compile time that should always be provided by the target CPU. This also means that if we don't provide any
+// means to detect CPU features the features reported by AsmJit will at least not report less features than the
+// target it was compiled to.
+
+#if ASMJIT_ARCH_ARM == 32
+ASMJIT_MAYBE_UNUSED
+static ASMJIT_FAVOR_SIZE void detectAArch32FeaturesViaCompilerFlags(CpuInfo& cpu) noexcept {
+ DebugUtils::unused(cpu);
+
+ // ARM targets have no baseline at the moment.
+#if defined(__ARM_ARCH_7A__)
+ cpu.addFeature(CpuFeatures::ARM::kARMv7);
+#endif
+
+#if defined(__ARM_ARCH_8A__)
+ cpu.addFeature(CpuFeatures::ARM::kARMv8a);
+#endif
+
+#if defined(__TARGET_ARCH_THUMB)
+ cpu.addFeature(CpuFeatures::ARM::kTHUMB);
+#if __TARGET_ARCH_THUMB >= 4
+ cpu.addFeature(CpuFeatures::ARM::kTHUMBv2);
+#endif
+#endif
+
+#if defined(__ARM_FEATURE_FMA)
+ cpu.addFeature(Ext::kFP);
+#endif
+
+#if defined(__ARM_NEON)
+ cpu.addFeature(Ext::kASIMD);
+#endif
+
+#if defined(__ARM_FEATURE_IDIV) && defined(__TARGET_ARCH_THUMB)
+ cpu.addFeature(Ext::kIDIVT);
+#endif
+#if defined(__ARM_FEATURE_IDIV) && !defined(__TARGET_ARCH_THUMB)
+ cpu.addFeature(Ext::kIDIVA);
+#endif
+}
+#endif // ASMJIT_ARCH_ARM == 32
+
+#if ASMJIT_ARCH_ARM == 64
+ASMJIT_MAYBE_UNUSED
+static ASMJIT_FAVOR_SIZE void detectAArch64FeaturesViaCompilerFlags(CpuInfo& cpu) noexcept {
+ DebugUtils::unused(cpu);
+
+#if defined(__ARM_ARCH_9_5A__)
+ populateARMv9AFeatures(cpu.features().arm(), 5);
+#elif defined(__ARM_ARCH_9_4A__)
+ populateARMv9AFeatures(cpu.features().arm(), 4);
+#elif defined(__ARM_ARCH_9_3A__)
+ populateARMv9AFeatures(cpu.features().arm(), 3);
+#elif defined(__ARM_ARCH_9_2A__)
+ populateARMv9AFeatures(cpu.features().arm(), 2);
+#elif defined(__ARM_ARCH_9_1A__)
+ populateARMv9AFeatures(cpu.features().arm(), 1);
+#elif defined(__ARM_ARCH_9A__)
+ populateARMv9AFeatures(cpu.features().arm(), 0);
+#elif defined(__ARM_ARCH_8_9A__)
+ populateARMv8AFeatures(cpu.features().arm(), 9);
+#elif defined(__ARM_ARCH_8_8A__)
+ populateARMv8AFeatures(cpu.features().arm(), 8);
+#elif defined(__ARM_ARCH_8_7A__)
+ populateARMv8AFeatures(cpu.features().arm(), 7);
+#elif defined(__ARM_ARCH_8_6A__)
+ populateARMv8AFeatures(cpu.features().arm(), 6);
+#elif defined(__ARM_ARCH_8_5A__)
+ populateARMv8AFeatures(cpu.features().arm(), 5);
+#elif defined(__ARM_ARCH_8_4A__)
+ populateARMv8AFeatures(cpu.features().arm(), 4);
+#elif defined(__ARM_ARCH_8_3A__)
+ populateARMv8AFeatures(cpu.features().arm(), 3);
+#elif defined(__ARM_ARCH_8_2A__)
+ populateARMv8AFeatures(cpu.features().arm(), 2);
+#elif defined(__ARM_ARCH_8_1A__)
+ populateARMv8AFeatures(cpu.features().arm(), 1);
+#else
+ populateARMv8AFeatures(cpu.features().arm(), 0);
+#endif
+
+#if defined(__ARM_FEATURE_AES)
+ cpu.addFeature(Ext::kAES);
+#endif
+
+#if defined(__ARM_FEATURE_BF16_SCALAR_ARITHMETIC) && defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC)
+ cpu.addFeature(Ext::kBF16);
+#endif
+
+#if defined(__ARM_FEATURE_CRC32)
+ cpu.addFeature(Ext::kCRC32);
+#endif
+
+#if defined(__ARM_FEATURE_CRYPTO)
+ cpu.addFeature(Ext::kAES, Ext::kSHA1, Ext::kSHA256);
+#endif
+
+#if defined(__ARM_FEATURE_DOTPROD)
+ cpu.addFeature(Ext::kDOTPROD);
+#endif
+
+#if defined(__ARM_FEATURE_FP16FML) || defined(__ARM_FEATURE_FP16_FML)
+ cpu.addFeature(Ext::kFHM);
+#endif
+
+#if defined(__ARM_FEATURE_FP16_SCALAR_ARITHMETIC)
+ cpu.addFeature(Ext::kFP16);
+#endif
+
+#if defined(__ARM_FEATURE_FRINT)
+ cpu.addFeature(Ext::kFRINTTS);
+#endif
+
+#if defined(__ARM_FEATURE_JCVT)
+ cpu.addFeature(Ext::kJSCVT);
+#endif
+
+#if defined(__ARM_FEATURE_MATMUL_INT8)
+ cpu.addFeature(Ext::kI8MM);
+#endif
+
+#if defined(__ARM_FEATURE_ATOMICS)
+ cpu.addFeature(Ext::kLSE);
+#endif
+
+#if defined(__ARM_FEATURE_MEMORY_TAGGING)
+ cpu.addFeature(Ext::kMTE);
+#endif
+
+#if defined(__ARM_FEATURE_QRDMX)
+ cpu.addFeature(Ext::kRDM);
+#endif
+
+#if defined(__ARM_FEATURE_RNG)
+ cpu.addFeature(Ext::kRNG);
+#endif
+
+#if defined(__ARM_FEATURE_SHA2)
+ cpu.addFeature(Ext::kSHA256);
+#endif
+
+#if defined(__ARM_FEATURE_SHA3)
+ cpu.addFeature(Ext::kSHA3);
+#endif
+
+#if defined(__ARM_FEATURE_SHA512)
+ cpu.addFeature(Ext::kSHA512);
+#endif
+
+#if defined(__ARM_FEATURE_SM3)
+ cpu.addFeature(Ext::kSM3);
+#endif
+
+#if defined(__ARM_FEATURE_SM4)
+ cpu.addFeature(Ext::kSM4);
+#endif
+
+#if defined(__ARM_FEATURE_SVE) || defined(__ARM_FEATURE_SVE_VECTOR_OPERATORS)
+ cpu.addFeature(Ext::kSVE);
+#endif
+
+#if defined(__ARM_FEATURE_SVE_MATMUL_INT8)
+ cpu.addFeature(Ext::kSVE_I8MM);
+#endif
+
+#if defined(__ARM_FEATURE_SVE_MATMUL_FP32)
+ cpu.addFeature(Ext::kSVE_F32MM);
+#endif
+
+#if defined(__ARM_FEATURE_SVE_MATMUL_FP64)
+ cpu.addFeature(Ext::kSVE_F64MM);
+#endif
+
+#if defined(__ARM_FEATURE_SVE2)
+ cpu.addFeature(Ext::kSVE2);
+#endif
+
+#if defined(__ARM_FEATURE_SVE2_AES)
+ cpu.addFeature(Ext::kSVE_AES);
+#endif
+
+#if defined(__ARM_FEATURE_SVE2_BITPERM)
+ cpu.addFeature(Ext::kSVE_BITPERM);
+#endif
+
+#if defined(__ARM_FEATURE_SVE2_SHA3)
+ cpu.addFeature(Ext::kSVE_SHA3);
+#endif
+
+#if defined(__ARM_FEATURE_SVE2_SM4)
+ cpu.addFeature(Ext::kSVE_SM4);
+#endif
+
+#if defined(__ARM_FEATURE_TME)
+ cpu.addFeature(Ext::kTME);
+#endif
+}
+#endif // ASMJIT_ARCH_ARM == 64
+
+ASMJIT_MAYBE_UNUSED
+static ASMJIT_FAVOR_SIZE void detectARMFeaturesViaCompilerFlags(CpuInfo& cpu) noexcept {
+#if ASMJIT_ARCH_ARM == 32
+ detectAArch32FeaturesViaCompilerFlags(cpu);
+#else
+ detectAArch64FeaturesViaCompilerFlags(cpu);
+#endif // ASMJIT_ARCH_ARM
+}
+
+// CpuInfo - Detect - ARM - Post Processing ARM Features
+// =====================================================
+
+// Postprocesses AArch32 features.
+ASMJIT_MAYBE_UNUSED
+static ASMJIT_FAVOR_SIZE void postProcessAArch32Features(CpuFeatures::ARM& features) noexcept {
+ DebugUtils::unused(features);
+}
+
+// Postprocesses AArch64 features.
+//
+// The only reason to use this function is to deduce some flags from others.
+ASMJIT_MAYBE_UNUSED
+static ASMJIT_FAVOR_SIZE void postProcessAArch64Features(CpuFeatures::ARM& features) noexcept {
+ if (features.hasFP16())
+ features.add(Ext::kFP16CONV);
+
+ if (features.hasMTE3())
+ features.add(Ext::kMTE2);
+
+ if (features.hasMTE2())
+ features.add(Ext::kMTE);
+
+ if (features.hasSSBS2())
+ features.add(Ext::kSSBS);
+}
+
+ASMJIT_MAYBE_UNUSED
+static ASMJIT_FAVOR_SIZE void postProcessARMCpuInfo(CpuInfo& cpu) noexcept {
+#if ASMJIT_ARCH_ARM == 32
+ postProcessAArch32Features(cpu.features().arm());
+#else
+ postProcessAArch64Features(cpu.features().arm());
+#endif // ASMJIT_ARCH_ARM
+}
+
+// CpuInfo - Detect - ARM - Detect by Reading CPUID Registers
+// ==========================================================
+
+// Support CPUID-based detection on AArch64.
+#if defined(ASMJIT_ARM_DETECT_VIA_CPUID)
+
+// Since the register ID is encoded with the instruction we have to create a function for each register ID to read.
+#define ASMJIT_AARCH64_DEFINE_CPUID_READ_FN(func, regId) \
+ASMJIT_MAYBE_UNUSED \
+static inline uint64_t func() noexcept { \
+ uint64_t output; \
+ __asm__ __volatile__("mrs %0, " #regId : "=r"(output)); \
+ return output; \
+}
+
+// NOTE: Older tools don't know the IDs. For example Ubuntu on RPI (GCC 9) won't compile ID_AA64ISAR2_EL1 in 2023.
+ASMJIT_AARCH64_DEFINE_CPUID_READ_FN(aarch64ReadPFR0, ID_AA64PFR0_EL1)
+ASMJIT_AARCH64_DEFINE_CPUID_READ_FN(aarch64ReadPFR1, ID_AA64PFR1_EL1)
+ASMJIT_AARCH64_DEFINE_CPUID_READ_FN(aarch64ReadISAR0, ID_AA64ISAR0_EL1)
+ASMJIT_AARCH64_DEFINE_CPUID_READ_FN(aarch64ReadISAR1, ID_AA64ISAR1_EL1)
+ASMJIT_AARCH64_DEFINE_CPUID_READ_FN(aarch64ReadISAR2, S3_0_C0_C6_2) // ID_AA64ISAR2_EL1
+ASMJIT_AARCH64_DEFINE_CPUID_READ_FN(aarch64ReadZFR0, S3_0_C0_C4_4) // ID_AA64ZFR0_EL1
+
+#undef ASMJIT_AARCH64_DEFINE_CPUID_READ_FN
+
+// Detects AArch64 features by reading CPUID bits directly from CPUID registers. This is the most reliable method
+// as the OS doesn't have to know all supported extensions this way (if there is something missing in HWCAPS then
+// there is no way to detect such feature without reading CPUID bits).
+//
+// This function uses MSR instructions, which means that it reads registers that cannot be read in user-mode. The
+// OS typically implements this feature by handling SIGILL internally and providing a filtered content of these
+// registers back to the user - at least this is what Linux documentation states - everything implementation
+// dependent is zeroed, only the bits that are used for CPU feature identification would be present.
+//
+// References:
+// - https://docs.kernel.org/arch/arm64/cpu-feature-registers.html
+ASMJIT_MAYBE_UNUSED
+static ASMJIT_FAVOR_SIZE void detectAArch64FeaturesViaCPUID(CpuInfo& cpu) noexcept {
+ populateBaseARMFeatures(cpu);
+
+ detectAArch64FeaturesViaCPUID_AA64PFR0_AA64PFR1(cpu,
+ aarch64ReadPFR0(),
+ aarch64ReadPFR1());
+
+ detectAArch64FeaturesViaCPUID_AA64ISAR0_AA64ISAR1(cpu,
+ aarch64ReadISAR0(),
+ aarch64ReadISAR1());
+
+ // TODO: Fix this on FreeBSD - I don't know what kernel version allows to access the registers below...
+
+#if defined(__linux__)
+ UNameKernelVersion kVer = getUNameKernelVersion();
+
+ // Introduced in Linux 4.19 by "arm64: add ID_AA64ISAR2_EL1 sys register"), so we want at least 4.20.
+ if (kVer.atLeast(4, 20)) {
+ detectAArch64FeaturesViaCPUID_AA64ISAR2(cpu, aarch64ReadISAR2());
+ }
+
+ // Introduced in Linux 5.10 by "arm64: Expose SVE2 features for userspace", so we want at least 5.11.
+ if (kVer.atLeast(5, 11) && cpu.features().arm().hasAny(Ext::kSVE, Ext::kSME)) {
+ // Only read CPU_ID_AA64ZFR0 when either SVE or SME is available.
+ detectAArch64FeaturesViaCPUID_AA64ZFR0(cpu, aarch64ReadZFR0());
+ }
+#endif
+}
+
+#endif // ASMJIT_ARM_DETECT_VIA_CPUID
+
+// CpuInfo - Detect - ARM - Detect by Windows API
+// ==============================================
+
+#if defined(_WIN32)
+struct WinPFPMapping {
+ uint8_t featureId;
+ uint8_t pfpFeatureId;
+};
+
+static ASMJIT_FAVOR_SIZE void detectPFPFeatures(CpuInfo& cpu, const WinPFPMapping* mapping, size_t size) noexcept {
+ for (size_t i = 0; i < size; i++) {
+ if (::IsProcessorFeaturePresent(mapping[i].pfpFeatureId)) {
+ cpu.addFeature(mapping[i].featureId);
+ }
+ }
+}
+
+//! Detect ARM CPU features on Windows.
+//!
+//! The detection is based on `IsProcessorFeaturePresent()` API call.
+static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept {
+ cpu._wasDetected = true;
+ populateBaseARMFeatures(cpu);
+
+ CpuFeatures::ARM& features = cpu.features().arm();
+
+ // Win32 for ARM requires ARMv7 with DSP extensions, VFPv3 (FP), and uses THUMBv2 by default.
+#if ASMJIT_ARCH_ARM == 32
+ features.add(Ext::kTHUMB);
+ features.add(Ext::kTHUMBv2);
+ features.add(Ext::kARMv6);
+ features.add(Ext::kARMv7);
+ features.add(Ext::kEDSP);
+#endif
+
+ // Windows for ARM requires FP and ASIMD.
+ features.add(Ext::kFP);
+ features.add(Ext::kASIMD);
+
+ // Detect additional CPU features by calling `IsProcessorFeaturePresent()`.
+ static const WinPFPMapping mapping[] = {
+#if ASMJIT_ARCH_ARM == 32
+ { uint8_t(Ext::kVFP_D32) , 18 }, // PF_ARM_VFP_32_REGISTERS_AVAILABLE
+ { uint8_t(Ext::kIDIVT) , 24 }, // PF_ARM_DIVIDE_INSTRUCTION_AVAILABLE
+ { uint8_t(Ext::kFMAC) , 27 }, // PF_ARM_FMAC_INSTRUCTIONS_AVAILABLE
+ { uint8_t(Ext::kARMv8a) , 29 }, // PF_ARM_V8_INSTRUCTIONS_AVAILABLE
+#endif
+ { uint8_t(Ext::kAES) , 30 }, // PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE
+ { uint8_t(Ext::kCRC32) , 31 }, // PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE
+ { uint8_t(Ext::kLSE) , 34 }, // PF_ARM_V81_ATOMIC_INSTRUCTIONS_AVAILABLE
+ { uint8_t(Ext::kDOTPROD) , 43 }, // PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE
+ { uint8_t(Ext::kJSCVT) , 44 }, // PF_ARM_V83_JSCVT_INSTRUCTIONS_AVAILABLE
+ { uint8_t(Ext::kLRCPC) , 45 } // PF_ARM_V83_LRCPC_INSTRUCTIONS_AVAILABLE
+ };
+ detectPFPFeatures(cpu, mapping, ASMJIT_ARRAY_SIZE(mapping));
+
+ // Windows can only report ARMv8A at the moment.
+ if (features.hasARMv8a()) {
+ populateARMv8AFeatures(cpu.features().arm(), 0);
+ }
+
+ // Windows provides several instructions under a single flag:
+ if (features.hasAES()) {
+ features.add(Ext::kPMULL, Ext::kSHA1, Ext::kSHA256);
+ }
+
+ postProcessARMCpuInfo(cpu);
+}
+
+// CpuInfo - Detect - ARM - Detect by Reading HWCAPS
+// =================================================
+
+#elif defined(ASMJIT_ARM_DETECT_VIA_HWCAPS)
+
+#ifndef AT_HWCAP
+ #define AT_HWCAP 16
+#endif // !AT_HWCAP
+
+#ifndef AT_HWCAP2
+ #define AT_HWCAP2 26
+#endif // !AT_HWCAP2
+
+#if defined(__linux__)
+static void getAuxValues(unsigned long* vals, const unsigned long* tags, size_t count) noexcept {
+ for (size_t i = 0; i < count; i++) {
+ vals[i] = getauxval(tags[i]);
+ }
+}
+#elif defined(__FreeBSD__)
+static void getAuxValues(unsigned long* vals, const unsigned long* tags, size_t count) noexcept {
+ for (size_t i = 0; i < count; i++) {
+ unsigned long result = 0;
+ if (elf_aux_info(int(tags[i]), &result, int(sizeof(unsigned long))) != 0)
+ result = 0;
+ vals[i] = result;
+ }
+}
+#else
+#error "[asmjit] getAuxValues() - Unsupported OS."
+#endif
+
+struct HWCapMapping {
+ uint8_t featureId;
+ uint8_t hwCapBit;
+};
+
+static const unsigned long hwCapTags[2] = { AT_HWCAP, AT_HWCAP2 };
+
+static ASMJIT_FAVOR_SIZE void mergeHWCaps(CpuInfo& cpu, unsigned long mask, const HWCapMapping* mapping, size_t size) noexcept {
+ for (size_t i = 0; i < size; i++) {
+ cpu.features().addIf(Support::bitTest(mask, mapping[i].hwCapBit), mapping[i].featureId);
+ }
+}
+
+#if ASMJIT_ARCH_ARM == 32
+
+// Reference:
+// - https://github.com/torvalds/linux/blob/master/arch/arm/include/uapi/asm/hwcap.h
+static const HWCapMapping hwCapMapping[] = {
+ { uint8_t(Ext::kEDSP) , 7 }, // HWCAP_EDSP
+ { uint8_t(Ext::kASIMD) , 12 }, // HWCAP_NEON
+ { uint8_t(Ext::kFP) , 13 }, // HWCAP_VFPv3
+ { uint8_t(Ext::kFMAC) , 16 }, // HWCAP_VFPv4
+ { uint8_t(Ext::kIDIVA) , 17 }, // HWCAP_IDIVA
+ { uint8_t(Ext::kIDIVT) , 18 }, // HWCAP_IDIVT
+ { uint8_t(Ext::kVFP_D32) , 19 }, // HWCAP_VFPD32
+ { uint8_t(Ext::kFP16CONV) , 22 }, // HWCAP_FPHP
+ { uint8_t(Ext::kFP16) , 23 }, // HWCAP_ASIMDHP
+ { uint8_t(Ext::kDOTPROD) , 24 }, // HWCAP_ASIMDDP
+ { uint8_t(Ext::kFHM) , 25 }, // HWCAP_ASIMDFHM
+ { uint8_t(Ext::kBF16) , 26 }, // HWCAP_ASIMDBF16
+ { uint8_t(Ext::kI8MM) , 27 } // HWCAP_I8MM
+};
+
+static const HWCapMapping hwCap2Mapping[] = {
+ { uint8_t(Ext::kAES) , 0 }, // HWCAP2_AES
+ { uint8_t(Ext::kPMULL) , 1 }, // HWCAP2_PMULL
+ { uint8_t(Ext::kSHA1) , 2 }, // HWCAP2_SHA1
+ { uint8_t(Ext::kSHA256) , 3 }, // HWCAP2_SHA2
+ { uint8_t(Ext::kCRC32) , 4 }, // HWCAP2_CRC32
+ { uint8_t(Ext::kSB) , 5 }, // HWCAP2_SB
+ { uint8_t(Ext::kSSBS) , 6 } // HWCAP2_SSBS
+};
+
+static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept {
+ cpu._wasDetected = true;
+ populateBaseARMFeatures(cpu);
+
+ unsigned long hwCapMasks[2] {};
+ getAuxValues(hwCapMasks, hwCapTags, 2u);
+
+ mergeHWCaps(cpu, hwCapMasks[0], hwCapMapping, ASMJIT_ARRAY_SIZE(hwCapMapping));
+ mergeHWCaps(cpu, hwCapMasks[1], hwCap2Mapping, ASMJIT_ARRAY_SIZE(hwCap2Mapping));
+
+ CpuFeatures::ARM& features = cpu.features().arm();
+
+ // ARMv7 provides FP|ASIMD.
+ if (features.hasFP() || features.hasASIMD())
+ features.add(CpuFeatures::ARM::kARMv7);
+
+ // ARMv8 provives AES, CRC32, PMULL, SHA1, and SHA256.
+ if (features.hasAES() || features.hasCRC32() || features.hasPMULL() || features.hasSHA1() || features.hasSHA256())
+ features.add(CpuFeatures::ARM::kARMv8a);
+
+ postProcessARMCpuInfo(cpu);
+}
+
+#else
+
+// Reference:
+// - https://docs.kernel.org/arch/arm64/elf_hwcaps.html
+// - https://github.com/torvalds/linux/blob/master/arch/arm64/include/uapi/asm/hwcap.h
+static const HWCapMapping hwCapMapping[] = {
+ { uint8_t(Ext::kFP) , 0 }, // HWCAP_FP
+ { uint8_t(Ext::kASIMD) , 1 }, // HWCAP_ASIMD
+ /*
+ { uint8_t(Ext::k) , 2 }, // HWCAP_EVTSTRM
+ */
+ { uint8_t(Ext::kAES) , 3 }, // HWCAP_AES
+ { uint8_t(Ext::kPMULL) , 4 }, // HWCAP_PMULL
+ { uint8_t(Ext::kSHA1) , 5 }, // HWCAP_SHA1
+ { uint8_t(Ext::kSHA256) , 6 }, // HWCAP_SHA2
+ { uint8_t(Ext::kCRC32) , 7 }, // HWCAP_CRC32
+ { uint8_t(Ext::kLSE) , 8 }, // HWCAP_ATOMICS
+ { uint8_t(Ext::kFP16CONV) , 9 }, // HWCAP_FPHP
+ { uint8_t(Ext::kFP16) , 10 }, // HWCAP_ASIMDHP
+ { uint8_t(Ext::kCPUID) , 11 }, // HWCAP_CPUID
+ { uint8_t(Ext::kRDM) , 12 }, // HWCAP_ASIMDRDM
+ { uint8_t(Ext::kJSCVT) , 13 }, // HWCAP_JSCVT
+ { uint8_t(Ext::kFCMA) , 14 }, // HWCAP_FCMA
+ { uint8_t(Ext::kLRCPC) , 15 }, // HWCAP_LRCPC
+ { uint8_t(Ext::kDPB) , 16 }, // HWCAP_DCPOP
+ { uint8_t(Ext::kSHA3) , 17 }, // HWCAP_SHA3
+ { uint8_t(Ext::kSM3) , 18 }, // HWCAP_SM3
+ { uint8_t(Ext::kSM4) , 19 }, // HWCAP_SM4
+ { uint8_t(Ext::kDOTPROD) , 20 }, // HWCAP_ASIMDDP
+ { uint8_t(Ext::kSHA512) , 21 }, // HWCAP_SHA512
+ { uint8_t(Ext::kSVE) , 22 }, // HWCAP_SVE
+ { uint8_t(Ext::kFHM) , 23 }, // HWCAP_ASIMDFHM
+ { uint8_t(Ext::kDIT) , 24 }, // HWCAP_DIT
+ { uint8_t(Ext::kLSE2) , 25 }, // HWCAP_USCAT
+ { uint8_t(Ext::kLRCPC2) , 26 }, // HWCAP_ILRCPC
+ { uint8_t(Ext::kFLAGM) , 27 }, // HWCAP_FLAGM
+ { uint8_t(Ext::kSSBS) , 28 }, // HWCAP_SSBS
+ { uint8_t(Ext::kSB) , 29 } // HWCAP_SB
+ /*
+ { uint8_t(Ext::k) , 30 }, // HWCAP_PACA
+ { uint8_t(Ext::k) , 31 } // HWCAP_PACG
+ */
+};
+
+static const HWCapMapping hwCap2Mapping[] = {
+ { uint8_t(Ext::kDPB2) , 0 }, // HWCAP2_DCPODP
+ { uint8_t(Ext::kSVE2) , 1 }, // HWCAP2_SVE2
+ { uint8_t(Ext::kSVE_AES) , 2 }, // HWCAP2_SVEAES
+ { uint8_t(Ext::kSVE_PMULL128) , 3 }, // HWCAP2_SVEPMULL
+ { uint8_t(Ext::kSVE_BITPERM) , 4 }, // HWCAP2_SVEBITPERM
+ { uint8_t(Ext::kSVE_SHA3) , 5 }, // HWCAP2_SVESHA3
+ { uint8_t(Ext::kSVE_SM4) , 6 }, // HWCAP2_SVESM4
+ { uint8_t(Ext::kFLAGM2) , 7 }, // HWCAP2_FLAGM2
+ { uint8_t(Ext::kFRINTTS) , 8 }, // HWCAP2_FRINT
+ { uint8_t(Ext::kSVE_I8MM) , 9 }, // HWCAP2_SVEI8MM
+ { uint8_t(Ext::kSVE_F32MM) , 10 }, // HWCAP2_SVEF32MM
+ { uint8_t(Ext::kSVE_F64MM) , 11 }, // HWCAP2_SVEF64MM
+ { uint8_t(Ext::kSVE_BF16) , 12 }, // HWCAP2_SVEBF16
+ { uint8_t(Ext::kI8MM) , 13 }, // HWCAP2_I8MM
+ { uint8_t(Ext::kBF16) , 14 }, // HWCAP2_BF16
+ { uint8_t(Ext::kDGH) , 15 }, // HWCAP2_DGH
+ { uint8_t(Ext::kRNG) , 16 }, // HWCAP2_RNG
+ { uint8_t(Ext::kBTI) , 17 }, // HWCAP2_BTI
+ { uint8_t(Ext::kMTE) , 18 }, // HWCAP2_MTE
+ { uint8_t(Ext::kECV) , 19 }, // HWCAP2_ECV
+ { uint8_t(Ext::kAFP) , 20 }, // HWCAP2_AFP
+ { uint8_t(Ext::kRPRES) , 21 }, // HWCAP2_RPRES
+ { uint8_t(Ext::kMTE3) , 22 }, // HWCAP2_MTE3
+ { uint8_t(Ext::kSME) , 23 }, // HWCAP2_SME
+ { uint8_t(Ext::kSME_I16I64) , 24 }, // HWCAP2_SME_I16I64
+ { uint8_t(Ext::kSME_F64F64) , 25 }, // HWCAP2_SME_F64F64
+ { uint8_t(Ext::kSME_I8I32) , 26 }, // HWCAP2_SME_I8I32
+ { uint8_t(Ext::kSME_F16F32) , 27 }, // HWCAP2_SME_F16F32
+ { uint8_t(Ext::kSME_B16F32) , 28 }, // HWCAP2_SME_B16F32
+ { uint8_t(Ext::kSME_F32F32) , 29 }, // HWCAP2_SME_F32F32
+ { uint8_t(Ext::kSME_FA64) , 30 }, // HWCAP2_SME_FA64
+ { uint8_t(Ext::kWFXT) , 31 }, // HWCAP2_WFXT
+ { uint8_t(Ext::kEBF16) , 32 }, // HWCAP2_EBF16
+ { uint8_t(Ext::kSVE_EBF16) , 33 }, // HWCAP2_SVE_EBF16
+ { uint8_t(Ext::kCSSC) , 34 }, // HWCAP2_CSSC
+ { uint8_t(Ext::kRPRFM) , 35 }, // HWCAP2_RPRFM
+ { uint8_t(Ext::kSVE2_1) , 36 }, // HWCAP2_SVE2P1
+ { uint8_t(Ext::kSME2) , 37 }, // HWCAP2_SME2
+ { uint8_t(Ext::kSME2_1) , 38 }, // HWCAP2_SME2P1
+ { uint8_t(Ext::kSME_I16I32) , 39 }, // HWCAP2_SME_I16I32
+ { uint8_t(Ext::kSME_BI32I32) , 40 }, // HWCAP2_SME_BI32I32
+ { uint8_t(Ext::kSME_B16B16) , 41 }, // HWCAP2_SME_B16B16
+ { uint8_t(Ext::kSME_F16F16) , 42 }, // HWCAP2_SME_F16F16
+ { uint8_t(Ext::kMOPS) , 43 }, // HWCAP2_MOPS
+ { uint8_t(Ext::kHBC) , 44 }, // HWCAP2_HBC
+ { uint8_t(Ext::kSVE_B16B16) , 45 }, // HWCAP2_SVE_B16B16
+ { uint8_t(Ext::kLRCPC3) , 46 }, // HWCAP2_LRCPC3
+ { uint8_t(Ext::kLSE128) , 47 }, // HWCAP2_LSE128
+};
+
+static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept {
+ cpu._wasDetected = true;
+ populateBaseARMFeatures(cpu);
+
+ unsigned long hwCapMasks[2] {};
+ getAuxValues(hwCapMasks, hwCapTags, 2u);
+
+ mergeHWCaps(cpu, hwCapMasks[0], hwCapMapping, ASMJIT_ARRAY_SIZE(hwCapMapping));
+ mergeHWCaps(cpu, hwCapMasks[1], hwCap2Mapping, ASMJIT_ARRAY_SIZE(hwCap2Mapping));
+
+#if defined(ASMJIT_ARM_DETECT_VIA_CPUID)
+ if (cpu.features().arm().hasCPUID()) {
+ detectAArch64FeaturesViaCPUID(cpu);
+ return;
+ }
+#endif // ASMJIT_ARM_DETECT_VIA_CPUID
+
+ postProcessARMCpuInfo(cpu);
+}
+
+#endif // ASMJIT_ARCH_ARM
+
+// CpuInfo - Detect - ARM - Detect by OpenBSD API That Reads CPUID
+// ===============================================================
+
+#elif defined(__OpenBSD__) && ASMJIT_ARCH_ARM >= 64
+
+// Supported CPUID registers on OpenBSD (CTL_MACHDEP definitions):
+// - https://github.com/openbsd/src/blob/master/sys/arch/arm64/include/cpu.h
+enum class OpenBSDAArch64CPUID {
+ kAA64ISAR0 = 2,
+ kAA64ISAR1 = 3,
+ kAA64ISAR2 = 4,
+ kAA64MMFR0 = 5,
+ kAA64MMFR1 = 6,
+ kAA64MMFR2 = 7,
+ kAA64PFR0 = 8,
+ kAA64PFR1 = 9,
+ kAA64SMFR0 = 10,
+ kAA64ZFR0 = 11
+};
+
+static uint64_t openbsdReadAArch64CPUID(OpenBSDAArch64CPUID id) noexcept {
+ uint64_t bits = 0;
+ size_t size = sizeof(bits);
+ int name[2] = { CTL_MACHDEP, int(id) };
+
+ return (sysctl(name, 2, &bits, &size, NULL, 0) < 0) ? uint64_t(0) : bits;
+}
+
+static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept {
+ typedef OpenBSDAArch64CPUID ID;
+
+ populateBaseARMFeatures(cpu);
+
+ detectAArch64FeaturesViaCPUID_AA64PFR0_AA64PFR1(cpu,
+ openbsdReadAArch64CPUID(ID::kAA64PFR0),
+ openbsdReadAArch64CPUID(ID::kAA64PFR1));
+
+ detectAArch64FeaturesViaCPUID_AA64ISAR0_AA64ISAR1(cpu,
+ openbsdReadAArch64CPUID(ID::kAA64ISAR0),
+ openbsdReadAArch64CPUID(ID::kAA64ISAR1));
+
+ detectAArch64FeaturesViaCPUID_AA64ISAR2(cpu, openbsdReadAArch64CPUID(ID::kAA64ISAR2));
+ detectAArch64FeaturesViaCPUID_AA64MMFR0(cpu, openbsdReadAArch64CPUID(ID::kAA64MMFR0));
+ detectAArch64FeaturesViaCPUID_AA64MMFR1(cpu, openbsdReadAArch64CPUID(ID::kAA64MMFR1));
+ detectAArch64FeaturesViaCPUID_AA64MMFR2(cpu, openbsdReadAArch64CPUID(ID::kAA64MMFR2));
+
+ // Only read CPU_ID_AA64ZFR0 when either SVE or SME is available.
+ if (cpu.features().arm().hasAny(Ext::kSVE, Ext::kSME)) {
+ detectAArch64FeaturesViaCPUID_AA64ZFR0(cpu, openbsdReadAArch64CPUID(ID::kAA64ZFR0));
+
+ if (cpu.features().arm().hasSME())
+ detectAArch64FeaturesViaCPUID_AA64SMFR0(cpu, openbsdReadAArch64CPUID(ID::kAA64SMFR0));
+ }
+
+ postProcessARMCpuInfo(cpu);
+}
+
+// CpuInfo - Detect - ARM - Detect by Apple API (sysctlbyname)
+// ===========================================================
+
+#elif defined(__APPLE__)
+
+enum class AppleFeatureType : uint8_t {
+ kHWOptional,
+ kHWOptionalArmFEAT
+};
+
+struct AppleFeatureMapping {
+ AppleFeatureType type;
+ char name[18];
+ uint8_t featureId;
+};
+
+template<typename T>
+static inline bool appleSysctlByName(const char* sysctlName, T* dst, size_t size = sizeof(T)) noexcept {
+ return sysctlbyname(sysctlName, dst, &size, nullptr, 0) == 0;
+}
+
+static ASMJIT_FAVOR_SIZE long appleDetectARMFeatureViaSysctl(AppleFeatureType type, const char* featureName) noexcept {
+ static const char hwOptionalPrefix[] = "hw.optional.";
+ static const char hwOptionalArmFeatPrefix[] = "hw.optional.arm.FEAT_";
+
+ char sysctlName[128];
+
+ const char* prefix = type == AppleFeatureType::kHWOptional ? hwOptionalPrefix : hwOptionalArmFeatPrefix;
+ size_t prefixSize = (type == AppleFeatureType::kHWOptional ? sizeof(hwOptionalPrefix) : sizeof(hwOptionalArmFeatPrefix)) - 1u;
+ size_t featureNameSize = strlen(featureName);
-#if defined(ASMJIT_BUILD_X86) && ASMJIT_ARCH_X86
-namespace x86 { void detectCpu(CpuInfo& cpu) noexcept; }
+ if (featureNameSize < 128 - prefixSize) {
+ memcpy(sysctlName, prefix, prefixSize);
+ memcpy(sysctlName + prefixSize, featureName, featureNameSize + 1u); // Include NULL terminator.
+
+ long val = 0;
+ if (appleSysctlByName<long>(sysctlName, &val))
+ return val;
+ }
+
+ return 0;
+}
+
+static ASMJIT_FAVOR_SIZE void appleDetectARMFeaturesViaSysctl(CpuInfo& cpu) noexcept {
+ typedef AppleFeatureType FT;
+
+ // Based on:
+ // - https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics
+ static const AppleFeatureMapping mappings[] = {
+ // Determine Advanced SIMD and Floating Point Capabilities:
+ { FT::kHWOptional , "AdvSIMD_HPFPCvt", uint8_t(Ext::kFP16CONV) },
+ { FT::kHWOptional , "neon_hpfp" , uint8_t(Ext::kFP16CONV) },
+ { FT::kHWOptionalArmFEAT, "BF16" , uint8_t(Ext::kBF16) },
+ { FT::kHWOptionalArmFEAT, "DotProd" , uint8_t(Ext::kDOTPROD) },
+ { FT::kHWOptionalArmFEAT, "FCMA" , uint8_t(Ext::kFCMA) },
+ { FT::kHWOptional , "armv8_3_compnum", uint8_t(Ext::kFCMA) },
+ { FT::kHWOptionalArmFEAT, "FHM" , uint8_t(Ext::kFHM) },
+ { FT::kHWOptional , "armv8_2_fhm" , uint8_t(Ext::kFHM) },
+ { FT::kHWOptionalArmFEAT, "FP16" , uint8_t(Ext::kFP16) },
+ { FT::kHWOptional , "neon_fp16" , uint8_t(Ext::kFP16) },
+ { FT::kHWOptionalArmFEAT, "FRINTTS" , uint8_t(Ext::kFRINTTS) },
+ { FT::kHWOptionalArmFEAT, "I8MM" , uint8_t(Ext::kI8MM) },
+ { FT::kHWOptionalArmFEAT, "JSCVT" , uint8_t(Ext::kJSCVT) },
+ { FT::kHWOptionalArmFEAT, "RDM" , uint8_t(Ext::kRDM) },
+
+ // Determine Integer Capabilities:
+ { FT::kHWOptional , "armv8_crc32" , uint8_t(Ext::kCRC32) },
+ { FT::kHWOptionalArmFEAT, "FlagM" , uint8_t(Ext::kFLAGM) },
+ { FT::kHWOptionalArmFEAT, "FlagM2" , uint8_t(Ext::kFLAGM2) },
+
+ // Determine Atomic and Memory Ordering Instruction Capabilities:
+ { FT::kHWOptionalArmFEAT, "LRCPC" , uint8_t(Ext::kLRCPC) },
+ { FT::kHWOptionalArmFEAT, "LRCPC2" , uint8_t(Ext::kLRCPC2) },
+ { FT::kHWOptional , "armv8_1_atomics", uint8_t(Ext::kLSE) },
+ { FT::kHWOptionalArmFEAT, "LSE" , uint8_t(Ext::kLSE) },
+ { FT::kHWOptionalArmFEAT, "LSE2" , uint8_t(Ext::kLSE2) },
+
+ // Determine Encryption Capabilities:
+ { FT::kHWOptionalArmFEAT, "AES" , uint8_t(Ext::kAES) },
+ { FT::kHWOptionalArmFEAT, "PMULL" , uint8_t(Ext::kPMULL) },
+ { FT::kHWOptionalArmFEAT, "SHA1" , uint8_t(Ext::kSHA1) },
+ { FT::kHWOptionalArmFEAT, "SHA256" , uint8_t(Ext::kSHA256) },
+ { FT::kHWOptionalArmFEAT, "SHA512" , uint8_t(Ext::kSHA512) },
+ { FT::kHWOptional , "armv8_2_sha512" , uint8_t(Ext::kSHA512) },
+ { FT::kHWOptionalArmFEAT, "SHA3" , uint8_t(Ext::kSHA3) },
+ { FT::kHWOptional , "armv8_2_sha3" , uint8_t(Ext::kSHA3) },
+
+ // Determine General Capabilities:
+ { FT::kHWOptionalArmFEAT, "BTI" , uint8_t(Ext::kBTI) },
+ { FT::kHWOptionalArmFEAT, "DPB" , uint8_t(Ext::kDPB) },
+ { FT::kHWOptionalArmFEAT, "DPB2" , uint8_t(Ext::kDPB2) },
+ { FT::kHWOptionalArmFEAT, "ECV" , uint8_t(Ext::kECV) },
+ { FT::kHWOptionalArmFEAT, "SB" , uint8_t(Ext::kSB) },
+ { FT::kHWOptionalArmFEAT, "SSBS" , uint8_t(Ext::kSSBS) }
+ };
+
+ for (size_t i = 0; i < ASMJIT_ARRAY_SIZE(mappings); i++) {
+ const AppleFeatureMapping& mapping = mappings[i];
+ if (!cpu.features().arm().has(mapping.featureId) && appleDetectARMFeatureViaSysctl(mapping.type, mapping.name)) {
+ cpu.features().arm().add(mapping.featureId);
+ }
+ }
+}
+
+static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept {
+ cpu._wasDetected = true;
+ populateBaseARMFeatures(cpu);
+
+ appleSysctlByName<uint32_t>("hw.cpufamily", &cpu._familyId);
+ appleSysctlByName<uint32_t>("hw.cachelinesize", &cpu._cacheLineSize);
+ appleSysctlByName<uint32_t>("machdep.cpu.logical_per_package", &cpu._maxLogicalProcessors);
+ appleSysctlByName<char>("machdep.cpu.brand_string", cpu._brand.str, sizeof(cpu._brand.str));
+
+ memcpy(cpu._vendor.str, "APPLE", 6);
+
+ bool cpuFeaturesPopulated = detectARMFeaturesViaAppleFamilyId(cpu);
+ if (!cpuFeaturesPopulated)
+ appleDetectARMFeaturesViaSysctl(cpu);
+ postProcessARMCpuInfo(cpu);
+}
+
+// CpuInfo - Detect - ARM - Detect by Fallback (Using Compiler Flags)
+// ==================================================================
+
+#else
+
+#if ASMJIT_ARCH_ARM == 32
+ #pragma message("[asmjit] Disabling runtime CPU detection - unsupported OS/CPU combination (Unknown OS with AArch32 CPU)")
+#else
+ #pragma message("[asmjit] Disabling runtime CPU detection - unsupported OS/CPU combination (Unknown OS with AArch64 CPU)")
#endif
-#if defined(ASMJIT_BUILD_ARM) && ASMJIT_ARCH_ARM
-namespace arm { void detectCpu(CpuInfo& cpu) noexcept; }
+static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept {
+ populateBaseARMFeatures(cpu);
+ detectARMFeaturesViaCompilerFlags(cpu);
+ postProcessARMCpuInfo(cpu);
+}
#endif
-// ============================================================================
-// [asmjit::CpuInfo - Detect - Static Initializer]
-// ============================================================================
+} // {arm}
-static uint32_t cpuInfoInitialized;
-static CpuInfo cpuInfoGlobal(Globals::NoInit);
+#endif
+
+// CpuInfo - Detect - Host
+// =======================
const CpuInfo& CpuInfo::host() noexcept {
- // This should never cause a problem as the resulting information should
- // always be the same.
- if (!cpuInfoInitialized) {
+ static std::atomic<uint32_t> cpuInfoInitialized;
+ static CpuInfo cpuInfoGlobal(Globals::NoInit);
+
+ // This should never cause a problem as the resulting information should always
+ // be the same. In the worst case it would just be overwritten non-atomically.
+ if (!cpuInfoInitialized.load(std::memory_order_relaxed)) {
CpuInfo cpuInfoLocal;
-#if defined(ASMJIT_BUILD_X86) && ASMJIT_ARCH_X86
- x86::detectCpu(cpuInfoLocal);
-#endif
+ cpuInfoLocal._arch = Arch::kHost;
+ cpuInfoLocal._subArch = SubArch::kHost;
-#if defined(ASMJIT_BUILD_ARM) && ASMJIT_ARCH_ARM
- arm::detectCpu(cpuInfoLocal);
+#if ASMJIT_ARCH_X86
+ x86::detectX86Cpu(cpuInfoLocal);
+#elif ASMJIT_ARCH_ARM
+ arm::detectARMCpu(cpuInfoLocal);
#endif
cpuInfoLocal._hwThreadCount = detectHWThreadCount();
cpuInfoGlobal = cpuInfoLocal;
- cpuInfoInitialized = 1;
+ cpuInfoInitialized.store(1, std::memory_order_seq_cst);
}
return cpuInfoGlobal;