diff options
| author | 2026-04-17 15:21:32 +0200 | |
|---|---|---|
| committer | 2026-04-17 15:21:32 +0200 | |
| commit | d3d82c776740320dc29eca249f8cd5b1ae4d1b40 (patch) | |
| tree | 96d2ab1e717f8fa20f73713a95c3f4ce3fecb4e8 /3rdparty/asmjit/asmjit-testing/tests/asmjit_test_unicompiler.cpp | |
| parent | 0b24b4c2f6052cea7d612408f4baa58ed63ee290 (diff) | |
| parent | 0034dde186fa6f594125ac3dc2b5977f18e7a062 (diff) | |
Merge branch 'master' into floppysound
Diffstat (limited to '3rdparty/asmjit/asmjit-testing/tests/asmjit_test_unicompiler.cpp')
| -rw-r--r-- | 3rdparty/asmjit/asmjit-testing/tests/asmjit_test_unicompiler.cpp | 5674 |
1 files changed, 5674 insertions, 0 deletions
diff --git a/3rdparty/asmjit/asmjit-testing/tests/asmjit_test_unicompiler.cpp b/3rdparty/asmjit/asmjit-testing/tests/asmjit_test_unicompiler.cpp new file mode 100644 index 00000000000..3ceba1af5e4 --- /dev/null +++ b/3rdparty/asmjit/asmjit-testing/tests/asmjit_test_unicompiler.cpp @@ -0,0 +1,5674 @@ +// This file is part of AsmJit project <https://asmjit.com> +// +// See <asmjit/core.h> or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include <asmjit/ujit.h> + +#include <asmjit-testing/commons/asmjitutils.h> +#include <asmjit-testing/commons/random.h> + +#include <algorithm> +#include <cmath> +#include <limits> + +static void print_app_info() noexcept { + printf("AsmJit UniCompiler Test Suite v%u.%u.%u [Arch=%s] [Mode=%s]\n\n", + unsigned((ASMJIT_LIBRARY_VERSION >> 16) ), + unsigned((ASMJIT_LIBRARY_VERSION >> 8) & 0xFF), + unsigned((ASMJIT_LIBRARY_VERSION ) & 0xFF), + asmjit_arch_as_string(asmjit::Arch::kHost), + asmjit_build_type() + ); +} + +#if !defined(ASMJIT_NO_UJIT) && !defined(ASMJIT_NO_JIT) + +#include <asmjit-testing/tests/broken.h> + +namespace UniCompilerTests { + +using namespace asmjit; +using namespace asmjit::ujit; + +// ujit::UniCompiler - Tests - Constants +// ===================================== + +static constexpr uint64_t kRandomSeed = 0x1234u; +static constexpr uint32_t kTestIterCount = 1000u; + +static ASMJIT_INLINE_CONSTEXPR uint32_t byte_width_from_vec_width(VecWidth vw) noexcept { + return 16u << uint32_t(vw); +} + +// ujit::UniCompiler - Tests - MulAdd +// ================================== + +#if defined(ASMJIT_UJIT_X86) + +float fadd(float a, float b) noexcept; +float fsub(float a, float b) noexcept; +float fmul(float a, float b) noexcept; +float fdiv(float a, float b) noexcept; +float fsqrt(float a) noexcept; +float fmadd_nofma_ref(float a, float b, float c) noexcept; +float fmadd_fma_ref(float a, float b, float c) noexcept; + +double fadd(double a, double b) noexcept; +double fsub(double a, double b) noexcept; +double fmul(double a, double b) noexcept; +double fdiv(double a, double b) noexcept; +double fsqrt(double a) noexcept; +double fmadd_nofma_ref(double a, double b, double c) noexcept; +double fmadd_fma_ref(double a, double b, double c) noexcept; + +void madd_fma_check_valgrind_bug(const float a[4], const float b[4], const float c[4], float dst[4]) noexcept; + +#else + +static ASMJIT_NOINLINE float fadd(float a, float b) noexcept { return a + b; } +static ASMJIT_NOINLINE float fsub(float a, float b) noexcept { return a - b; } +static ASMJIT_NOINLINE float fmul(float a, float b) noexcept { return a * b; } +static ASMJIT_NOINLINE float fdiv(float a, float b) noexcept { return a / b; } +static ASMJIT_NOINLINE float fsqrt(float a) noexcept { return std::sqrt(a); } +static ASMJIT_NOINLINE float fmadd_nofma_ref(float a, float b, float c) noexcept { return a * b + c; } +static ASMJIT_NOINLINE float fmadd_fma_ref(float a, float b, float c) noexcept { return std::fma(a, b, c); } + +static ASMJIT_NOINLINE double fadd(double a, double b) noexcept { return a + b; } +static ASMJIT_NOINLINE double fsub(double a, double b) noexcept { return a - b; } +static ASMJIT_NOINLINE double fmul(double a, double b) noexcept { return a * b; } +static ASMJIT_NOINLINE double fdiv(double a, double b) noexcept { return a / b; } +static ASMJIT_NOINLINE double fsqrt(double a) noexcept { return std::sqrt(a); } +static ASMJIT_NOINLINE double fmadd_nofma_ref(double a, double b, double c) noexcept { return fadd(fmul(a, b), c); } +static ASMJIT_NOINLINE double fmadd_fma_ref(double a, double b, double c) noexcept { return std::fma(a, b, c); } + +#endif + +static ASMJIT_INLINE float fsign(float a) noexcept { return Support::bit_cast<float>(Support::bit_cast<uint32_t>(a) & (uint32_t(1) << 31)); } +static ASMJIT_INLINE double fsign(double a) noexcept { return Support::bit_cast<double>(Support::bit_cast<uint64_t>(a) & (uint64_t(1) << 63)); } + +static ASMJIT_INLINE float fxor(float a, float b) noexcept { return Support::bit_cast<float>(Support::bit_cast<uint32_t>(a) ^ Support::bit_cast<uint32_t>(b)); } +static ASMJIT_INLINE double fxor(double a, double b) noexcept { return Support::bit_cast<double>(Support::bit_cast<uint64_t>(a) ^ Support::bit_cast<uint64_t>(b)); } + +// ujit::UniCompiler - Tests - Types +// ================================= + +struct Variation { + uint32_t value; + + [[nodiscard]] ASMJIT_INLINE_CONSTEXPR bool operator==(uint32_t v) const noexcept { return value == v; } + [[nodiscard]] ASMJIT_INLINE_CONSTEXPR bool operator!=(uint32_t v) const noexcept { return value != v; } + [[nodiscard]] ASMJIT_INLINE_CONSTEXPR bool operator<=(uint32_t v) const noexcept { return value <= v; } + [[nodiscard]] ASMJIT_INLINE_CONSTEXPR bool operator< (uint32_t v) const noexcept { return value < v; } + [[nodiscard]] ASMJIT_INLINE_CONSTEXPR bool operator>=(uint32_t v) const noexcept { return value >= v; } + [[nodiscard]] ASMJIT_INLINE_CONSTEXPR bool operator> (uint32_t v) const noexcept { return value > v; } +}; + +// ujit::UniCompiler - Tests - JIT Function Prototypes +// =================================================== + +typedef uint32_t (*TestCondRRFunc)(int32_t a, int32_t b); +typedef uint32_t (*TestCondRIFunc)(int32_t a); + +typedef void (*TestMFunc)(void* ptr); +typedef uintptr_t (*TestRMFunc)(uintptr_t reg, void* ptr); +typedef void (*TestMRFunc)(void* ptr, uintptr_t reg); + +typedef uint32_t (*TestRRFunc)(uint32_t a); +typedef uint32_t (*TestRRRFunc)(uint32_t a, uint32_t b); +typedef uint32_t (*TestRRIFunc)(uint32_t a); + +typedef void (*TestVVFunc)(void* dst, const void* src); +typedef void (*TestVVVFunc)(void* dst, const void* src1, const void* src2); +typedef void (*TestVVVVFunc)(void* dst, const void* src1, const void* src2, const void* src3); + +// ujit::UniCompiler - Tests - JIT Context Error Handler +// ===================================================== + +class TestErrorHandler : public ErrorHandler { +public: + TestErrorHandler() noexcept {} + ~TestErrorHandler() noexcept override {} + + void handle_error(Error err, const char* message, BaseEmitter* origin) override { + Support::maybe_unused(origin); + EXPECT_EQ(err, Error::kOk) + .message("AsmJit Error: %s", message); + } +}; +// ujit::UniCompiler - Tests - JIT Context for Testing +// =================================================== + +class JitContext { +public: + JitRuntime rt; + CpuFeatures features {}; + CpuHints cpu_hints {}; + +#if !defined(ASMJIT_NO_LOGGING) + StringLogger logger; +#endif // !ASMJIT_NO_LOGGING + + TestErrorHandler eh; + CodeHolder code; + BackendCompiler cc; + + void prepare() noexcept { + code.reset(); + code.init(rt.environment()); + code.set_error_handler(&eh); + +#if !defined(ASMJIT_NO_LOGGING) + logger.clear(); + code.set_logger(&logger); +#endif // !ASMJIT_NO_LOGGING + + code.attach(&cc); + cc.add_diagnostic_options(DiagnosticOptions::kRAAnnotate); + cc.add_diagnostic_options(DiagnosticOptions::kValidateAssembler); + cc.add_diagnostic_options(DiagnosticOptions::kValidateIntermediate); + } + + template<typename Fn> + Fn finish() { + Fn fn; + EXPECT_EQ(cc.finalize(), Error::kOk); + EXPECT_EQ(rt.add(&fn, &code), Error::kOk); + code.reset(); + return fn; + } + +#if !defined(ASMJIT_NO_LOGGING) + const char* logger_content() const noexcept { return logger.data(); } +#else + const char* logger_content() const noexcept { return "<ASMJIT_NO_LOGGING>"; } +#endif +}; + +// ujit::UniCompiler - Tests - Conditional Operations - Functions +// ============================================================== + +static TestCondRRFunc create_func_cond_rr(JitContext& ctx, UniOpCond op, CondCode cond_code, uint32_t variation) { + ctx.prepare(); + + UniCompiler uc(&ctx.cc, ctx.features, ctx.cpu_hints); + uc.init_vec_width(VecWidth::k128); + + FuncNode* node = uc.add_func(FuncSignature::build<uint32_t, int32_t, int32_t>()); + EXPECT_NOT_NULL(node); + + Gp a = uc.new_gp32("a"); + Gp b = uc.new_gp32("b"); + Gp result = uc.new_gp32("result"); + + node->set_arg(0, a); + node->set_arg(1, b); + + switch (variation) { + case 0: { + // Test a conditional branch based on the given condition. + Label done = uc.new_label(); + uc.mov(result, 1); + uc.j(done, UniCondition(op, cond_code, a, b)); + uc.mov(result, 0); + uc.bind(done); + break; + } + + case 1: { + // Test a cmov functionality. + Gp true_value = uc.new_gp32("true_value"); + uc.mov(result, 0); + uc.mov(true_value, 1); + uc.cmov(result, true_value, UniCondition(op, cond_code, a, b)); + break; + } + + case 2: { + // Test a select functionality. + Gp false_value = uc.new_gp32("false_value"); + Gp true_value = uc.new_gp32("true_value"); + uc.mov(false_value, 0); + uc.mov(true_value, 1); + uc.select(result, true_value, false_value, UniCondition(op, cond_code, a, b)); + break; + } + } + + uc.ret(result); + uc.end_func(); + + return ctx.finish<TestCondRRFunc>(); +} + +static TestCondRIFunc create_func_cond_ri(JitContext& ctx, UniOpCond op, CondCode cond_code, Imm b_imm) { + ctx.prepare(); + + UniCompiler uc(&ctx.cc, ctx.features, ctx.cpu_hints); + uc.init_vec_width(VecWidth::k128); + + FuncNode* node = uc.add_func(FuncSignature::build<uint32_t, int32_t>()); + EXPECT_NOT_NULL(node); + + Gp a = uc.new_gp32("a"); + Gp result = uc.new_gp32("result"); + Label done = uc.new_label(); + + node->set_arg(0, a); + uc.mov(result, 1); + uc.j(done, UniCondition(op, cond_code, a, b_imm)); + uc.mov(result, 0); + uc.bind(done); + uc.ret(result); + + uc.end_func(); + return ctx.finish<TestCondRIFunc>(); +} + +// ujit::UniCompiler - Tests - Conditional Operations - Runner +// =========================================================== + +static ASMJIT_NOINLINE void test_conditional_op(JitContext& ctx, UniOpCond op, CondCode cond_code, int32_t a, int32_t b, bool expected) { + for (uint32_t variation = 0; variation < 3; variation++) { + TestCondRRFunc fn_rr = create_func_cond_rr(ctx, op, cond_code, variation); + TestCondRIFunc fn_ri = create_func_cond_ri(ctx, op, cond_code, b); + + uint32_t observed_rr = fn_rr(a, b); + EXPECT_EQ(observed_rr, uint32_t(expected)) + .message("Operation failed (RR):\n" + " Input #1: %d\n" + " Input #2: %d\n" + " Expected: %d\n" + " Observed: %d\n" + "Assembly:\n%s", + a, + b, + uint32_t(expected), + observed_rr, + ctx.logger_content()); + + uint32_t observed_ri = fn_ri(a); + EXPECT_EQ(observed_ri, uint32_t(expected)) + .message("Operation failed (RI):\n" + " Input #1: %d\n" + " Input #2: %d\n" + " Expected: %d\n" + " Observed: %d\n" + "Assembly:\n%s", + a, + b, + uint32_t(expected), + observed_ri, + ctx.logger_content()); + + ctx.rt.reset(); + } +} + +static ASMJIT_NOINLINE void test_cond_ops(JitContext& ctx) { + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kEqual, 0, 0, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kEqual, 1, 1, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kEqual, 1, 2, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kEqual, 100, 31, false); + + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kNotEqual, 0, 0, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kNotEqual, 1, 1, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kNotEqual, 1, 2, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kNotEqual, 100, 31, true); + + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedGT, 0, 0, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedGT, 1, 0, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedGT, 111111, 0, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedGT, 111111, 222, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedGT, 222, 111111, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedGT, 222, 111, true); + + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedGE, 0, 0, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedGE, 1, 0, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedGE, 111111, 0, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedGE, 111111, 111111, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedGE, 111111, 222, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedGE, 222, 111111, false); + + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedLT, 0, 0, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedLT, 1, 0, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedLT, 0, 1, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedLT, 111111, 0, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedLT, 111111, 222, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedLT, 222, 111111, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedLT, 222, 111, false); + + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedLE, 0, 0, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedLE, 1, 0, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedLE, 0, 1, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedLE, 111111, 0, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedLE, 111111, 222, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedLE, 222, 111111, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kUnsignedLE, 22222, 22222, true); + + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedGT, 0, 0, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedGT, 1, 0, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedGT, 111111, 0, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedGT, 111111, -222, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedGT, -222, 111111, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedGT, -222, -111, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedGT, -111, -1, false); + + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedGE, 0, 0, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedGE, 1, 0, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedGE, 111111, 0, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedGE, 111111, 111111, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedGE, 111111, -222, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedGE, -222, 111111, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedGE, -111, -1, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedGE, -111, -111, true); + + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedLT, 0, 0, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedLT, 1, 0, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedLT, 111111, 0, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedLT, 111111, -222, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedLT, -222, 111111, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedLT, -222, -111, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedLT, -111, -1, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedLT, -1, -1, false); + + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedLE, 0, 0, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedLE, 1, 0, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedLE, 111111, 0, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedLE, 111111, -222, false); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedLE, -222, 111111, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedLE, -222, -111, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedLE, -111, -1, true); + test_conditional_op(ctx, UniOpCond::kCompare, CondCode::kSignedLE, -1, -1, true); + + test_conditional_op(ctx, UniOpCond::kTest, CondCode::kZero, 0, 0, true); + test_conditional_op(ctx, UniOpCond::kTest, CondCode::kZero, 1, 0, true); + test_conditional_op(ctx, UniOpCond::kTest, CondCode::kZero, 111111, 0, true); + test_conditional_op(ctx, UniOpCond::kTest, CondCode::kZero, 111111, -222, false); + test_conditional_op(ctx, UniOpCond::kTest, CondCode::kZero, -222, 111111, false); + + test_conditional_op(ctx, UniOpCond::kTest, CondCode::kNotZero, 0, 0, false); + test_conditional_op(ctx, UniOpCond::kTest, CondCode::kNotZero, 1, 0, false); + test_conditional_op(ctx, UniOpCond::kTest, CondCode::kNotZero, 111111, 0, false); + test_conditional_op(ctx, UniOpCond::kTest, CondCode::kNotZero, 111111, -222, true); + test_conditional_op(ctx, UniOpCond::kTest, CondCode::kNotZero, -222, 111111, true); + + test_conditional_op(ctx, UniOpCond::kBitTest, CondCode::kBTZero, int32_t(0x0), 0, true); + test_conditional_op(ctx, UniOpCond::kBitTest, CondCode::kBTZero, int32_t(0x1), 0, false); + test_conditional_op(ctx, UniOpCond::kBitTest, CondCode::kBTZero, int32_t(0xFF), 7, false); + test_conditional_op(ctx, UniOpCond::kBitTest, CondCode::kBTZero, int32_t(0xFF), 9, true); + test_conditional_op(ctx, UniOpCond::kBitTest, CondCode::kBTZero, int32_t(0xFFFFFFFF), 31, false); + test_conditional_op(ctx, UniOpCond::kBitTest, CondCode::kBTZero, int32_t(0x7FFFFFFF), 31, true); + + test_conditional_op(ctx, UniOpCond::kBitTest, CondCode::kBTNotZero, int32_t(0x0), 0, false); + test_conditional_op(ctx, UniOpCond::kBitTest, CondCode::kBTNotZero, int32_t(0x1), 0, true); + test_conditional_op(ctx, UniOpCond::kBitTest, CondCode::kBTNotZero, int32_t(0xFF), 7, true); + test_conditional_op(ctx, UniOpCond::kBitTest, CondCode::kBTNotZero, int32_t(0xFF), 9, false); + test_conditional_op(ctx, UniOpCond::kBitTest, CondCode::kBTNotZero, int32_t(0xFFFFFFFF), 31, true); + test_conditional_op(ctx, UniOpCond::kBitTest, CondCode::kBTNotZero, int32_t(0x7FFFFFFF), 31, false); + + test_conditional_op(ctx, UniOpCond::kAssignAnd, CondCode::kZero, int32_t(0x00000000), int32_t(0x00000000), true); + test_conditional_op(ctx, UniOpCond::kAssignAnd, CondCode::kZero, int32_t(0x00000001), int32_t(0x00000000), true); + test_conditional_op(ctx, UniOpCond::kAssignAnd, CondCode::kZero, int32_t(0x000000FF), int32_t(0x00000000), true); + test_conditional_op(ctx, UniOpCond::kAssignAnd, CondCode::kZero, int32_t(0x000000FF), int32_t(0x000000FF), false); + test_conditional_op(ctx, UniOpCond::kAssignAnd, CondCode::kZero, int32_t(0xFFFFFFFF), int32_t(0xFF000000), false); + test_conditional_op(ctx, UniOpCond::kAssignAnd, CondCode::kZero, int32_t(0x7FFFFFFF), int32_t(0x80000000), true); + + test_conditional_op(ctx, UniOpCond::kAssignAnd, CondCode::kNotZero, int32_t(0x00000000), int32_t(0x00000000), false); + test_conditional_op(ctx, UniOpCond::kAssignAnd, CondCode::kNotZero, int32_t(0x00000001), int32_t(0x00000000), false); + test_conditional_op(ctx, UniOpCond::kAssignAnd, CondCode::kNotZero, int32_t(0x000000FF), int32_t(0x00000000), false); + test_conditional_op(ctx, UniOpCond::kAssignAnd, CondCode::kNotZero, int32_t(0x000000FF), int32_t(0x000000FF), true); + test_conditional_op(ctx, UniOpCond::kAssignAnd, CondCode::kNotZero, int32_t(0xFFFFFFFF), int32_t(0xFF000000), true); + test_conditional_op(ctx, UniOpCond::kAssignAnd, CondCode::kNotZero, int32_t(0x7FFFFFFF), int32_t(0x80000000), false); + + test_conditional_op(ctx, UniOpCond::kAssignOr, CondCode::kZero, int32_t(0x00000000), int32_t(0x00000000), true); + test_conditional_op(ctx, UniOpCond::kAssignOr, CondCode::kZero, int32_t(0x00000001), int32_t(0x00000000), false); + test_conditional_op(ctx, UniOpCond::kAssignOr, CondCode::kZero, int32_t(0x000000FF), int32_t(0x00000000), false); + test_conditional_op(ctx, UniOpCond::kAssignOr, CondCode::kZero, int32_t(0x000000FF), int32_t(0x000000FF), false); + test_conditional_op(ctx, UniOpCond::kAssignOr, CondCode::kZero, int32_t(0xFFFFFFFF), int32_t(0xFF000000), false); + test_conditional_op(ctx, UniOpCond::kAssignOr, CondCode::kZero, int32_t(0x7FFFFFFF), int32_t(0x80000000), false); + + test_conditional_op(ctx, UniOpCond::kAssignOr, CondCode::kNotZero, int32_t(0x00000000), int32_t(0x00000000), false); + test_conditional_op(ctx, UniOpCond::kAssignOr, CondCode::kNotZero, int32_t(0x00000001), int32_t(0x00000000), true); + test_conditional_op(ctx, UniOpCond::kAssignOr, CondCode::kNotZero, int32_t(0x000000FF), int32_t(0x00000000), true); + test_conditional_op(ctx, UniOpCond::kAssignOr, CondCode::kNotZero, int32_t(0x000000FF), int32_t(0x000000FF), true); + test_conditional_op(ctx, UniOpCond::kAssignOr, CondCode::kNotZero, int32_t(0xFFFFFFFF), int32_t(0xFF000000), true); + test_conditional_op(ctx, UniOpCond::kAssignOr, CondCode::kNotZero, int32_t(0x7FFFFFFF), int32_t(0x80000000), true); + + test_conditional_op(ctx, UniOpCond::kAssignXor, CondCode::kZero, int32_t(0x00000000), int32_t(0x00000000), true); + test_conditional_op(ctx, UniOpCond::kAssignXor, CondCode::kZero, int32_t(0x00000001), int32_t(0x00000000), false); + test_conditional_op(ctx, UniOpCond::kAssignXor, CondCode::kZero, int32_t(0x000000FF), int32_t(0x00000000), false); + test_conditional_op(ctx, UniOpCond::kAssignXor, CondCode::kZero, int32_t(0x000000FF), int32_t(0x000000FF), true); + test_conditional_op(ctx, UniOpCond::kAssignXor, CondCode::kZero, int32_t(0xFFFFFFFF), int32_t(0xFF000000), false); + test_conditional_op(ctx, UniOpCond::kAssignXor, CondCode::kZero, int32_t(0x7FFFFFFF), int32_t(0x80000000), false); + + test_conditional_op(ctx, UniOpCond::kAssignXor, CondCode::kNotZero, int32_t(0x00000000), int32_t(0x00000000), false); + test_conditional_op(ctx, UniOpCond::kAssignXor, CondCode::kNotZero, int32_t(0x00000001), int32_t(0x00000000), true); + test_conditional_op(ctx, UniOpCond::kAssignXor, CondCode::kNotZero, int32_t(0x000000FF), int32_t(0x00000000), true); + test_conditional_op(ctx, UniOpCond::kAssignXor, CondCode::kNotZero, int32_t(0x000000FF), int32_t(0x000000FF), false); + test_conditional_op(ctx, UniOpCond::kAssignXor, CondCode::kNotZero, int32_t(0xFFFFFFFF), int32_t(0xFF000000), true); + test_conditional_op(ctx, UniOpCond::kAssignXor, CondCode::kNotZero, int32_t(0x7FFFFFFF), int32_t(0x80000000), true); + + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kZero, int32_t(0x00000000), int32_t(0x00000000), true); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kZero, int32_t(0xFF000000), int32_t(0x01000000), true); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kZero, int32_t(0x000000FF), int32_t(0x00000000), false); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kZero, int32_t(0x000000FF), int32_t(0x000000FF), false); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kZero, int32_t(0xFFFFFFFF), int32_t(0xFF000000), false); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kZero, int32_t(0x7FFFFFFF), int32_t(0x80000000), false); + + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kNotZero, int32_t(0x00000000), int32_t(0x00000000), false); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kNotZero, int32_t(0xFF000000), int32_t(0x01000000), false); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kNotZero, int32_t(0x000000FF), int32_t(0x00000000), true); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kNotZero, int32_t(0x000000FF), int32_t(0x000000FF), true); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kNotZero, int32_t(0xFFFFFFFF), int32_t(0xFF000000), true); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kNotZero, int32_t(0x7FFFFFFF), int32_t(0x80000000), true); + + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kCarry, int32_t(0x00000000), int32_t(0x00000000), false); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kCarry, int32_t(0xFF000000), int32_t(0x01000000), true); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kCarry, int32_t(0x000000FF), int32_t(0x00000000), false); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kCarry, int32_t(0x000000FF), int32_t(0x000000FF), false); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kCarry, int32_t(0xFFFFFFFF), int32_t(0xFF000000), true); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kCarry, int32_t(0x7FFFFFFF), int32_t(0x80000000), false); + + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kNotCarry, int32_t(0x00000000), int32_t(0x00000000), true); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kNotCarry, int32_t(0xFF000000), int32_t(0x01000000), false); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kNotCarry, int32_t(0x000000FF), int32_t(0x00000000), true); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kNotCarry, int32_t(0x000000FF), int32_t(0x000000FF), true); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kNotCarry, int32_t(0xFFFFFFFF), int32_t(0xFF000000), false); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kNotCarry, int32_t(0x7FFFFFFF), int32_t(0x80000000), true); + + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kSign, int32_t(0x00000000), int32_t(0x00000000), false); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kSign, int32_t(0xFF000000), int32_t(0x01000000), false); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kSign, int32_t(0x000000FF), int32_t(0x80000000), true); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kSign, int32_t(0x000000FF), int32_t(0x800000FF), true); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kSign, int32_t(0xFFFFFFFF), int32_t(0xFF000000), true); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kSign, int32_t(0x7FFFFFFF), int32_t(0x80000000), true); + + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kNotSign, int32_t(0x00000000), int32_t(0x00000000), true); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kNotSign, int32_t(0xFF000000), int32_t(0x01000000), true); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kNotSign, int32_t(0x000000FF), int32_t(0x80000000), false); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kNotSign, int32_t(0x000000FF), int32_t(0x800000FF), false); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kNotSign, int32_t(0xFFFFFFFF), int32_t(0xFF000000), false); + test_conditional_op(ctx, UniOpCond::kAssignAdd, CondCode::kNotSign, int32_t(0x7FFFFFFF), int32_t(0x80000000), false); + + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kZero, int32_t(0x00000000), int32_t(0x00000000), true); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kZero, int32_t(0xFF000000), int32_t(0x01000000), false); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kZero, int32_t(0x000000FF), int32_t(0x00000000), false); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kZero, int32_t(0x000000FF), int32_t(0x000000FF), true); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kZero, int32_t(0xFFFFFFFF), int32_t(0xFF000000), false); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kZero, int32_t(0x7FFFFFFF), int32_t(0x80000000), false); + + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kNotZero, int32_t(0x00000000), int32_t(0x00000000), false); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kNotZero, int32_t(0xFF000000), int32_t(0x01000000), true); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kNotZero, int32_t(0x000000FF), int32_t(0x00000000), true); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kNotZero, int32_t(0x000000FF), int32_t(0x000000FF), false); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kNotZero, int32_t(0xFFFFFFFF), int32_t(0xFF000000), true); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kNotZero, int32_t(0x7FFFFFFF), int32_t(0x80000000), true); + + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kUnsignedLT, int32_t(0x00000000), int32_t(0x00000000), false); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kUnsignedLT, int32_t(0xFF000000), int32_t(0x01000000), false); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kUnsignedLT, int32_t(0x000000FF), int32_t(0x00000000), false); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kUnsignedLT, int32_t(0x000000FF), int32_t(0x000000FF), false); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kUnsignedLT, int32_t(0xFFFFFFFF), int32_t(0xFF000000), false); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kUnsignedLT, int32_t(0x7FFFFFFF), int32_t(0x80000000), true); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kUnsignedLT, int32_t(0x00000111), int32_t(0x0000F0FF), true); + + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kUnsignedGE, int32_t(0x00000000), int32_t(0x00000000), true); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kUnsignedGE, int32_t(0xFF000000), int32_t(0x01000000), true); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kUnsignedGE, int32_t(0x000000FF), int32_t(0x00000000), true); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kUnsignedGE, int32_t(0x000000FF), int32_t(0x000000FF), true); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kUnsignedGE, int32_t(0xFFFFFFFF), int32_t(0xFF000000), true); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kUnsignedGE, int32_t(0x7FFFFFFF), int32_t(0x80000000), false); + + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kSign, int32_t(0x00000000), int32_t(0x00000000), false); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kSign, int32_t(0x00000000), int32_t(0xFFFFFFFF), false); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kSign, int32_t(0x00000000), int32_t(0x00000001), true); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kSign, int32_t(0x00000001), int32_t(0x00000010), true); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kSign, int32_t(0xFFFFFFFF), int32_t(0xFF000000), false); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kSign, int32_t(0x7FFFFFFF), int32_t(0x80000000), true); + + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kNotSign, int32_t(0x00000000), int32_t(0x00000000), true); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kNotSign, int32_t(0x00000000), int32_t(0xFFFFFFFF), true); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kNotSign, int32_t(0x00000000), int32_t(0x00000001), false); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kNotSign, int32_t(0x00000001), int32_t(0x00000010), false); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kNotSign, int32_t(0xFFFFFFFF), int32_t(0xFF000000), true); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kNotSign, int32_t(0x7FFFFFFF), int32_t(0x80000000), false); + + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kUnsignedGT, int32_t(0x00000000), int32_t(0x00000000), false); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kUnsignedGT, int32_t(0xFF000000), int32_t(0x01000000), true); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kUnsignedGT, int32_t(0x000000FF), int32_t(0x00000000), true); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kUnsignedGT, int32_t(0x000000FF), int32_t(0x000000FF), false); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kUnsignedGT, int32_t(0xFFFFFFFF), int32_t(0xFF000000), true); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kUnsignedGT, int32_t(0x7FFFFFFF), int32_t(0x80000000), false); + test_conditional_op(ctx, UniOpCond::kAssignSub, CondCode::kUnsignedGT, int32_t(0x00000111), int32_t(0x0000F0FF), false); + + test_conditional_op(ctx, UniOpCond::kAssignShr, CondCode::kZero, int32_t(0x00000000), 1, true); + test_conditional_op(ctx, UniOpCond::kAssignShr, CondCode::kZero, int32_t(0x000000FF), 8, true); + test_conditional_op(ctx, UniOpCond::kAssignShr, CondCode::kZero, int32_t(0x000000FF), 7, false); + test_conditional_op(ctx, UniOpCond::kAssignShr, CondCode::kZero, int32_t(0xFFFFFFFF), 31, false); + test_conditional_op(ctx, UniOpCond::kAssignShr, CondCode::kZero, int32_t(0x7FFFFFFF), 31, true); + + test_conditional_op(ctx, UniOpCond::kAssignShr, CondCode::kNotZero, int32_t(0x00000000), 1, false); + test_conditional_op(ctx, UniOpCond::kAssignShr, CondCode::kNotZero, int32_t(0x000000FF), 8, false); + test_conditional_op(ctx, UniOpCond::kAssignShr, CondCode::kNotZero, int32_t(0x000000FF), 7, true); + test_conditional_op(ctx, UniOpCond::kAssignShr, CondCode::kNotZero, int32_t(0xFFFFFFFF), 31, true); + test_conditional_op(ctx, UniOpCond::kAssignShr, CondCode::kNotZero, int32_t(0x7FFFFFFF), 31, false); +} + +// ujit::UniCompiler - Tests - M Operations - Functions +// ==================================================== + +static TestMFunc create_func_m(JitContext& ctx, UniOpM op) { + ctx.prepare(); + + UniCompiler uc(&ctx.cc, ctx.features, ctx.cpu_hints); + uc.init_vec_width(VecWidth::k128); + + FuncNode* node = uc.add_func(FuncSignature::build<void, void*>()); + EXPECT_NOT_NULL(node); + + Gp ptr = uc.new_gpz("ptr"); + node->set_arg(0, ptr); + uc.emit_m(op, mem_ptr(ptr)); + + uc.end_func(); + return ctx.finish<TestMFunc>(); +} + +// ujit::UniCompiler - Tests - M Operations - Runner +// ================================================= + +static ASMJIT_NOINLINE void test_m_ops(JitContext& ctx) { + uint8_t buffer[8]; + + TestMFunc fn_zero_u8 = create_func_m(ctx, UniOpM::kStoreZeroU8); + memcpy(buffer, "ABCDEFGH", 8); + fn_zero_u8(buffer + 0); + EXPECT_EQ(memcmp(buffer, "\0BCDEFGH", 8), 0); + fn_zero_u8(buffer + 5); + EXPECT_EQ(memcmp(buffer, "\0BCDE\0GH", 8), 0); + + TestMFunc fn_zero_u16 = create_func_m(ctx, UniOpM::kStoreZeroU16); + memcpy(buffer, "ABCDEFGH", 8); + fn_zero_u16(buffer + 0); + EXPECT_EQ(memcmp(buffer, "\0\0CDEFGH", 8), 0); + fn_zero_u16(buffer + 4); + EXPECT_EQ(memcmp(buffer, "\0\0CD\0\0GH", 8), 0); + + TestMFunc fn_zero_u32 = create_func_m(ctx, UniOpM::kStoreZeroU32); + memcpy(buffer, "ABCDEFGH", 8); + fn_zero_u32(buffer + 0); + EXPECT_EQ(memcmp(buffer, "\0\0\0\0EFGH", 8), 0); + fn_zero_u32(buffer + 4); + EXPECT_EQ(memcmp(buffer, "\0\0\0\0\0\0\0\0", 8), 0); + +#if ASMJIT_ARCH_BITS >= 64 + TestMFunc fn_zero_u64 = create_func_m(ctx, UniOpM::kStoreZeroU64); + memcpy(buffer, "ABCDEFGH", 8); + fn_zero_u64(buffer + 0); + EXPECT_EQ(memcmp(buffer, "\0\0\0\0\0\0\0\0", 8), 0); +#endif + + TestMFunc fn_zero_reg = create_func_m(ctx, UniOpM::kStoreZeroReg); + memcpy(buffer, "ABCDEFGH", 8); + fn_zero_reg(buffer + 0); +#if ASMJIT_ARCH_BITS >= 64 + EXPECT_EQ(memcmp(buffer, "\0\0\0\0\0\0\0\0", 8), 0); +#else + EXPECT_EQ(memcmp(buffer, "\0\0\0\0EFGH", 8), 0); +#endif + + ctx.rt.reset(); +} + +// ujit::UniCompiler - Tests - RM Operations - Functions +// ===================================================== + +static TestRMFunc create_func_rm(JitContext& ctx, UniOpRM op) { + ctx.prepare(); + + UniCompiler uc(&ctx.cc, ctx.features, ctx.cpu_hints); + uc.init_vec_width(VecWidth::k128); + + FuncNode* node = uc.add_func(FuncSignature::build<uintptr_t, uintptr_t, void*>()); + EXPECT_NOT_NULL(node); + + Gp reg = uc.new_gpz("reg"); + Gp ptr = uc.new_gpz("ptr"); + + node->set_arg(0, reg); + node->set_arg(1, ptr); + + uc.emit_rm(op, reg, mem_ptr(ptr)); + uc.ret(reg); + + uc.end_func(); + return ctx.finish<TestRMFunc>(); +} + +// ujit::UniCompiler - Tests - RM Operations - Runner +// ================================================== + +static ASMJIT_NOINLINE void test_rm_ops(JitContext& ctx) { + union Mem { + uint8_t buffer[8]; + uint16_t u8; + uint16_t u16; + uint32_t u32; + uint64_t u64; + }; + + Mem mem{}; + + TestRMFunc fn_load_i8 = create_func_rm(ctx, UniOpRM::kLoadI8); + mem.u8 = uint8_t(int8_t(6)); + EXPECT_EQ(fn_load_i8(0, mem.buffer), uintptr_t(intptr_t(6))); + + mem.u8 = uint8_t(int8_t(-6)); + EXPECT_EQ(fn_load_i8(0, mem.buffer), uintptr_t(intptr_t(-6))); + + TestRMFunc fn_load_u8 = create_func_rm(ctx, UniOpRM::kLoadU8); + mem.u8 = uint8_t(0x80); + EXPECT_EQ(fn_load_u8(0, mem.buffer), 0x80u); + + mem.u8 = uint8_t(0xFF); + EXPECT_EQ(fn_load_u8(0, mem.buffer), 0xFFu); + + TestRMFunc fn_load_i16 = create_func_rm(ctx, UniOpRM::kLoadI16); + mem.u16 = uint16_t(int16_t(666)); + EXPECT_EQ(fn_load_i16(0, mem.buffer), uintptr_t(intptr_t(666))); + + mem.u16 = uint16_t(int16_t(-666)); + EXPECT_EQ(fn_load_i16(0, mem.buffer), uintptr_t(intptr_t(-666))); + + TestRMFunc fn_load_u16 = create_func_rm(ctx, UniOpRM::kLoadU16); + mem.u16 = uint16_t(0x8000); + EXPECT_EQ(fn_load_u16(0, mem.buffer), 0x8000u); + + mem.u16 = uint16_t(0xFEED); + EXPECT_EQ(fn_load_u16(0, mem.buffer), 0xFEEDu); + + TestRMFunc fn_load_i32 = create_func_rm(ctx, UniOpRM::kLoadI32); + mem.u32 = uint32_t(int32_t(666666)); + EXPECT_EQ(fn_load_i32(0, mem.buffer), uintptr_t(intptr_t(666666))); + + mem.u32 = uint32_t(int32_t(-666666)); + EXPECT_EQ(fn_load_i32(0, mem.buffer), uintptr_t(intptr_t(-666666))); + + TestRMFunc fn_load_u32 = create_func_rm(ctx, UniOpRM::kLoadU32); + mem.u32 = 0x12345678; + EXPECT_EQ(fn_load_u32(0, mem.buffer), uint32_t(0x12345678)); + +#if ASMJIT_ARCH_BITS >= 64 + TestRMFunc fn_load_i64 = create_func_rm(ctx, UniOpRM::kLoadI64); + mem.u64 = 0xF123456789ABCDEFu; + EXPECT_EQ(fn_load_i64(0, mem.buffer), 0xF123456789ABCDEFu); + + TestRMFunc fn_load_u64 = create_func_rm(ctx, UniOpRM::kLoadU64); + mem.u64 = 0xF123456789ABCDEFu; + EXPECT_EQ(fn_load_u64(0, mem.buffer), 0xF123456789ABCDEFu); +#endif + + TestRMFunc fn_load_reg = create_func_rm(ctx, UniOpRM::kLoadReg); + mem.u64 = 0xF123456789ABCDEFu; +#if ASMJIT_ARCH_BITS >= 64 + EXPECT_EQ(fn_load_reg(0, mem.buffer), 0xF123456789ABCDEFu); +#else + EXPECT_EQ(fn_load_reg(0, mem.buffer), 0x89ABCDEFu); +#endif + + TestRMFunc fn_load_merge_u8 = create_func_rm(ctx, UniOpRM::kLoadMergeU8); + mem.u8 = uint8_t(0xAA); + EXPECT_EQ(fn_load_merge_u8(0x1F2FFF00, mem.buffer), 0x1F2FFFAAu); + + TestRMFunc fn_load_shift_u8 = create_func_rm(ctx, UniOpRM::kLoadShiftU8); + mem.u8 = uint8_t(0xAA); + EXPECT_EQ(fn_load_shift_u8(0x002FFF00, mem.buffer), 0x2FFF00AAu); + + TestRMFunc fn_load_merge_u16 = create_func_rm(ctx, UniOpRM::kLoadMergeU16); + mem.u16 = uint16_t(0xAABB); + EXPECT_EQ(fn_load_merge_u16(0x1F2F0000, mem.buffer), 0x1F2FAABBu); + + TestRMFunc fn_load_shift_u16 = create_func_rm(ctx, UniOpRM::kLoadShiftU16); + mem.u16 = uint16_t(0xAABB); + EXPECT_EQ(fn_load_shift_u16(0x00001F2F, mem.buffer), 0x1F2FAABBu); + + ctx.rt.reset(); +} + +// ujit::UniCompiler - Tests - MR Operations - Functions +// ===================================================== + +static TestMRFunc create_func_mr(JitContext& ctx, UniOpMR op) { + ctx.prepare(); + + UniCompiler uc(&ctx.cc, ctx.features, ctx.cpu_hints); + uc.init_vec_width(VecWidth::k128); + + FuncNode* node = uc.add_func(FuncSignature::build<void, void*, uintptr_t>()); + EXPECT_NOT_NULL(node); + + Gp ptr = uc.new_gpz("ptr"); + Gp reg = uc.new_gpz("reg"); + + node->set_arg(0, ptr); + node->set_arg(1, reg); + + uc.emit_mr(op, mem_ptr(ptr), reg); + + uc.end_func(); + return ctx.finish<TestMRFunc>(); +} + +// ujit::UniCompiler - Tests - MR Operations - Runner +// ================================================== + +static ASMJIT_NOINLINE void test_mr_ops(JitContext& ctx) { + union Mem { + uint8_t buffer[8]; + uint16_t u8; + uint16_t u16; + uint32_t u32; + uint64_t u64; + }; + + Mem mem{}; + + TestMRFunc fn_store_u8 = create_func_mr(ctx, UniOpMR::kStoreU8); + memcpy(mem.buffer, "ABCDEFGH", 8); + fn_store_u8(mem.buffer, 0x7A); + EXPECT_EQ(memcmp(mem.buffer, "zBCDEFGH", 8), 0); + + TestMRFunc fn_store_u16 = create_func_mr(ctx, UniOpMR::kStoreU16); + memcpy(mem.buffer, "ABCDEFGH", 8); + fn_store_u16(mem.buffer, 0x7A7A); + EXPECT_EQ(memcmp(mem.buffer, "zzCDEFGH", 8), 0); + + TestMRFunc fn_store_u32 = create_func_mr(ctx, UniOpMR::kStoreU32); + memcpy(mem.buffer, "ABCDEFGH", 8); + fn_store_u32(mem.buffer, 0x7A7A7A7A); + EXPECT_EQ(memcmp(mem.buffer, "zzzzEFGH", 8), 0); + +#if ASMJIT_ARCH_BITS >= 64 + TestMRFunc fn_store_u64 = create_func_mr(ctx, UniOpMR::kStoreU64); + memcpy(mem.buffer, "ABCDEFGH", 8); + fn_store_u64(mem.buffer, 0x7A7A7A7A7A7A7A7A); + EXPECT_EQ(memcmp(mem.buffer, "zzzzzzzz", 8), 0); +#endif + + TestMRFunc fn_store_reg = create_func_mr(ctx, UniOpMR::kStoreReg); + memcpy(mem.buffer, "ABCDEFGH", 8); +#if ASMJIT_ARCH_BITS >= 64 + fn_store_reg(mem.buffer, 0x7A7A7A7A7A7A7A7A); + EXPECT_EQ(memcmp(mem.buffer, "zzzzzzzz", 8), 0); +#else + fn_store_reg(mem.buffer, 0x7A7A7A7A); + EXPECT_EQ(memcmp(mem.buffer, "zzzzEFGH", 8), 0); +#endif + + TestMRFunc fn_add_u8 = create_func_mr(ctx, UniOpMR::kAddU8); + mem.u64 = 0; + mem.u8 = 42; + fn_add_u8(mem.buffer, 13); + EXPECT_EQ(mem.u8, 55u); + EXPECT_EQ(memcmp(mem.buffer + 1, "\0\0\0\0\0\0\0", 7), 0); + + TestMRFunc fn_add_u16 = create_func_mr(ctx, UniOpMR::kAddU16); + mem.u64 = 0; + mem.u16 = 442; + fn_add_u16(mem.buffer, 335); + EXPECT_EQ(mem.u16, 777u); + EXPECT_EQ(memcmp(mem.buffer + 2, "\0\0\0\0\0\0", 6), 0); + + TestMRFunc fn_add_u32 = create_func_mr(ctx, UniOpMR::kAddU32); + mem.u64 = 0; + mem.u32 = 442332; + fn_add_u32(mem.buffer, 335223); + EXPECT_EQ(mem.u32, 777555u); + EXPECT_EQ(memcmp(mem.buffer + 4, "\0\0\0\0", 4), 0); + +#if ASMJIT_ARCH_BITS >= 64 + TestMRFunc fn_add_u64 = create_func_mr(ctx, UniOpMR::kAddU64); + mem.u64 = 0xF123456789ABCDEFu; + fn_add_u64(mem.buffer, 0x0102030405060708u); + EXPECT_EQ(mem.u64, 0xF225486B8EB1D4F7u); +#endif + + TestMRFunc fn_add_reg = create_func_mr(ctx, UniOpMR::kAddReg); + mem.u64 = 0xFFFFFFFFFFFFFFFF; +#if ASMJIT_ARCH_BITS >= 64 + fn_add_reg(mem.buffer, 1); + EXPECT_EQ(mem.u64, 0u); +#else + mem.u32 = 0x01020304; + fn_add_reg(mem.buffer, 0x02030405); + EXPECT_EQ(mem.u32, 0x03050709u); + EXPECT_EQ(memcmp(mem.buffer + 4, "\xFF\xFF\xFF\xFF", 4), 0); +#endif + + ctx.rt.reset(); +} + +// ujit::UniCompiler - Tests - RR Operations - Functions +// ===================================================== + +static TestRRFunc create_func_rr(JitContext& ctx, UniOpRR op) { + ctx.prepare(); + + UniCompiler uc(&ctx.cc, ctx.features, ctx.cpu_hints); + uc.init_vec_width(VecWidth::k128); + + FuncNode* node = uc.add_func(FuncSignature::build<uint32_t, uint32_t>()); + EXPECT_NOT_NULL(node); + + Gp r = uc.new_gp32("r"); + node->set_arg(0, r); + uc.emit_2i(op, r, r); + uc.ret(r); + + uc.end_func(); + return ctx.finish<TestRRFunc>(); +} + +// ujit::UniCompiler - Tests - RR Operations - Runner +// ================================================== + +static ASMJIT_NOINLINE void test_rr_ops(JitContext& ctx) { + TestRRFunc fn_abs = create_func_rr(ctx, UniOpRR::kAbs); + EXPECT_EQ(fn_abs(0u), 0u); + EXPECT_EQ(fn_abs(1u), 1u); + EXPECT_EQ(fn_abs(uint32_t(-1)), 1u); + EXPECT_EQ(fn_abs(uint32_t(-333)), 333u); + EXPECT_EQ(fn_abs(0x80000000u), 0x80000000u); + + TestRRFunc fn_neg = create_func_rr(ctx, UniOpRR::kNeg); + EXPECT_EQ(fn_neg(0u), 0u); + EXPECT_EQ(fn_neg(1u), uint32_t(-1)); + EXPECT_EQ(fn_neg(uint32_t(-1)), 1u); + EXPECT_EQ(fn_neg(uint32_t(-333)), 333u); + EXPECT_EQ(fn_neg(333u), uint32_t(-333)); + EXPECT_EQ(fn_neg(0x80000000u), 0x80000000u); + + TestRRFunc fn_not = create_func_rr(ctx, UniOpRR::kNot); + EXPECT_EQ(fn_not(0u), 0xFFFFFFFFu); + EXPECT_EQ(fn_not(1u), 0xFFFFFFFEu); + EXPECT_EQ(fn_not(0xFFFFFFFF), 0u); + EXPECT_EQ(fn_not(0x12333245), ~0x12333245u); + EXPECT_EQ(fn_not(0x80000000u), 0x7FFFFFFFu); + + TestRRFunc fn_bswap32 = create_func_rr(ctx, UniOpRR::kBSwap); + EXPECT_EQ(fn_bswap32(0x11223344u), 0x44332211u); + EXPECT_EQ(fn_bswap32(0xFFFF0000u), 0x0000FFFFu); + EXPECT_EQ(fn_bswap32(0x00000000u), 0x00000000u); + + TestRRFunc fn_clz32 = create_func_rr(ctx, UniOpRR::kCLZ); + EXPECT_EQ(fn_clz32(0x80000000u), 0u); + EXPECT_EQ(fn_clz32(0x40000000u), 1u); + EXPECT_EQ(fn_clz32(0x00800000u), 8u); + EXPECT_EQ(fn_clz32(0x00008000u), 16u); + EXPECT_EQ(fn_clz32(0x00000080u), 24u); + EXPECT_EQ(fn_clz32(0x00000001u), 31u); + + TestRRFunc fn_ctz32 = create_func_rr(ctx, UniOpRR::kCTZ); + EXPECT_EQ(fn_ctz32(0x80000000u), 31u); + EXPECT_EQ(fn_ctz32(0x40000000u), 30u); + EXPECT_EQ(fn_ctz32(0x00800000u), 23u); + EXPECT_EQ(fn_ctz32(0x00008000u), 15u); + EXPECT_EQ(fn_ctz32(0x00000080u), 7u); + EXPECT_EQ(fn_ctz32(0x00000001u), 0u); + + TestRRFunc fn_reflect = create_func_rr(ctx, UniOpRR::kReflect); + EXPECT_EQ(fn_reflect(0x00000000u), 0x00000000u); + EXPECT_EQ(fn_reflect(0x00FF0000u), 0x00FF0000u); + EXPECT_EQ(fn_reflect(0x000000FFu), 0x000000FFu); + EXPECT_EQ(fn_reflect(0x80000000u), 0x7FFFFFFFu); + EXPECT_EQ(fn_reflect(0xFFFFFFFFu), 0x00000000u); + EXPECT_EQ(fn_reflect(0x88FF0000u), 0x7700FFFFu); + + ctx.rt.reset(); +} + +// ujit::UniCompiler - Tests - RRR Operations - Functions +// ====================================================== + +static TestRRRFunc create_func_rrr(JitContext& ctx, UniOpRRR op) { + ctx.prepare(); + + UniCompiler uc(&ctx.cc, ctx.features, ctx.cpu_hints); + uc.init_vec_width(VecWidth::k128); + + FuncNode* node = uc.add_func(FuncSignature::build<uint32_t, uint32_t, uint32_t>()); + EXPECT_NOT_NULL(node); + + Gp a = uc.new_gp32("a"); + Gp b = uc.new_gp32("b"); + Gp result = uc.new_gp32("result"); + + node->set_arg(0, a); + node->set_arg(1, b); + + uc.emit_3i(op, result, a, b); + uc.ret(result); + + uc.end_func(); + return ctx.finish<TestRRRFunc>(); +} + +static TestRRIFunc create_func_rri(JitContext& ctx, UniOpRRR op, Imm bImm) { + ctx.prepare(); + + UniCompiler uc(&ctx.cc, ctx.features, ctx.cpu_hints); + uc.init_vec_width(VecWidth::k128); + + FuncNode* node = uc.add_func(FuncSignature::build<uint32_t, uint32_t>()); + EXPECT_NOT_NULL(node); + + Gp a = uc.new_gp32("a"); + Gp result = uc.new_gp32("result"); + + node->set_arg(0, a); + + uc.emit_3i(op, result, a, bImm); + uc.ret(result); + + uc.end_func(); + return ctx.finish<TestRRIFunc>(); +} + +// ujit::UniCompiler - Tests - RRR Operations - Runner +// =================================================== + +static ASMJIT_NOINLINE void test_rrr_op(JitContext& ctx, UniOpRRR op, uint32_t a, uint32_t b, uint32_t expected) { + TestRRRFunc fn_rrr = create_func_rrr(ctx, op); + uint32_t observed_rrr = fn_rrr(a, b); + EXPECT_EQ(observed_rrr, expected) + .message("Operation failed (RRR):\n" + " Input #1: %d\n" + " Input #2: %d\n" + " Expected: %d\n" + " Observed: %d\n" + "Assembly:\n%s", + a, + b, + uint32_t(expected), + observed_rrr, + ctx.logger_content()); + + TestRRIFunc fn_rri = create_func_rri(ctx, op, Imm(b)); + uint32_t observed_rri = fn_rri(a); + EXPECT_EQ(observed_rri, expected) + .message("Operation failed (RRI):\n" + " Input #1: %d\n" + " Input #2: %d\n" + " Expected: %d\n" + " Observed: %d\n" + "Assembly:\n%s", + a, + b, + uint32_t(expected), + observed_rri, + ctx.logger_content()); + + ctx.rt.reset(); +} + +static ASMJIT_NOINLINE void test_rrr_ops(JitContext& ctx) { + test_rrr_op(ctx, UniOpRRR::kAnd, 0u, 0u, 0u); + test_rrr_op(ctx, UniOpRRR::kAnd, 0xFFu, 0x11u, 0x11u); + test_rrr_op(ctx, UniOpRRR::kAnd, 0x11u, 0xFFu, 0x11u); + test_rrr_op(ctx, UniOpRRR::kAnd, 0xFF11u, 0x1111u, 0x1111u); + test_rrr_op(ctx, UniOpRRR::kAnd, 0x1111u, 0xFF11u, 0x1111u); + test_rrr_op(ctx, UniOpRRR::kAnd, 0x0000FFFFu, 0xFFFF0000u, 0u); + test_rrr_op(ctx, UniOpRRR::kAnd, 0xFFFFFFFFu, 0xFFFF0000u, 0xFFFF0000u); + test_rrr_op(ctx, UniOpRRR::kAnd, 0x11111111u, 0x11223344u, 0x11001100u); + + test_rrr_op(ctx, UniOpRRR::kOr, 0u, 0u, 0u); + test_rrr_op(ctx, UniOpRRR::kOr, 0xFFu, 0x11u, 0xFFu); + test_rrr_op(ctx, UniOpRRR::kOr, 0x11u, 0xFFu, 0xFFu); + test_rrr_op(ctx, UniOpRRR::kOr, 0xFF11u, 0x1111u, 0xFF11u); + test_rrr_op(ctx, UniOpRRR::kOr, 0x1111u, 0xFF11u, 0xFF11u); + test_rrr_op(ctx, UniOpRRR::kOr, 0x0000FFFFu, 0xFFFF0001u, 0xFFFFFFFFu); + test_rrr_op(ctx, UniOpRRR::kOr, 0xFFFFFFFFu, 0xFF000000u, 0xFFFFFFFFu); + test_rrr_op(ctx, UniOpRRR::kOr, 0x11111111u, 0x00223344u, 0x11333355u); + + test_rrr_op(ctx, UniOpRRR::kXor, 0u, 0u, 0u); + test_rrr_op(ctx, UniOpRRR::kXor, 0xFFu, 0x11u, 0xEEu); + test_rrr_op(ctx, UniOpRRR::kXor, 0x11u, 0xFFu, 0xEEu); + test_rrr_op(ctx, UniOpRRR::kXor, 0xFF11u, 0x1111u, 0xEE00u); + test_rrr_op(ctx, UniOpRRR::kXor, 0x1111u, 0xFF11u, 0xEE00u); + test_rrr_op(ctx, UniOpRRR::kXor, 0x0000FFFFu, 0xFFFF0001u, 0xFFFFFFFEu); + test_rrr_op(ctx, UniOpRRR::kXor, 0xFFFFFFFFu, 0xFF000000u, 0x00FFFFFFu); + test_rrr_op(ctx, UniOpRRR::kXor, 0x11111111u, 0x00223344u, 0x11332255u); + + test_rrr_op(ctx, UniOpRRR::kBic, 0u, 0u, 0u); + test_rrr_op(ctx, UniOpRRR::kBic, 0xFFu, 0x11u, 0xEEu); + test_rrr_op(ctx, UniOpRRR::kBic, 0x11u, 0xFFu, 0x00u); + test_rrr_op(ctx, UniOpRRR::kBic, 0xFF11u, 0x1111u, 0xEE00u); + test_rrr_op(ctx, UniOpRRR::kBic, 0x1111u, 0xFF11u, 0x0000u); + test_rrr_op(ctx, UniOpRRR::kBic, 0x0000FFFFu, 0xFFFF0000u, 0x0000FFFFu); + test_rrr_op(ctx, UniOpRRR::kBic, 0xFFFFFFFFu, 0xFFFF0000u, 0x0000FFFFu); + test_rrr_op(ctx, UniOpRRR::kBic, 0x11111111u, 0x11223344u, 0x00110011u); + + test_rrr_op(ctx, UniOpRRR::kAdd, 0u, 0u, 0u); + test_rrr_op(ctx, UniOpRRR::kAdd, 1u, 2u, 3u); + test_rrr_op(ctx, UniOpRRR::kAdd, 0xFF000000u, 0x00FFFFFFu, 0xFFFFFFFFu); + test_rrr_op(ctx, UniOpRRR::kAdd, 1u, 0xFFFu, 0x1000u); + test_rrr_op(ctx, UniOpRRR::kAdd, 1u, 0xFFF000u, 0xFFF001u); + + test_rrr_op(ctx, UniOpRRR::kSub, 1u, 2u, 0xFFFFFFFFu); + + test_rrr_op(ctx, UniOpRRR::kMul, 1000u, 999u, 999000u); + test_rrr_op(ctx, UniOpRRR::kMul, 0xFFFFu, 0x00010001u, 0xFFFFFFFFu); + + test_rrr_op(ctx, UniOpRRR::kUDiv, 100000u, 1000u, 100u); + + test_rrr_op(ctx, UniOpRRR::kUMod, 1999u, 1000u, 999u); + + test_rrr_op(ctx, UniOpRRR::kSMin, uint32_t(1111), uint32_t(0), uint32_t(0)); + test_rrr_op(ctx, UniOpRRR::kSMin, uint32_t(-1111), uint32_t(0), uint32_t(-1111)); + test_rrr_op(ctx, UniOpRRR::kSMin, uint32_t(1), uint32_t(22), uint32_t(1)); + test_rrr_op(ctx, UniOpRRR::kSMin, uint32_t(1), uint32_t(0), uint32_t(0)); + test_rrr_op(ctx, UniOpRRR::kSMin, uint32_t(100101033), uint32_t(999), uint32_t(999)); + test_rrr_op(ctx, UniOpRRR::kSMin, uint32_t(100101033), uint32_t(112), uint32_t(112)); + test_rrr_op(ctx, UniOpRRR::kSMin, uint32_t(112), uint32_t(1125532), uint32_t(112)); + test_rrr_op(ctx, UniOpRRR::kSMin, uint32_t(1111), uint32_t(-1), uint32_t(-1)); + test_rrr_op(ctx, UniOpRRR::kSMin, uint32_t(-1111), uint32_t(-1), uint32_t(-1111)); + test_rrr_op(ctx, UniOpRRR::kSMin, uint32_t(-1), uint32_t(-22), uint32_t(-22)); + test_rrr_op(ctx, UniOpRRR::kSMin, uint32_t(-1), uint32_t(-128), uint32_t(-128)); + test_rrr_op(ctx, UniOpRRR::kSMin, uint32_t(-128), uint32_t(-1), uint32_t(-128)); + test_rrr_op(ctx, UniOpRRR::kSMin, uint32_t(-128), uint32_t(9), uint32_t(-128)); + test_rrr_op(ctx, UniOpRRR::kSMin, uint32_t(12444), uint32_t(-1), uint32_t(-1)); + + test_rrr_op(ctx, UniOpRRR::kSMax, uint32_t(1), uint32_t(22), uint32_t(22)); + test_rrr_op(ctx, UniOpRRR::kSMax, uint32_t(1), uint32_t(0), uint32_t(1)); + test_rrr_op(ctx, UniOpRRR::kSMax, uint32_t(100101033), uint32_t(999), uint32_t(100101033)); + test_rrr_op(ctx, UniOpRRR::kSMax, uint32_t(100101033), uint32_t(112), uint32_t(100101033)); + test_rrr_op(ctx, UniOpRRR::kSMax, uint32_t(112), uint32_t(1125532), uint32_t(1125532)); + test_rrr_op(ctx, UniOpRRR::kSMax, uint32_t(1111), uint32_t(-1), uint32_t(1111)); + test_rrr_op(ctx, UniOpRRR::kSMax, uint32_t(-1111), uint32_t(-1), uint32_t(-1)); + test_rrr_op(ctx, UniOpRRR::kSMax, uint32_t(-1), uint32_t(-22), uint32_t(-1)); + test_rrr_op(ctx, UniOpRRR::kSMax, uint32_t(-1), uint32_t(-128), uint32_t(-1)); + test_rrr_op(ctx, UniOpRRR::kSMax, uint32_t(-128), uint32_t(-1), uint32_t(-1)); + test_rrr_op(ctx, UniOpRRR::kSMax, uint32_t(-128), uint32_t(9), uint32_t(9)); + test_rrr_op(ctx, UniOpRRR::kSMax, uint32_t(12444), uint32_t(-1), uint32_t(12444)); + + test_rrr_op(ctx, UniOpRRR::kUMin, 1, 22, 1); + test_rrr_op(ctx, UniOpRRR::kUMin, 22, 1, 1); + test_rrr_op(ctx, UniOpRRR::kUMin, 1, 255, 1); + test_rrr_op(ctx, UniOpRRR::kUMin, 255, 1, 1); + test_rrr_op(ctx, UniOpRRR::kUMin, 1023, 255, 255); + test_rrr_op(ctx, UniOpRRR::kUMin, 255, 1023, 255); + test_rrr_op(ctx, UniOpRRR::kUMin, 0xFFFFFFFFu, 255, 255); + test_rrr_op(ctx, UniOpRRR::kUMin, 255, 0xFFFFFFFFu, 255); + test_rrr_op(ctx, UniOpRRR::kUMin, 0xFFFFFFFFu, 0xFFFFFF00u, 0xFFFFFF00u); + test_rrr_op(ctx, UniOpRRR::kUMin, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu); + + test_rrr_op(ctx, UniOpRRR::kUMax, 1, 22, 22); + test_rrr_op(ctx, UniOpRRR::kUMax, 22, 1, 22); + test_rrr_op(ctx, UniOpRRR::kUMax, 1, 255, 255); + test_rrr_op(ctx, UniOpRRR::kUMax, 255, 1, 255); + test_rrr_op(ctx, UniOpRRR::kUMax, 1023, 255, 1023); + test_rrr_op(ctx, UniOpRRR::kUMax, 255, 1023, 1023); + test_rrr_op(ctx, UniOpRRR::kUMax, 0xFFFFFFFFu, 255, 0xFFFFFFFFu); + test_rrr_op(ctx, UniOpRRR::kUMax, 255, 0xFFFFFFFFu, 0xFFFFFFFFu); + test_rrr_op(ctx, UniOpRRR::kUMax, 0xFFFFFFFFu, 0xFFFFFF00u, 0xFFFFFFFFu); + test_rrr_op(ctx, UniOpRRR::kUMax, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu); + + test_rrr_op(ctx, UniOpRRR::kSll, 1u, 1u, 1u << 1); + test_rrr_op(ctx, UniOpRRR::kSll, 1u, 22u, 1u << 22); + test_rrr_op(ctx, UniOpRRR::kSll, 1u, 31u, 1u << 31); + test_rrr_op(ctx, UniOpRRR::kSll, 0x7FFFFFFFu, 1u, 0xFFFFFFFEu); + + test_rrr_op(ctx, UniOpRRR::kSrl, 1u, 1u, 1u >> 1); + test_rrr_op(ctx, UniOpRRR::kSrl, 1u, 22u, 1u >> 22); + test_rrr_op(ctx, UniOpRRR::kSrl, 1u, 31u, 1u >> 31); + test_rrr_op(ctx, UniOpRRR::kSrl, 0x7FFFFFFFu, 1u, 0x7FFFFFFFu >> 1); + + test_rrr_op(ctx, UniOpRRR::kSra, 1u, 1u, 1u >> 1); + test_rrr_op(ctx, UniOpRRR::kSra, 1u, 22u, 1u >> 22); + test_rrr_op(ctx, UniOpRRR::kSra, 1u, 31u, 1u >> 31); + test_rrr_op(ctx, UniOpRRR::kSra, 0x7FFFFFFFu, 1u, 0x7FFFFFFFu >> 1); + test_rrr_op(ctx, UniOpRRR::kSra, 0xF0000000u, 4u, 0xFF000000u); + test_rrr_op(ctx, UniOpRRR::kSra, 0x80000000u, 31u, 0xFFFFFFFFu); + + test_rrr_op(ctx, UniOpRRR::kRol, 0x11223344u, 8u, 0x22334411u); + test_rrr_op(ctx, UniOpRRR::kRol, 0x11223344u, 16u, 0x33441122u); + test_rrr_op(ctx, UniOpRRR::kRol, 0xFCFFDABBu, 1u, 0xF9FFB577u); + + test_rrr_op(ctx, UniOpRRR::kRor, 0x11223344u, 8u, 0x44112233u); + test_rrr_op(ctx, UniOpRRR::kRor, 0x11223344u, 16u, 0x33441122u); + test_rrr_op(ctx, UniOpRRR::kRor, 0xF0000000u, 1u, 0x78000000u); + + test_rrr_op(ctx, UniOpRRR::kSBound, 0, 244u, 0); + test_rrr_op(ctx, UniOpRRR::kSBound, 42, 244u, 42u); + test_rrr_op(ctx, UniOpRRR::kSBound, 1111, 244u, 244u); + test_rrr_op(ctx, UniOpRRR::kSBound, 9999999, 111244u, 111244u); + test_rrr_op(ctx, UniOpRRR::kSBound, uint32_t(int32_t(-1)), 1000u, 0u); + test_rrr_op(ctx, UniOpRRR::kSBound, uint32_t(INT32_MIN), 100000u, 0u); + test_rrr_op(ctx, UniOpRRR::kSBound, uint32_t(INT32_MAX), 0u, 0u); + test_rrr_op(ctx, UniOpRRR::kSBound, uint32_t(INT32_MAX), 100000u, 100000u); + test_rrr_op(ctx, UniOpRRR::kSBound, uint32_t(INT32_MAX), uint32_t(INT32_MAX), uint32_t(INT32_MAX)); +} + +// ujit::UniCompiler - Tests - SIMD - Functions +// ============================================ + +// The following variations are supported: +// - 0 - separate destination & source registers +// - 1 - destination register is a source register as well +// - 2 - source is a memory operand +// - 3 - source register is a GP register (only for broadcasts from a GP register, otherwise maps to 0) +static constexpr uint32_t kNumVariationsVV = 3; +static constexpr uint32_t kNumVariationsVV_Broadcast = 4; + +static TestVVFunc create_func_vv(JitContext& ctx, VecWidth vw, UniOpVV op, Variation variation = Variation{0}) { + ctx.prepare(); + + UniCompiler uc(&ctx.cc, ctx.features, ctx.cpu_hints); + uc.init_vec_width(vw); + + FuncNode* node = uc.add_func(FuncSignature::build<void, void*, const void*>()); + EXPECT_NOT_NULL(node); + + Gp dst_ptr = uc.new_gpz("dst_ptr"); + Gp src_ptr = uc.new_gpz("src_ptr"); + + node->set_arg(0, dst_ptr); + node->set_arg(1, src_ptr); + + Vec dst_vec = uc.new_vec_with_width(vw, "dst_vec"); + + // There are some instructions that fill the high part of the register, so just zero the destination to make + // sure that we can test this function (that the low part is actually zeroed and doesn't contain garbage). + uc.v_zero_i(dst_vec); + + if (variation == 3u && (op == UniOpVV::kBroadcastU8 || + op == UniOpVV::kBroadcastU8Z || + op == UniOpVV::kBroadcastU32 || + op == UniOpVV::kBroadcastU64 || + op == UniOpVV::kBroadcastF32 || + op == UniOpVV::kBroadcastF64)) { + // This is used to test broadcasts from a GP register to a vector register. + Gp src_gp = uc.new_gpz("src_gp"); + + switch (op) { + case UniOpVV::kBroadcastU8: + case UniOpVV::kBroadcastU8Z: + uc.load_u8(src_gp, mem_ptr(src_ptr)); + uc.emit_2v(op, dst_vec, src_gp); + break; + + case UniOpVV::kBroadcastU16: + case UniOpVV::kBroadcastU16Z: + uc.load_u16(src_gp, mem_ptr(src_ptr)); + uc.emit_2v(op, dst_vec, src_gp); + break; + + case UniOpVV::kBroadcastU32: + case UniOpVV::kBroadcastF32: + uc.load_u32(src_gp, mem_ptr(src_ptr)); + uc.emit_2v(op, dst_vec, src_gp); + break; + + case UniOpVV::kBroadcastU64: + case UniOpVV::kBroadcastF64: + // Prevent using 64-bit registers on 32-bit architectures (that would fail). + if (uc.is_64bit()) { + uc.load_u64(src_gp, mem_ptr(src_ptr)); + uc.emit_2v(op, dst_vec, src_gp); + } + else { + uc.emit_2v(op, dst_vec, mem_ptr(src_ptr)); + } + break; + + default: + ASMJIT_NOT_REACHED(); + } + } + else if (variation == 2u) { + uc.emit_2v(op, dst_vec, mem_ptr(src_ptr)); + } + else if (variation == 1u) { + uc.v_loaduvec(dst_vec, mem_ptr(src_ptr)); + uc.emit_2v(op, dst_vec, dst_vec); + } + else { + Vec src_vec = uc.new_vec_with_width(vw, "src_vec"); + uc.v_loaduvec(src_vec, mem_ptr(src_ptr)); + uc.emit_2v(op, dst_vec, src_vec); + } + + uc.v_storeuvec(mem_ptr(dst_ptr), dst_vec); + + uc.end_func(); + return ctx.finish<TestVVFunc>(); +} + +// The following variations are supported: +// - 0 - separate destination & source registers +// - 1 - destination register is a source register as well +// - 2 - source is a memory operand +static constexpr uint32_t kNumVariationsVVI = 3; + +static TestVVFunc create_func_vvi(JitContext& ctx, VecWidth vw, UniOpVVI op, uint32_t imm, Variation variation = Variation{0}) { + ctx.prepare(); + + UniCompiler uc(&ctx.cc, ctx.features, ctx.cpu_hints); + uc.init_vec_width(vw); + + FuncNode* node = uc.add_func(FuncSignature::build<void, void*, const void*>()); + EXPECT_NOT_NULL(node); + + Gp dst_ptr = uc.new_gpz("dst_ptr"); + Gp src_ptr = uc.new_gpz("src_ptr"); + + node->set_arg(0, dst_ptr); + node->set_arg(1, src_ptr); + + Vec src_vec = uc.new_vec_with_width(vw, "src_vec"); + + switch (variation.value) { + default: + case 0: { + // There are some instructions that fill the high part of the register, so just zero the destination to make + // sure that we can test this function (that the low part is actually zeroed and doesn't contain garbage). + Vec dst_vec = uc.new_vec_with_width(vw, "dst_vec"); + uc.v_zero_i(dst_vec); + + uc.v_loaduvec(src_vec, mem_ptr(src_ptr)); + uc.emit_2vi(op, dst_vec, src_vec, imm); + uc.v_storeuvec(mem_ptr(dst_ptr), dst_vec); + + break; + } + + case 1: { + uc.v_loaduvec(src_vec, mem_ptr(src_ptr)); + uc.emit_2vi(op, src_vec, src_vec, imm); + uc.v_storeuvec(mem_ptr(dst_ptr), src_vec); + break; + } + + case 2: { + Vec dst_vec = uc.new_vec_with_width(vw, "dst_vec"); + uc.emit_2vi(op, dst_vec, mem_ptr(src_ptr), imm); + uc.v_storeuvec(mem_ptr(dst_ptr), dst_vec); + break; + } + } + + uc.end_func(); + return ctx.finish<TestVVFunc>(); +} + +// The following variations are supported: +// - 0 - separate destination & source registers +// - 1 - destination register is the same as the first source register +// - 2 - destination register is the same as the second source register +// - 3 - separate destination & source registers, the second source is a memory operand +// - 4 - destination register is the same as the first source register, second source is a memory operand +static constexpr uint32_t kNumVariationsVVV = 5; + +static TestVVVFunc create_func_vvv(JitContext& ctx, VecWidth vw, UniOpVVV op, Variation variation = Variation{0}) { + ctx.prepare(); + + UniCompiler uc(&ctx.cc, ctx.features, ctx.cpu_hints); + uc.init_vec_width(vw); + + FuncNode* node = uc.add_func(FuncSignature::build<void, void*, const void*, const void*>()); + EXPECT_NOT_NULL(node); + + Gp dst_ptr = uc.new_gpz("dst_ptr"); + Gp src1_ptr = uc.new_gpz("src1_ptr"); + Gp src2_ptr = uc.new_gpz("src2_ptr"); + + node->set_arg(0, dst_ptr); + node->set_arg(1, src1_ptr); + node->set_arg(2, src2_ptr); + + Vec src1_vec = uc.new_vec_with_width(vw, "src1_vec"); + Vec src2_vec = uc.new_vec_with_width(vw, "src2_vec"); + + switch (variation.value) { + default: + case 0: { + // There are some instructions that fill the high part of the register, so just zero the destination to make + // sure that we can test this function (that the low part is actually zeroed and doesn't contain garbage). + Vec dst_vec = uc.new_vec_with_width(vw, "dst_vec"); + uc.v_zero_i(dst_vec); + + uc.v_loaduvec(src1_vec, mem_ptr(src1_ptr)); + uc.v_loaduvec(src2_vec, mem_ptr(src2_ptr)); + uc.emit_3v(op, dst_vec, src1_vec, src2_vec); + uc.v_storeuvec(mem_ptr(dst_ptr), dst_vec); + + break; + } + + case 1: { + uc.v_loaduvec(src1_vec, mem_ptr(src1_ptr)); + uc.v_loaduvec(src2_vec, mem_ptr(src2_ptr)); + uc.emit_3v(op, src1_vec, src1_vec, src2_vec); + uc.v_storeuvec(mem_ptr(dst_ptr), src1_vec); + + break; + } + + case 2: { + uc.v_loaduvec(src1_vec, mem_ptr(src1_ptr)); + uc.v_loaduvec(src2_vec, mem_ptr(src2_ptr)); + uc.emit_3v(op, src2_vec, src1_vec, src2_vec); + uc.v_storeuvec(mem_ptr(dst_ptr), src2_vec); + + break; + } + + case 3: { + Vec dst_vec = uc.new_vec_with_width(vw, "dst_vec"); + uc.v_zero_i(dst_vec); + + uc.v_loaduvec(src1_vec, mem_ptr(src1_ptr)); + uc.emit_3v(op, dst_vec, src1_vec, mem_ptr(src2_ptr)); + uc.v_storeuvec(mem_ptr(dst_ptr), dst_vec); + + break; + } + + case 4: { + uc.v_loaduvec(src1_vec, mem_ptr(src1_ptr)); + uc.emit_3v(op, src1_vec, src1_vec, mem_ptr(src2_ptr)); + uc.v_storeuvec(mem_ptr(dst_ptr), src1_vec); + + break; + } + } + + uc.end_func(); + return ctx.finish<TestVVVFunc>(); +} + +static constexpr uint32_t kNumVariationsVVVI = 5; + +static TestVVVFunc create_func_vvvi(JitContext& ctx, VecWidth vw, UniOpVVVI op, uint32_t imm, Variation variation = Variation{0}) { + ctx.prepare(); + + UniCompiler uc(&ctx.cc, ctx.features, ctx.cpu_hints); + uc.init_vec_width(vw); + + FuncNode* node = uc.add_func(FuncSignature::build<void, void*, const void*, const void*>()); + EXPECT_NOT_NULL(node); + + Gp dst_ptr = uc.new_gpz("dst_ptr"); + Gp src1_ptr = uc.new_gpz("src1_ptr"); + Gp src2_ptr = uc.new_gpz("src2_ptr"); + + node->set_arg(0, dst_ptr); + node->set_arg(1, src1_ptr); + node->set_arg(2, src2_ptr); + + Vec src1_vec = uc.new_vec_with_width(vw, "src1_vec"); + Vec src2_vec = uc.new_vec_with_width(vw, "src2_vec"); + + switch (variation.value) { + default: + case 0: { + // There are some instructions that fill the high part of the register, so just zero the destination to make + // sure that we can test this function (that the low part is actually zeroed and doesn't contain garbage). + Vec dst_vec = uc.new_vec_with_width(vw, "dst_vec"); + uc.v_zero_i(dst_vec); + + uc.v_loaduvec(src1_vec, mem_ptr(src1_ptr)); + uc.v_loaduvec(src2_vec, mem_ptr(src2_ptr)); + uc.emit_3vi(op, dst_vec, src1_vec, src2_vec, imm); + uc.v_storeuvec(mem_ptr(dst_ptr), dst_vec); + + break; + } + + case 1: { + uc.v_loaduvec(src1_vec, mem_ptr(src1_ptr)); + uc.v_loaduvec(src2_vec, mem_ptr(src2_ptr)); + uc.emit_3vi(op, src1_vec, src1_vec, src2_vec, imm); + uc.v_storeuvec(mem_ptr(dst_ptr), src1_vec); + + break; + } + + case 2: { + uc.v_loaduvec(src1_vec, mem_ptr(src1_ptr)); + uc.v_loaduvec(src2_vec, mem_ptr(src2_ptr)); + uc.emit_3vi(op, src2_vec, src1_vec, src2_vec, imm); + uc.v_storeuvec(mem_ptr(dst_ptr), src2_vec); + + break; + } + + case 3: { + Vec dst_vec = uc.new_vec_with_width(vw, "dst_vec"); + uc.v_zero_i(dst_vec); + + uc.v_loaduvec(src1_vec, mem_ptr(src1_ptr)); + uc.emit_3vi(op, dst_vec, src1_vec, mem_ptr(src2_ptr), imm); + uc.v_storeuvec(mem_ptr(dst_ptr), dst_vec); + + break; + } + + case 4: { + uc.v_loaduvec(src1_vec, mem_ptr(src1_ptr)); + uc.emit_3vi(op, src1_vec, src1_vec, mem_ptr(src2_ptr), imm); + uc.v_storeuvec(mem_ptr(dst_ptr), src1_vec); + + break; + } + } + + uc.end_func(); + return ctx.finish<TestVVVFunc>(); +} + +// The following variations are supported: +// - 0 - separate destination & source registers +// - 1 - destination register is the first source register +// - 2 - destination register is the second source register +// - 3 - destination register is the third source register +static constexpr uint32_t kNumVariationsVVVV = 4; + +static TestVVVVFunc create_func_vvvv(JitContext& ctx, VecWidth vw, UniOpVVVV op, Variation variation = Variation{0}) { + ctx.prepare(); + + UniCompiler uc(&ctx.cc, ctx.features, ctx.cpu_hints); + uc.init_vec_width(vw); + + FuncNode* node = uc.add_func(FuncSignature::build<void, void*, const void*, const void*, const void*>()); + EXPECT_NOT_NULL(node); + + Gp dst_ptr = uc.new_gpz("dst_ptr"); + Gp src1_ptr = uc.new_gpz("src1_ptr"); + Gp src2_ptr = uc.new_gpz("src2_ptr"); + Gp src3_ptr = uc.new_gpz("src3_ptr"); + + node->set_arg(0, dst_ptr); + node->set_arg(1, src1_ptr); + node->set_arg(2, src2_ptr); + node->set_arg(3, src3_ptr); + + Vec src1_vec = uc.new_vec_with_width(vw, "src1_vec"); + Vec src2_vec = uc.new_vec_with_width(vw, "src2_vec"); + Vec src3_vec = uc.new_vec_with_width(vw, "src3_vec"); + + uc.v_loaduvec(src1_vec, mem_ptr(src1_ptr)); + uc.v_loaduvec(src2_vec, mem_ptr(src2_ptr)); + uc.v_loaduvec(src3_vec, mem_ptr(src3_ptr)); + + switch (variation.value) { + default: + case 0: { + // There are some instructions that fill the high part of the register, so just zero the destination to make + // sure that we can test this function (that the low part is actually zeroed and doesn't contain garbage). + Vec dst_vec = uc.new_vec_with_width(vw, "dst_vec"); + uc.v_zero_i(dst_vec); + + uc.emit_4v(op, dst_vec, src1_vec, src2_vec, src3_vec); + uc.v_storeuvec(mem_ptr(dst_ptr), dst_vec); + + break; + } + + case 1: { + uc.emit_4v(op, src1_vec, src1_vec, src2_vec, src3_vec); + uc.v_storeuvec(mem_ptr(dst_ptr), src1_vec); + + break; + } + + case 2: { + uc.emit_4v(op, src2_vec, src1_vec, src2_vec, src3_vec); + uc.v_storeuvec(mem_ptr(dst_ptr), src2_vec); + + break; + } + + case 3: { + uc.emit_4v(op, src3_vec, src1_vec, src2_vec, src3_vec); + uc.v_storeuvec(mem_ptr(dst_ptr), src3_vec); + + break; + } + } + + uc.end_func(); + return ctx.finish<TestVVVVFunc>(); +} +// ujit::UniCompiler - Tests - SIMD - Vector Overlay +// ================================================= + +enum class VecElementType : uint8_t { + kInt8, + kInt16, + kInt32, + kInt64, + kUInt8, + kUInt16, + kUInt32, + kUInt64, + kFloat32, + kFloat64 +}; + +struct VecOpInfo { + uint32_t _data; + + ASMJIT_INLINE_NODEBUG uint32_t count() const noexcept { return _data >> 28; } + + ASMJIT_INLINE_NODEBUG VecElementType ret() const noexcept { return VecElementType((_data >> 24) & 0xFu); } + ASMJIT_INLINE_NODEBUG VecElementType arg(uint32_t i) const noexcept { return VecElementType((_data >> (i * 4)) & 0xFu); } + + static ASMJIT_INLINE_NODEBUG VecOpInfo make(VecElementType ret, VecElementType arg0) noexcept { + return VecOpInfo{(1u << 28) | (uint32_t(ret) << 24) | uint32_t(arg0)}; + } + + static ASMJIT_INLINE_NODEBUG VecOpInfo make(VecElementType ret, VecElementType arg0, VecElementType arg1) noexcept { + return VecOpInfo{(1u << 28) | (uint32_t(ret) << 24) | uint32_t(arg0) | (uint32_t(arg1) << 4)}; + } + + static ASMJIT_INLINE_NODEBUG VecOpInfo make(VecElementType ret, VecElementType arg0, VecElementType arg1, VecElementType arg2) noexcept { + return VecOpInfo{(1u << 28) | (uint32_t(ret) << 24) | uint32_t(arg0) | (uint32_t(arg1) << 4) | (uint32_t(arg2) << 8)}; + } + + static ASMJIT_INLINE_NODEBUG VecOpInfo make(VecElementType ret, VecElementType arg0, VecElementType arg1, VecElementType arg2, VecElementType arg3) noexcept { + return VecOpInfo{(1u << 28) | (uint32_t(ret) << 24) | uint32_t(arg0) | (uint32_t(arg1) << 4) | (uint32_t(arg2) << 8) | (uint32_t(arg3) << 12)}; + } +}; + +template<uint32_t kW> +struct alignas(16) VecOverlay { + union { + int8_t data_i8[kW]; + uint8_t data_u8[kW]; + + int16_t data_i16[kW / 2u]; + uint16_t data_u16[kW / 2u]; + + int32_t data_i32[kW / 4u]; + uint32_t data_u32[kW / 4u]; + + int64_t data_i64[kW / 8u]; + uint64_t data_u64[kW / 8u]; + + float data_f32[kW / 4u]; + double data_f64[kW / 8u]; + }; + + template<typename T> + ASMJIT_INLINE_NODEBUG T* data() noexcept; + + template<typename T> + ASMJIT_INLINE_NODEBUG const T* data() const noexcept; + + template<typename T> + ASMJIT_INLINE_NODEBUG T get(size_t index) const noexcept; + + template<typename T> + ASMJIT_INLINE_NODEBUG void set(size_t index, const T& value) noexcept; + + template<uint32_t kOtherW> + ASMJIT_INLINE_NODEBUG void copy_16b_from(const VecOverlay<kOtherW>& other) noexcept { + data_u64[0] = other.data_u64[0]; + data_u64[1] = other.data_u64[1]; + } +}; + +template<typename T> +struct VecAccess; + +template<> +struct VecAccess<int8_t> { + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG int8_t* data(VecOverlay<kW>& vec) noexcept { return vec.data_i8; } + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG const int8_t* data(const VecOverlay<kW>& vec) noexcept { return vec.data_i8; } + + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG int8_t get(const VecOverlay<kW>& vec, size_t index) noexcept { return vec.data_i8[index]; } + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG void set(VecOverlay<kW>& vec, size_t index, int8_t value) noexcept { vec.data_i8[index] = value; } +}; + +template<> +struct VecAccess<int16_t> { + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG int16_t* data(VecOverlay<kW>& vec) noexcept { return vec.data_i16; } + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG const int16_t* data(const VecOverlay<kW>& vec) noexcept { return vec.data_i16; } + + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG int16_t get(const VecOverlay<kW>& vec, size_t index) noexcept { return vec.data_i16[index]; } + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG void set(VecOverlay<kW>& vec, size_t index, int16_t value) noexcept { vec.data_i16[index] = value; } +}; + +template<> +struct VecAccess<int32_t> { + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG int32_t* data(VecOverlay<kW>& vec) noexcept { return vec.data_i32; } + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG const int32_t* data(const VecOverlay<kW>& vec) noexcept { return vec.data_i32; } + + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG int32_t get(const VecOverlay<kW>& vec, size_t index) noexcept { return vec.data_i32[index]; } + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG void set(VecOverlay<kW>& vec, size_t index, int32_t value) noexcept { vec.data_i32[index] = value; } +}; + +template<> +struct VecAccess<int64_t> { + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG int64_t* data(VecOverlay<kW>& vec) noexcept { return vec.data_i64; } + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG const int64_t* data(const VecOverlay<kW>& vec) noexcept { return vec.data_i64; } + + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG int64_t get(const VecOverlay<kW>& vec, size_t index) noexcept { return vec.data_i64[index]; } + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG void set(VecOverlay<kW>& vec, size_t index, int64_t value) noexcept { vec.data_i64[index] = value; } +}; + +template<> +struct VecAccess<uint8_t> { + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG uint8_t* data(VecOverlay<kW>& vec) noexcept { return vec.data_u8; } + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG const uint8_t* data(const VecOverlay<kW>& vec) noexcept { return vec.data_u8; } + + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG uint8_t get(const VecOverlay<kW>& vec, size_t index) noexcept { return vec.data_u8[index]; } + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG void set(VecOverlay<kW>& vec, size_t index, uint8_t value) noexcept { vec.data_u8[index] = value; } +}; + +template<> +struct VecAccess<uint16_t> { + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG uint16_t* data(VecOverlay<kW>& vec) noexcept { return vec.data_u16; } + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG const uint16_t* data(const VecOverlay<kW>& vec) noexcept { return vec.data_u16; } + + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG uint16_t get(const VecOverlay<kW>& vec, size_t index) noexcept { return vec.data_u16[index]; } + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG void set(VecOverlay<kW>& vec, size_t index, uint16_t value) noexcept { vec.data_u16[index] = value; } +}; + +template<> +struct VecAccess<uint32_t> { + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG uint32_t* data(VecOverlay<kW>& vec) noexcept { return vec.data_u32; } + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG const uint32_t* data(const VecOverlay<kW>& vec) noexcept { return vec.data_u32; } + + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG uint32_t get(const VecOverlay<kW>& vec, size_t index) noexcept { return vec.data_u32[index]; } + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG void set(VecOverlay<kW>& vec, size_t index, uint32_t value) noexcept { vec.data_u32[index] = value; } +}; + +template<> +struct VecAccess<uint64_t> { + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG uint64_t* data(VecOverlay<kW>& vec) noexcept { return vec.data_u64; } + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG const uint64_t* data(const VecOverlay<kW>& vec) noexcept { return vec.data_u64; } + + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG uint64_t get(const VecOverlay<kW>& vec, size_t index) noexcept { return vec.data_u64[index]; } + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG void set(VecOverlay<kW>& vec, size_t index, uint64_t value) noexcept { vec.data_u64[index] = value; } +}; + +template<> +struct VecAccess<float> { + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG float* data(VecOverlay<kW>& vec) noexcept { return vec.data_f32; } + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG const float* data(const VecOverlay<kW>& vec) noexcept { return vec.data_f32; } + + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG float get(const VecOverlay<kW>& vec, size_t index) noexcept { return vec.data_f32[index]; } + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG void set(VecOverlay<kW>& vec, size_t index, float value) noexcept { vec.data_f32[index] = value; } +}; + +template<> +struct VecAccess<double> { + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG double* data(VecOverlay<kW>& vec) noexcept { return vec.data_f64; } + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG const double* data(const VecOverlay<kW>& vec) noexcept { return vec.data_f64; } + + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG double get(const VecOverlay<kW>& vec, size_t index) noexcept { return vec.data_f64[index]; } + template<uint32_t kW> static ASMJIT_INLINE_NODEBUG void set(VecOverlay<kW>& vec, size_t index, double value) noexcept { vec.data_f64[index] = value; } +}; + +template<uint32_t kW> +template<typename T> +ASMJIT_INLINE_NODEBUG T* VecOverlay<kW>::data() noexcept { return VecAccess<T>::data(*this); } + +template<uint32_t kW> +template<typename T> +ASMJIT_INLINE_NODEBUG const T* VecOverlay<kW>::data() const noexcept { return VecAccess<T>::data(*this); } + +template<uint32_t kW> +template<typename T> +ASMJIT_INLINE_NODEBUG T VecOverlay<kW>::get(size_t index) const noexcept { return VecAccess<T>::get(*this, index); } + +template<uint32_t kW> +template<typename T> +ASMJIT_INLINE_NODEBUG void VecOverlay<kW>::set(size_t index, const T& value) noexcept { return VecAccess<T>::set(*this, index, value); } + +template<typename T> struct TypeNameToString {}; +template<> struct TypeNameToString<int8_t > { static ASMJIT_INLINE_NODEBUG const char* get() noexcept { return "int8"; } }; +template<> struct TypeNameToString<int16_t > { static ASMJIT_INLINE_NODEBUG const char* get() noexcept { return "int16"; } }; +template<> struct TypeNameToString<int32_t > { static ASMJIT_INLINE_NODEBUG const char* get() noexcept { return "int32"; } }; +template<> struct TypeNameToString<int64_t > { static ASMJIT_INLINE_NODEBUG const char* get() noexcept { return "int64"; } }; +template<> struct TypeNameToString<uint8_t > { static ASMJIT_INLINE_NODEBUG const char* get() noexcept { return "uint8"; } }; +template<> struct TypeNameToString<uint16_t> { static ASMJIT_INLINE_NODEBUG const char* get() noexcept { return "uint16"; } }; +template<> struct TypeNameToString<uint32_t> { static ASMJIT_INLINE_NODEBUG const char* get() noexcept { return "uint32"; } }; +template<> struct TypeNameToString<uint64_t> { static ASMJIT_INLINE_NODEBUG const char* get() noexcept { return "uint64"; } }; +template<> struct TypeNameToString<float > { static ASMJIT_INLINE_NODEBUG const char* get() noexcept { return "float32"; } }; +template<> struct TypeNameToString<double > { static ASMJIT_INLINE_NODEBUG const char* get() noexcept { return "float64"; } }; + +template<uint32_t kW> +static bool vec_eq(const VecOverlay<kW>& a, const VecOverlay<kW>& b) noexcept { + return memcmp(a.data_u8, b.data_u8, kW) == 0; +} + +template<typename T> +static bool float_eq(const T& a, const T& b) noexcept { + return a == b || (std::isnan(a) && std::isnan(b)); +} + +template<uint32_t kW> +static bool vec_eq(const VecOverlay<kW>& a, const VecOverlay<kW>& b, VecElementType element_type) noexcept { + if (element_type == VecElementType::kFloat32) { + size_t count = kW / sizeof(float); + for (size_t i = 0; i < count; i++) { + if (!float_eq(a.data_f32[i], b.data_f32[i])) { + return false; + } + } + return true; + } + else if (element_type == VecElementType::kFloat64) { + size_t count = kW / sizeof(double); + for (size_t i = 0; i < count; i++) { + if (!float_eq(a.data_f64[i], b.data_f64[i])) { + return false; + } + } + return true; + } + else { + return vec_eq(a, b); + } +} + +template<uint32_t kW> +static ASMJIT_NOINLINE String vec_stringify(const VecOverlay<kW>& vec, VecElementType element_type) noexcept { + String s; + s.append('{'); + + switch (element_type) { + case VecElementType::kInt8 : { for (uint32_t i = 0; i < kW ; i++) s.append_format("%s%d" , i == 0 ? "" : ", ", vec.data_i8[i]); break; } + case VecElementType::kInt16 : { for (uint32_t i = 0; i < kW / 2; i++) s.append_format("%s%d" , i == 0 ? "" : ", ", vec.data_i16[i]); break; } + case VecElementType::kInt32 : { for (uint32_t i = 0; i < kW / 4; i++) s.append_format("%s%d" , i == 0 ? "" : ", ", vec.data_i32[i]); break; } + case VecElementType::kInt64 : { for (uint32_t i = 0; i < kW / 8; i++) s.append_format("%s%lld", i == 0 ? "" : ", ", (long long)vec.data_i64[i]); break; } + case VecElementType::kUInt8 : { for (uint32_t i = 0; i < kW ; i++) s.append_format("%s%u" , i == 0 ? "" : ", ", vec.data_u8[i]); break; } + case VecElementType::kUInt16 : { for (uint32_t i = 0; i < kW / 2; i++) s.append_format("%s%u" , i == 0 ? "" : ", ", vec.data_u16[i]); break; } + case VecElementType::kUInt32 : { for (uint32_t i = 0; i < kW / 4; i++) s.append_format("%s%u" , i == 0 ? "" : ", ", vec.data_u32[i]); break; } + case VecElementType::kUInt64 : { for (uint32_t i = 0; i < kW / 8; i++) s.append_format("%s%llu", i == 0 ? "" : ", ", (unsigned long long)vec.data_u64[i]); break; } + case VecElementType::kFloat32: { for (uint32_t i = 0; i < kW / 4; i++) s.append_format("%s%.20f" , i == 0 ? "" : ", ", double(vec.data_f32[i])); break; } + case VecElementType::kFloat64: { for (uint32_t i = 0; i < kW / 8; i++) s.append_format("%s%.20f" , i == 0 ? "" : ", ", double(vec.data_f64[i])); break; } + + default: + ASMJIT_NOT_REACHED(); + } + + s.append('}'); + return s; +} + +// ujit::UniCompiler - Tests - SIMD - Metadata +// =========================================== + +static const char* vec_op_name_vv(UniOpVV op) noexcept { + switch (op) { + case UniOpVV::kMov : return "v_mov"; + case UniOpVV::kMovU64 : return "v_mov_u64"; + case UniOpVV::kBroadcastU8Z : return "v_broadcast_u8z"; + case UniOpVV::kBroadcastU16Z : return "v_broadcast_u16z"; + case UniOpVV::kBroadcastU8 : return "v_broadcast_u8"; + case UniOpVV::kBroadcastU16 : return "v_broadcast_u16"; + case UniOpVV::kBroadcastU32 : return "v_broadcast_u32"; + case UniOpVV::kBroadcastU64 : return "v_broadcast_u64"; + case UniOpVV::kBroadcastF32 : return "v_broadcast_f32"; + case UniOpVV::kBroadcastF64 : return "v_broadcast_f64"; + case UniOpVV::kBroadcastV128_U32 : return "v_broadcast_v128_u32"; + case UniOpVV::kBroadcastV128_U64 : return "v_broadcast_v128_u64"; + case UniOpVV::kBroadcastV128_F32 : return "v_broadcast_v128_f32"; + case UniOpVV::kBroadcastV128_F64 : return "v_broadcast_v128_f64"; + case UniOpVV::kBroadcastV256_U32 : return "v_broadcast_v256_u32"; + case UniOpVV::kBroadcastV256_U64 : return "v_broadcast_v256_u64"; + case UniOpVV::kBroadcastV256_F32 : return "v_broadcast_v256_f32"; + case UniOpVV::kBroadcastV256_F64 : return "v_broadcast_v256_f64"; + case UniOpVV::kAbsI8 : return "v_abs_i8"; + case UniOpVV::kAbsI16 : return "v_abs_i16"; + case UniOpVV::kAbsI32 : return "v_abs_i32"; + case UniOpVV::kAbsI64 : return "v_abs_i64"; + case UniOpVV::kNotU32 : return "v_not_u32"; + case UniOpVV::kNotU64 : return "v_not_u64"; + case UniOpVV::kCvtI8LoToI16 : return "v_cvt_i8_lo_to_i16"; + case UniOpVV::kCvtI8HiToI16 : return "v_cvt_i8_hi_to_i16"; + case UniOpVV::kCvtU8LoToU16 : return "v_cvt_u8_lo_to_u16"; + case UniOpVV::kCvtU8HiToU16 : return "v_cvt_u8_hi_to_u16"; + case UniOpVV::kCvtI8ToI32 : return "v_cvt_i8_to_i32"; + case UniOpVV::kCvtU8ToU32 : return "v_cvt_u8_to_u32"; + case UniOpVV::kCvtI16LoToI32 : return "v_cvt_i16_lo_to_i32"; + case UniOpVV::kCvtI16HiToI32 : return "v_cvt_i16_hi_to_i32"; + case UniOpVV::kCvtU16LoToU32 : return "v_cvt_u16_lo_to_u32"; + case UniOpVV::kCvtU16HiToU32 : return "v_cvt_u16_hi_to_u32"; + case UniOpVV::kCvtI32LoToI64 : return "v_cvt_i32_lo_to_i64"; + case UniOpVV::kCvtI32HiToI64 : return "v_cvt_i32_hi_to_i64"; + case UniOpVV::kCvtU32LoToU64 : return "v_cvt_u32_lo_to_u64"; + case UniOpVV::kCvtU32HiToU64 : return "v_cvt_u32_hi_to_u64"; + case UniOpVV::kAbsF32S : return "s_abs_f32"; + case UniOpVV::kAbsF64S : return "s_abs_f64"; + case UniOpVV::kAbsF32 : return "v_abs_f32"; + case UniOpVV::kAbsF64 : return "v_abs_f64"; + case UniOpVV::kNegF32S : return "s_neg_f32"; + case UniOpVV::kNegF64S : return "s_neg_f64"; + case UniOpVV::kNegF32 : return "v_neg_f32"; + case UniOpVV::kNegF64 : return "v_neg_f64"; + case UniOpVV::kNotF32 : return "v_not_f32"; + case UniOpVV::kNotF64 : return "v_not_f64"; + case UniOpVV::kTruncF32S : return "v_trunc_f32s"; + case UniOpVV::kTruncF64S : return "v_trunc_f64s"; + case UniOpVV::kTruncF32 : return "v_trunc_f32"; + case UniOpVV::kTruncF64 : return "v_trunc_f64"; + case UniOpVV::kFloorF32S : return "v_floor_f32s"; + case UniOpVV::kFloorF64S : return "v_floor_f64s"; + case UniOpVV::kFloorF32 : return "v_floor_f32"; + case UniOpVV::kFloorF64 : return "v_floor_f64"; + case UniOpVV::kCeilF32S : return "v_ceil_f32s"; + case UniOpVV::kCeilF64S : return "v_ceil_f64s"; + case UniOpVV::kCeilF32 : return "v_ceil_f32"; + case UniOpVV::kCeilF64 : return "v_ceil_f64"; + case UniOpVV::kRoundEvenF32S : return "v_round_even_f32s"; + case UniOpVV::kRoundEvenF64S : return "v_round_even_f64s"; + case UniOpVV::kRoundEvenF32 : return "v_round_even_f32"; + case UniOpVV::kRoundEvenF64 : return "v_round_even_f64"; + case UniOpVV::kRoundHalfAwayF32S : return "v_round_half_away_f32s"; + case UniOpVV::kRoundHalfAwayF64S : return "v_round_half_away_f64s"; + case UniOpVV::kRoundHalfAwayF32 : return "v_round_half_away_f32"; + case UniOpVV::kRoundHalfAwayF64 : return "v_round_half_away_f64"; + case UniOpVV::kRoundHalfUpF32S : return "v_round_half_up_f32s"; + case UniOpVV::kRoundHalfUpF64S : return "v_round_half_up_f64s"; + case UniOpVV::kRoundHalfUpF32 : return "v_round_half_up_f32"; + case UniOpVV::kRoundHalfUpF64 : return "v_round_half_up_f64"; + case UniOpVV::kRcpF32 : return "v_rcp_f32"; + case UniOpVV::kRcpF64 : return "v_rcp_f64"; + case UniOpVV::kSqrtF32S : return "v_sqrt_f32s"; + case UniOpVV::kSqrtF64S : return "v_sqrt_f64s"; + case UniOpVV::kSqrtF32 : return "v_sqrt_f32"; + case UniOpVV::kSqrtF64 : return "v_sqrt_f64"; + case UniOpVV::kCvtF32ToF64S : return "v_cvt_f32_to_f64s"; + case UniOpVV::kCvtF64ToF32S : return "v_cvt_f64_to_f32s"; + case UniOpVV::kCvtI32ToF32 : return "v_cvt_i32_to_f32"; + case UniOpVV::kCvtF32LoToF64 : return "v_cvt_f32_lo_to_f64"; + case UniOpVV::kCvtF32HiToF64 : return "v_cvt_f32_hi_to_f64"; + case UniOpVV::kCvtF64ToF32Lo : return "v_cvt_f64_to_f32_lo"; + case UniOpVV::kCvtF64ToF32Hi : return "v_cvt_f64_to_f32_hi"; + case UniOpVV::kCvtI32LoToF64 : return "v_cvt_i32_lo_to_f64"; + case UniOpVV::kCvtI32HiToF64 : return "v_cvt_i32_hi_to_f64"; + case UniOpVV::kCvtTruncF32ToI32 : return "v_cvt_trunc_f32_to_i32"; + case UniOpVV::kCvtTruncF64ToI32Lo: return "v_cvt_trunc_f64_to_i32_lo"; + case UniOpVV::kCvtTruncF64ToI32Hi: return "v_cvt_trunc_f64_to_i32_hi"; + case UniOpVV::kCvtRoundF32ToI32 : return "v_cvt_round_f32_to_i32"; + case UniOpVV::kCvtRoundF64ToI32Lo: return "v_cvt_round_f64_to_i32_lo"; + case UniOpVV::kCvtRoundF64ToI32Hi: return "v_cvt_round_f64_to_i32_hi"; + } + + ASMJIT_NOT_REACHED(); +} + +static VecOpInfo vec_op_info_vv(UniOpVV op) noexcept { + using VE = VecElementType; + + switch (op) { + case UniOpVV::kMov : return VecOpInfo::make(VE::kUInt8, VE::kUInt8); + case UniOpVV::kMovU64 : return VecOpInfo::make(VE::kUInt8, VE::kUInt8); + case UniOpVV::kBroadcastU8Z : return VecOpInfo::make(VE::kUInt8, VE::kUInt8); + case UniOpVV::kBroadcastU16Z : return VecOpInfo::make(VE::kUInt16, VE::kUInt16); + case UniOpVV::kBroadcastU8 : return VecOpInfo::make(VE::kUInt8, VE::kUInt8); + case UniOpVV::kBroadcastU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt16); + case UniOpVV::kBroadcastU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32); + case UniOpVV::kBroadcastU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64); + case UniOpVV::kBroadcastF32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVV::kBroadcastF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVV::kBroadcastV128_U32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32); + case UniOpVV::kBroadcastV128_U64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64); + case UniOpVV::kBroadcastV128_F32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVV::kBroadcastV128_F64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVV::kBroadcastV256_U32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32); + case UniOpVV::kBroadcastV256_U64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64); + case UniOpVV::kBroadcastV256_F32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVV::kBroadcastV256_F64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVV::kAbsI8 : return VecOpInfo::make(VE::kUInt8, VE::kInt8); + case UniOpVV::kAbsI16 : return VecOpInfo::make(VE::kUInt16, VE::kInt16); + case UniOpVV::kAbsI32 : return VecOpInfo::make(VE::kUInt32, VE::kInt32); + case UniOpVV::kAbsI64 : return VecOpInfo::make(VE::kUInt64, VE::kInt64); + case UniOpVV::kNotU32 : return VecOpInfo::make(VE::kUInt32, VE::kInt32); + case UniOpVV::kNotU64 : return VecOpInfo::make(VE::kUInt64, VE::kInt64); + case UniOpVV::kCvtI8LoToI16 : return VecOpInfo::make(VE::kInt16, VE::kInt8); + case UniOpVV::kCvtI8HiToI16 : return VecOpInfo::make(VE::kInt16, VE::kInt8); + case UniOpVV::kCvtU8LoToU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt8); + case UniOpVV::kCvtU8HiToU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt8); + case UniOpVV::kCvtI8ToI32 : return VecOpInfo::make(VE::kInt32, VE::kInt8); + case UniOpVV::kCvtU8ToU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt8); + case UniOpVV::kCvtI16LoToI32 : return VecOpInfo::make(VE::kInt32, VE::kInt16); + case UniOpVV::kCvtI16HiToI32 : return VecOpInfo::make(VE::kInt32, VE::kInt16); + case UniOpVV::kCvtU16LoToU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt16); + case UniOpVV::kCvtU16HiToU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt16); + case UniOpVV::kCvtI32LoToI64 : return VecOpInfo::make(VE::kInt64, VE::kInt32); + case UniOpVV::kCvtI32HiToI64 : return VecOpInfo::make(VE::kInt64, VE::kInt32); + case UniOpVV::kCvtU32LoToU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt32); + case UniOpVV::kCvtU32HiToU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt32); + case UniOpVV::kAbsF32S : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVV::kAbsF64S : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVV::kAbsF32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVV::kAbsF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVV::kNegF32S : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVV::kNegF64S : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVV::kNegF32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVV::kNegF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVV::kNotF32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32); + case UniOpVV::kNotF64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64); + case UniOpVV::kTruncF32S : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVV::kTruncF64S : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVV::kTruncF32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVV::kTruncF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVV::kFloorF32S : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVV::kFloorF64S : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVV::kFloorF32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVV::kFloorF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVV::kCeilF32S : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVV::kCeilF64S : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVV::kCeilF32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVV::kCeilF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVV::kRoundEvenF32S : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVV::kRoundEvenF64S : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVV::kRoundEvenF32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVV::kRoundEvenF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVV::kRoundHalfAwayF32S : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVV::kRoundHalfAwayF64S : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVV::kRoundHalfAwayF32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVV::kRoundHalfAwayF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVV::kRoundHalfUpF32S : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVV::kRoundHalfUpF64S : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVV::kRoundHalfUpF32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVV::kRoundHalfUpF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVV::kRcpF32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVV::kRcpF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVV::kSqrtF32S : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVV::kSqrtF64S : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVV::kSqrtF32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVV::kSqrtF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVV::kCvtF32ToF64S : return VecOpInfo::make(VE::kFloat64, VE::kFloat32); + case UniOpVV::kCvtF64ToF32S : return VecOpInfo::make(VE::kFloat32, VE::kFloat64); + case UniOpVV::kCvtI32ToF32 : return VecOpInfo::make(VE::kFloat32, VE::kInt32); + case UniOpVV::kCvtF32LoToF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat32); + case UniOpVV::kCvtF32HiToF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat32); + case UniOpVV::kCvtF64ToF32Lo : return VecOpInfo::make(VE::kFloat32, VE::kFloat64); + case UniOpVV::kCvtF64ToF32Hi : return VecOpInfo::make(VE::kFloat32, VE::kFloat64); + case UniOpVV::kCvtI32LoToF64 : return VecOpInfo::make(VE::kFloat64, VE::kInt32); + case UniOpVV::kCvtI32HiToF64 : return VecOpInfo::make(VE::kFloat64, VE::kInt32); + case UniOpVV::kCvtTruncF32ToI32 : return VecOpInfo::make(VE::kInt32, VE::kFloat32); + case UniOpVV::kCvtTruncF64ToI32Lo: return VecOpInfo::make(VE::kInt32, VE::kFloat64); + case UniOpVV::kCvtTruncF64ToI32Hi: return VecOpInfo::make(VE::kInt32, VE::kFloat64); + case UniOpVV::kCvtRoundF32ToI32 : return VecOpInfo::make(VE::kInt32, VE::kFloat32); + case UniOpVV::kCvtRoundF64ToI32Lo: return VecOpInfo::make(VE::kInt32, VE::kFloat64); + case UniOpVV::kCvtRoundF64ToI32Hi: return VecOpInfo::make(VE::kInt32, VE::kFloat64); + } + + ASMJIT_NOT_REACHED(); +} + +static const char* vec_op_name_vvi(UniOpVVI op) noexcept { + switch (op) { + case UniOpVVI::kSllU16 : return "v_sll_u16"; + case UniOpVVI::kSllU32 : return "v_sll_u32"; + case UniOpVVI::kSllU64 : return "v_sll_u64"; + case UniOpVVI::kSrlU16 : return "v_srl_u16"; + case UniOpVVI::kSrlU32 : return "v_srl_u32"; + case UniOpVVI::kSrlU64 : return "v_srl_u64"; + case UniOpVVI::kSraI16 : return "v_sra_i16"; + case UniOpVVI::kSraI32 : return "v_sra_i32"; + case UniOpVVI::kSraI64 : return "v_sra_i64"; + case UniOpVVI::kSllbU128 : return "v_sllb_u128"; + case UniOpVVI::kSrlbU128 : return "v_srlb_u128"; + case UniOpVVI::kSwizzleU16x4 : return "v_swizzle_u16x4"; + case UniOpVVI::kSwizzleLoU16x4 : return "v_swizzle_lo_u16x4"; + case UniOpVVI::kSwizzleHiU16x4 : return "v_swizzle_hi_u16x4"; + case UniOpVVI::kSwizzleU32x4 : return "v_swizzle_u32x4"; + case UniOpVVI::kSwizzleU64x2 : return "v_swizzle_u64x2"; + case UniOpVVI::kSwizzleF32x4 : return "v_swizzle_f32x4"; + case UniOpVVI::kSwizzleF64x2 : return "v_swizzle_f64x2"; + case UniOpVVI::kSwizzleU64x4 : return "v_swizzle_u64x4"; + case UniOpVVI::kSwizzleF64x4 : return "v_swizzle_f64x4"; + case UniOpVVI::kExtractV128_I32: return "v_extract_v128_i32"; + case UniOpVVI::kExtractV128_I64: return "v_extract_v128_i64"; + case UniOpVVI::kExtractV128_F32: return "v_extract_v128_f32"; + case UniOpVVI::kExtractV128_F64: return "v_extract_v128_f64"; + case UniOpVVI::kExtractV256_I32: return "v_extract_v256_i32"; + case UniOpVVI::kExtractV256_I64: return "v_extract_v256_i64"; + case UniOpVVI::kExtractV256_F32: return "v_extract_v256_f32"; + case UniOpVVI::kExtractV256_F64: return "v_extract_v256_f64"; + +#if defined(ASMJIT_UJIT_AARCH64) + case UniOpVVI::kSrlRndU16 : return "v_srl_rnd_u16"; + case UniOpVVI::kSrlRndU32 : return "v_srl_rnd_u32"; + case UniOpVVI::kSrlRndU64 : return "v_srl_rnd_u64"; + case UniOpVVI::kSrlAccU16 : return "v_srl_acc_u16"; + case UniOpVVI::kSrlAccU32 : return "v_srl_acc_u32"; + case UniOpVVI::kSrlAccU64 : return "v_srl_acc_u64"; + case UniOpVVI::kSrlRndAccU16 : return "v_srl_rnd_acc_u16"; + case UniOpVVI::kSrlRndAccU32 : return "v_srl_rnd_acc_u32"; + case UniOpVVI::kSrlRndAccU64 : return "v_srl_rnd_acc_u64"; + case UniOpVVI::kSrlnLoU16 : return "v_srln_lo_u16"; + case UniOpVVI::kSrlnHiU16 : return "v_srln_hi_u16"; + case UniOpVVI::kSrlnLoU32 : return "v_srln_lo_u32"; + case UniOpVVI::kSrlnHiU32 : return "v_srln_hi_u32"; + case UniOpVVI::kSrlnLoU64 : return "v_srln_lo_u64"; + case UniOpVVI::kSrlnHiU64 : return "v_srln_hi_u64"; + case UniOpVVI::kSrlnRndLoU16 : return "v_srln_rnd_lo_u16"; + case UniOpVVI::kSrlnRndHiU16 : return "v_srln_rnd_hi_u16"; + case UniOpVVI::kSrlnRndLoU32 : return "v_srln_rnd_lo_u32"; + case UniOpVVI::kSrlnRndHiU32 : return "v_srln_rnd_hi_u32"; + case UniOpVVI::kSrlnRndLoU64 : return "v_srln_rnd_lo_u64"; + case UniOpVVI::kSrlnRndHiU64 : return "v_srln_rnd_hi_u64"; +#endif // ASMJIT_UJIT_AARCH64 + } + + ASMJIT_NOT_REACHED(); +} + +static VecOpInfo vec_op_info_vvi(UniOpVVI op) noexcept { + using VE = VecElementType; + + switch (op) { + case UniOpVVI::kSllU16 : return VecOpInfo::make(VE::kUInt16 , VE::kUInt16); + case UniOpVVI::kSllU32 : return VecOpInfo::make(VE::kUInt32 , VE::kUInt32); + case UniOpVVI::kSllU64 : return VecOpInfo::make(VE::kUInt64 , VE::kUInt64); + case UniOpVVI::kSrlU16 : return VecOpInfo::make(VE::kUInt16 , VE::kUInt16); + case UniOpVVI::kSrlU32 : return VecOpInfo::make(VE::kUInt32 , VE::kUInt32); + case UniOpVVI::kSrlU64 : return VecOpInfo::make(VE::kUInt64 , VE::kUInt64); + case UniOpVVI::kSraI16 : return VecOpInfo::make(VE::kInt16 , VE::kInt16); + case UniOpVVI::kSraI32 : return VecOpInfo::make(VE::kInt32 , VE::kInt32); + case UniOpVVI::kSraI64 : return VecOpInfo::make(VE::kInt64 , VE::kInt64); + case UniOpVVI::kSllbU128 : return VecOpInfo::make(VE::kUInt8 , VE::kUInt8); + case UniOpVVI::kSrlbU128 : return VecOpInfo::make(VE::kUInt8 , VE::kUInt8); + case UniOpVVI::kSwizzleU16x4 : return VecOpInfo::make(VE::kUInt16 , VE::kUInt16); + case UniOpVVI::kSwizzleLoU16x4 : return VecOpInfo::make(VE::kUInt16 , VE::kUInt16); + case UniOpVVI::kSwizzleHiU16x4 : return VecOpInfo::make(VE::kUInt16 , VE::kUInt16); + case UniOpVVI::kSwizzleU32x4 : return VecOpInfo::make(VE::kUInt32 , VE::kUInt32); + case UniOpVVI::kSwizzleU64x2 : return VecOpInfo::make(VE::kUInt64 , VE::kUInt64); + case UniOpVVI::kSwizzleF32x4 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVVI::kSwizzleF64x2 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVVI::kSwizzleU64x4 : return VecOpInfo::make(VE::kUInt64 , VE::kUInt64); + case UniOpVVI::kSwizzleF64x4 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVVI::kExtractV128_I32: return VecOpInfo::make(VE::kInt32 , VE::kInt32); + case UniOpVVI::kExtractV128_I64: return VecOpInfo::make(VE::kInt64 , VE::kInt64); + case UniOpVVI::kExtractV128_F32: return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVVI::kExtractV128_F64: return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + case UniOpVVI::kExtractV256_I32: return VecOpInfo::make(VE::kUInt32 , VE::kUInt32); + case UniOpVVI::kExtractV256_I64: return VecOpInfo::make(VE::kUInt64 , VE::kUInt64); + case UniOpVVI::kExtractV256_F32: return VecOpInfo::make(VE::kFloat32, VE::kFloat32); + case UniOpVVI::kExtractV256_F64: return VecOpInfo::make(VE::kFloat64, VE::kFloat64); + +#if defined(ASMJIT_UJIT_AARCH64) + case UniOpVVI::kSrlRndU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt16); + case UniOpVVI::kSrlRndU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32); + case UniOpVVI::kSrlRndU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64); + case UniOpVVI::kSrlAccU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt16); + case UniOpVVI::kSrlAccU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32); + case UniOpVVI::kSrlAccU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64); + case UniOpVVI::kSrlRndAccU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt16); + case UniOpVVI::kSrlRndAccU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32); + case UniOpVVI::kSrlRndAccU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64); + case UniOpVVI::kSrlnLoU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt16); + case UniOpVVI::kSrlnHiU16 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32); + case UniOpVVI::kSrlnLoU32 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64); + case UniOpVVI::kSrlnHiU32 : return VecOpInfo::make(VE::kUInt16, VE::kUInt16); + case UniOpVVI::kSrlnLoU64 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32); + case UniOpVVI::kSrlnHiU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64); + case UniOpVVI::kSrlnRndLoU16 : return VecOpInfo::make(VE::kUInt8 , VE::kUInt16); + case UniOpVVI::kSrlnRndHiU16 : return VecOpInfo::make(VE::kUInt8 , VE::kUInt16); + case UniOpVVI::kSrlnRndLoU32 : return VecOpInfo::make(VE::kUInt16, VE::kUInt32); + case UniOpVVI::kSrlnRndHiU32 : return VecOpInfo::make(VE::kUInt16, VE::kUInt32); + case UniOpVVI::kSrlnRndLoU64 : return VecOpInfo::make(VE::kUInt32, VE::kUInt64); + case UniOpVVI::kSrlnRndHiU64 : return VecOpInfo::make(VE::kUInt32, VE::kUInt64); +#endif // ASMJIT_UJIT_AARCH64 + } + + ASMJIT_NOT_REACHED(); +} + +static const char* vec_op_name_vvv(UniOpVVV op) noexcept { + switch (op) { + case UniOpVVV::kAndU32 : return "v_and_u32"; + case UniOpVVV::kAndU64 : return "v_and_u64"; + case UniOpVVV::kOrU32 : return "v_or_u32"; + case UniOpVVV::kOrU64 : return "v_or_u64"; + case UniOpVVV::kXorU32 : return "v_xor_u32"; + case UniOpVVV::kXorU64 : return "v_xor_u64"; + case UniOpVVV::kAndnU32 : return "v_andn_u32"; + case UniOpVVV::kAndnU64 : return "v_andn_u64"; + case UniOpVVV::kBicU32 : return "v_bic_u32"; + case UniOpVVV::kBicU64 : return "v_bic_u64"; + case UniOpVVV::kAvgrU8 : return "v_avgr_u8"; + case UniOpVVV::kAvgrU16 : return "v_avgr_u16"; + case UniOpVVV::kAddU8 : return "v_add_u8"; + case UniOpVVV::kAddU16 : return "v_add_u16"; + case UniOpVVV::kAddU32 : return "v_add_u32"; + case UniOpVVV::kAddU64 : return "v_add_u64"; + case UniOpVVV::kSubU8 : return "v_sub_u8"; + case UniOpVVV::kSubU16 : return "v_sub_u16"; + case UniOpVVV::kSubU32 : return "v_sub_u32"; + case UniOpVVV::kSubU64 : return "v_sub_u64"; + case UniOpVVV::kAddsI8 : return "v_adds_i8"; + case UniOpVVV::kAddsU8 : return "v_adds_u8"; + case UniOpVVV::kAddsI16 : return "v_adds_i16"; + case UniOpVVV::kAddsU16 : return "v_adds_u16"; + case UniOpVVV::kSubsI8 : return "v_subs_i8"; + case UniOpVVV::kSubsU8 : return "v_subs_u8"; + case UniOpVVV::kSubsI16 : return "v_subs_i16"; + case UniOpVVV::kSubsU16 : return "v_subs_u16"; + case UniOpVVV::kMulU16 : return "v_mul_u16"; + case UniOpVVV::kMulU32 : return "v_mul_u32"; + case UniOpVVV::kMulU64 : return "v_mul_u64"; + case UniOpVVV::kMulhI16 : return "v_mulh_i16"; + case UniOpVVV::kMulhU16 : return "v_mulh_u16"; + case UniOpVVV::kMulU64_LoU32 : return "v_mul_u64_lo_u32"; + case UniOpVVV::kMHAddI16_I32 : return "v_mhadd_i16_i32"; + case UniOpVVV::kMinI8 : return "v_min_i8"; + case UniOpVVV::kMinU8 : return "v_min_u8"; + case UniOpVVV::kMinI16 : return "v_min_i16"; + case UniOpVVV::kMinU16 : return "v_min_u16"; + case UniOpVVV::kMinI32 : return "v_min_i32"; + case UniOpVVV::kMinU32 : return "v_min_u32"; + case UniOpVVV::kMinI64 : return "v_min_i64"; + case UniOpVVV::kMinU64 : return "v_min_u64"; + case UniOpVVV::kMaxI8 : return "v_max_i8"; + case UniOpVVV::kMaxU8 : return "v_max_u8"; + case UniOpVVV::kMaxI16 : return "v_max_i16"; + case UniOpVVV::kMaxU16 : return "v_max_u16"; + case UniOpVVV::kMaxI32 : return "v_max_i32"; + case UniOpVVV::kMaxU32 : return "v_max_u32"; + case UniOpVVV::kMaxI64 : return "v_max_i64"; + case UniOpVVV::kMaxU64 : return "v_max_u64"; + case UniOpVVV::kCmpEqU8 : return "v_cmp_eq_u8"; + case UniOpVVV::kCmpEqU16 : return "v_cmp_eq_u16"; + case UniOpVVV::kCmpEqU32 : return "v_cmp_eq_u32"; + case UniOpVVV::kCmpEqU64 : return "v_cmp_eq_u64"; + case UniOpVVV::kCmpGtI8 : return "v_cmp_gt_i8"; + case UniOpVVV::kCmpGtU8 : return "v_cmp_gt_u8"; + case UniOpVVV::kCmpGtI16 : return "v_cmp_gt_i16"; + case UniOpVVV::kCmpGtU16 : return "v_cmp_gt_u16"; + case UniOpVVV::kCmpGtI32 : return "v_cmp_gt_i32"; + case UniOpVVV::kCmpGtU32 : return "v_cmp_gt_u32"; + case UniOpVVV::kCmpGtI64 : return "v_cmp_gt_i64"; + case UniOpVVV::kCmpGtU64 : return "v_cmp_gt_u64"; + case UniOpVVV::kCmpGeI8 : return "v_cmp_ge_i8"; + case UniOpVVV::kCmpGeU8 : return "v_cmp_ge_u8"; + case UniOpVVV::kCmpGeI16 : return "v_cmp_ge_i16"; + case UniOpVVV::kCmpGeU16 : return "v_cmp_ge_u16"; + case UniOpVVV::kCmpGeI32 : return "v_cmp_ge_i32"; + case UniOpVVV::kCmpGeU32 : return "v_cmp_ge_u32"; + case UniOpVVV::kCmpGeI64 : return "v_cmp_ge_i64"; + case UniOpVVV::kCmpGeU64 : return "v_cmp_ge_u64"; + case UniOpVVV::kCmpLtI8 : return "v_cmp_lt_i8"; + case UniOpVVV::kCmpLtU8 : return "v_cmp_lt_u8"; + case UniOpVVV::kCmpLtI16 : return "v_cmp_lt_i16"; + case UniOpVVV::kCmpLtU16 : return "v_cmp_lt_u16"; + case UniOpVVV::kCmpLtI32 : return "v_cmp_lt_i32"; + case UniOpVVV::kCmpLtU32 : return "v_cmp_lt_u32"; + case UniOpVVV::kCmpLtI64 : return "v_cmp_lt_i64"; + case UniOpVVV::kCmpLtU64 : return "v_cmp_lt_u64"; + case UniOpVVV::kCmpLeI8 : return "v_cmp_le_i8"; + case UniOpVVV::kCmpLeU8 : return "v_cmp_le_u8"; + case UniOpVVV::kCmpLeI16 : return "v_cmp_le_i16"; + case UniOpVVV::kCmpLeU16 : return "v_cmp_le_u16"; + case UniOpVVV::kCmpLeI32 : return "v_cmp_le_i32"; + case UniOpVVV::kCmpLeU32 : return "v_cmp_le_u32"; + case UniOpVVV::kCmpLeI64 : return "v_cmp_le_i64"; + case UniOpVVV::kCmpLeU64 : return "v_cmp_le_u64"; + case UniOpVVV::kAndF32 : return "v_and_f32"; + case UniOpVVV::kAndF64 : return "v_and_f64"; + case UniOpVVV::kOrF32 : return "v_or_f32"; + case UniOpVVV::kOrF64 : return "v_or_f64"; + case UniOpVVV::kXorF32 : return "v_xor_f32"; + case UniOpVVV::kXorF64 : return "v_xor_f64"; + case UniOpVVV::kAndnF32 : return "v_andn_f32"; + case UniOpVVV::kAndnF64 : return "v_andn_f64"; + case UniOpVVV::kBicF32 : return "v_bic_f32"; + case UniOpVVV::kBicF64 : return "v_bic_f64"; + case UniOpVVV::kAddF32S : return "v_add_f32s"; + case UniOpVVV::kAddF64S : return "v_add_f64s"; + case UniOpVVV::kAddF32 : return "v_add_f32"; + case UniOpVVV::kAddF64 : return "v_add_f64"; + case UniOpVVV::kSubF32S : return "v_sub_f32s"; + case UniOpVVV::kSubF64S : return "v_sub_f64s"; + case UniOpVVV::kSubF32 : return "v_sub_f32"; + case UniOpVVV::kSubF64 : return "v_sub_f64"; + case UniOpVVV::kMulF32S : return "v_mul_f32s"; + case UniOpVVV::kMulF64S : return "v_mul_f64s"; + case UniOpVVV::kMulF32 : return "v_mul_f32"; + case UniOpVVV::kMulF64 : return "v_mul_f64"; + case UniOpVVV::kDivF32S : return "v_div_f32s"; + case UniOpVVV::kDivF64S : return "v_div_f64s"; + case UniOpVVV::kDivF32 : return "v_div_f32"; + case UniOpVVV::kDivF64 : return "v_div_f64"; + case UniOpVVV::kModF32S : return "v_mod_f32s"; + case UniOpVVV::kModF64S : return "v_mod_f64s"; + case UniOpVVV::kModF32 : return "v_mod_f32"; + case UniOpVVV::kModF64 : return "v_mod_f64"; + case UniOpVVV::kMinF32S : return "v_min_f32s"; + case UniOpVVV::kMinF64S : return "v_min_f64s"; + case UniOpVVV::kMinF32 : return "v_min_f32"; + case UniOpVVV::kMinF64 : return "v_min_f64"; + case UniOpVVV::kMaxF32S : return "v_max_f32s"; + case UniOpVVV::kMaxF64S : return "v_max_f64s"; + case UniOpVVV::kMaxF32 : return "v_max_f32"; + case UniOpVVV::kMaxF64 : return "v_max_f64"; + case UniOpVVV::kCmpEqF32S : return "v_cmp_eq_f32s"; + case UniOpVVV::kCmpEqF64S : return "v_cmp_eq_f64s"; + case UniOpVVV::kCmpEqF32 : return "v_cmp_eq_f32"; + case UniOpVVV::kCmpEqF64 : return "v_cmp_eq_f64"; + case UniOpVVV::kCmpNeF32S : return "v_cmp_ne_f32s"; + case UniOpVVV::kCmpNeF64S : return "v_cmp_ne_f64s"; + case UniOpVVV::kCmpNeF32 : return "v_cmp_ne_f32"; + case UniOpVVV::kCmpNeF64 : return "v_cmp_ne_f64"; + case UniOpVVV::kCmpGtF32S : return "v_cmp_gt_f32s"; + case UniOpVVV::kCmpGtF64S : return "v_cmp_gt_f64s"; + case UniOpVVV::kCmpGtF32 : return "v_cmp_gt_f32"; + case UniOpVVV::kCmpGtF64 : return "v_cmp_gt_f64"; + case UniOpVVV::kCmpGeF32S : return "v_cmp_ge_f32s"; + case UniOpVVV::kCmpGeF64S : return "v_cmp_ge_f64s"; + case UniOpVVV::kCmpGeF32 : return "v_cmp_ge_f32"; + case UniOpVVV::kCmpGeF64 : return "v_cmp_ge_f64"; + case UniOpVVV::kCmpLtF32S : return "v_cmp_lt_f32s"; + case UniOpVVV::kCmpLtF64S : return "v_cmp_lt_f64s"; + case UniOpVVV::kCmpLtF32 : return "v_cmp_lt_f32"; + case UniOpVVV::kCmpLtF64 : return "v_cmp_lt_f64"; + case UniOpVVV::kCmpLeF32S : return "v_cmp_le_f32s"; + case UniOpVVV::kCmpLeF64S : return "v_cmp_le_f64s"; + case UniOpVVV::kCmpLeF32 : return "v_cmp_le_f32"; + case UniOpVVV::kCmpLeF64 : return "v_cmp_le_f64"; + case UniOpVVV::kCmpOrdF32S : return "v_cmp_ord_f32s"; + case UniOpVVV::kCmpOrdF64S : return "v_cmp_ord_f64s"; + case UniOpVVV::kCmpOrdF32 : return "v_cmp_ord_f32"; + case UniOpVVV::kCmpOrdF64 : return "v_cmp_ord_f64"; + case UniOpVVV::kCmpUnordF32S : return "v_cmp_unord_f32s"; + case UniOpVVV::kCmpUnordF64S : return "v_cmp_unord_f64s"; + case UniOpVVV::kCmpUnordF32 : return "v_cmp_unord_f32"; + case UniOpVVV::kCmpUnordF64 : return "v_cmp_unord_f64"; + case UniOpVVV::kHAddF64 : return "v_hadd_f64"; + case UniOpVVV::kCombineLoHiU64 : return "v_combine_lo_hi_u64"; + case UniOpVVV::kCombineLoHiF64 : return "v_combine_lo_hi_f64"; + case UniOpVVV::kCombineHiLoU64 : return "v_combine_hi_lo_u64"; + case UniOpVVV::kCombineHiLoF64 : return "v_combine_hi_lo_f64"; + case UniOpVVV::kInterleaveLoU8 : return "v_interleave_lo_u8"; + case UniOpVVV::kInterleaveHiU8 : return "v_interleave_hi_u8"; + case UniOpVVV::kInterleaveLoU16: return "v_interleave_lo_u16"; + case UniOpVVV::kInterleaveHiU16: return "v_interleave_hi_u16"; + case UniOpVVV::kInterleaveLoU32: return "v_interleave_lo_u32"; + case UniOpVVV::kInterleaveHiU32: return "v_interleave_hi_u32"; + case UniOpVVV::kInterleaveLoU64: return "v_interleave_lo_u64"; + case UniOpVVV::kInterleaveHiU64: return "v_interleave_hi_u64"; + case UniOpVVV::kInterleaveLoF32: return "v_interleave_lo_f32"; + case UniOpVVV::kInterleaveHiF32: return "v_interleave_hi_f32"; + case UniOpVVV::kInterleaveLoF64: return "v_interleave_lo_f64"; + case UniOpVVV::kInterleaveHiF64: return "v_interleave_hi_f64"; + case UniOpVVV::kPacksI16_I8 : return "v_packs_i16_i8"; + case UniOpVVV::kPacksI16_U8 : return "v_packs_i16_u8"; + case UniOpVVV::kPacksI32_I16 : return "v_packs_i32_i16"; + case UniOpVVV::kPacksI32_U16 : return "v_packs_i32_u16"; + case UniOpVVV::kSwizzlev_U8 : return "v_swizzlev_u8"; + +#if defined(ASMJIT_UJIT_AARCH64) + case UniOpVVV::kMulwLoI8 : return "v_mulw_lo_i8"; + case UniOpVVV::kMulwLoU8 : return "v_mulw_lo_u8"; + case UniOpVVV::kMulwHiI8 : return "v_mulw_hi_i8"; + case UniOpVVV::kMulwHiU8 : return "v_mulw_hi_u8"; + case UniOpVVV::kMulwLoI16 : return "v_mulw_lo_i16"; + case UniOpVVV::kMulwLoU16 : return "v_mulw_lo_u16"; + case UniOpVVV::kMulwHiI16 : return "v_mulw_hi_i16"; + case UniOpVVV::kMulwHiU16 : return "v_mulw_hi_u16"; + case UniOpVVV::kMulwLoI32 : return "v_mulw_lo_i32"; + case UniOpVVV::kMulwLoU32 : return "v_mulw_lo_u32"; + case UniOpVVV::kMulwHiI32 : return "v_mulw_hi_i32"; + case UniOpVVV::kMulwHiU32 : return "v_mulw_hi_u32"; + case UniOpVVV::kMAddwLoI8 : return "v_maddw_lo_i8"; + case UniOpVVV::kMAddwLoU8 : return "v_maddw_lo_u8"; + case UniOpVVV::kMAddwHiI8 : return "v_maddw_hi_i8"; + case UniOpVVV::kMAddwHiU8 : return "v_maddw_hi_u8"; + case UniOpVVV::kMAddwLoI16 : return "v_maddw_lo_i16"; + case UniOpVVV::kMAddwLoU16 : return "v_maddw_lo_u16"; + case UniOpVVV::kMAddwHiI16 : return "v_maddw_hi_i16"; + case UniOpVVV::kMAddwHiU16 : return "v_maddw_hi_u16"; + case UniOpVVV::kMAddwLoI32 : return "v_maddw_lo_i32"; + case UniOpVVV::kMAddwLoU32 : return "v_maddw_lo_u32"; + case UniOpVVV::kMAddwHiI32 : return "v_maddw_hi_i32"; + case UniOpVVV::kMAddwHiU32 : return "v_maddw_hi_u32"; +#endif // ASMJIT_UJIT_AARCH64 + +#if defined(ASMJIT_UJIT_X86) + case UniOpVVV::kPermuteU8 : return "v_permute_u8"; + case UniOpVVV::kPermuteU16 : return "v_permute_u16"; + case UniOpVVV::kPermuteU32 : return "v_permute_u32"; + case UniOpVVV::kPermuteU64 : return "v_permute_u64"; +#endif // ASMJIT_UJIT_X86 + } + + ASMJIT_NOT_REACHED(); +} + +static VecOpInfo vec_op_info_vvv(UniOpVVV op) noexcept { + using VE = VecElementType; + + switch (op) { + case UniOpVVV::kAndU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kAndU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kOrU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kOrU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kXorU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kXorU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kAndnU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kAndnU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kBicU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kBicU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kAvgrU8 : return VecOpInfo::make(VE::kUInt8, VE::kUInt8, VE::kUInt8); + case UniOpVVV::kAvgrU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt16, VE::kUInt16); + case UniOpVVV::kAddU8 : return VecOpInfo::make(VE::kUInt8, VE::kUInt8, VE::kUInt8); + case UniOpVVV::kAddU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt16, VE::kUInt16); + case UniOpVVV::kAddU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kAddU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kSubU8 : return VecOpInfo::make(VE::kUInt8, VE::kUInt8, VE::kUInt8); + case UniOpVVV::kSubU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt16, VE::kUInt16); + case UniOpVVV::kSubU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kSubU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kAddsI8 : return VecOpInfo::make(VE::kInt8, VE::kInt8, VE::kInt8); + case UniOpVVV::kAddsU8 : return VecOpInfo::make(VE::kUInt8, VE::kUInt8, VE::kUInt8); + case UniOpVVV::kAddsI16 : return VecOpInfo::make(VE::kInt16, VE::kInt16, VE::kInt16); + case UniOpVVV::kAddsU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt16, VE::kUInt16); + case UniOpVVV::kSubsI8 : return VecOpInfo::make(VE::kInt8, VE::kInt8, VE::kInt8); + case UniOpVVV::kSubsU8 : return VecOpInfo::make(VE::kUInt8, VE::kUInt8, VE::kUInt8); + case UniOpVVV::kSubsI16 : return VecOpInfo::make(VE::kInt16, VE::kInt16, VE::kInt16); + case UniOpVVV::kSubsU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt16, VE::kUInt16); + case UniOpVVV::kMulU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt16, VE::kUInt16); + case UniOpVVV::kMulU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kMulU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kMulhI16 : return VecOpInfo::make(VE::kInt16, VE::kInt16, VE::kInt16); + case UniOpVVV::kMulhU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt16, VE::kUInt16); + case UniOpVVV::kMulU64_LoU32 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt32); + case UniOpVVV::kMHAddI16_I32 : return VecOpInfo::make(VE::kInt32, VE::kInt16, VE::kInt16); + case UniOpVVV::kMinI8 : return VecOpInfo::make(VE::kInt8, VE::kInt8, VE::kInt8); + case UniOpVVV::kMinU8 : return VecOpInfo::make(VE::kUInt8, VE::kUInt8, VE::kUInt8); + case UniOpVVV::kMinI16 : return VecOpInfo::make(VE::kInt16, VE::kInt16, VE::kInt16); + case UniOpVVV::kMinU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt16, VE::kUInt16); + case UniOpVVV::kMinI32 : return VecOpInfo::make(VE::kInt32, VE::kInt32, VE::kInt32); + case UniOpVVV::kMinU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kMinI64 : return VecOpInfo::make(VE::kInt64, VE::kInt64, VE::kInt64); + case UniOpVVV::kMinU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kMaxI8 : return VecOpInfo::make(VE::kInt8, VE::kInt8, VE::kInt8); + case UniOpVVV::kMaxU8 : return VecOpInfo::make(VE::kUInt8, VE::kUInt8, VE::kUInt8); + case UniOpVVV::kMaxI16 : return VecOpInfo::make(VE::kInt16, VE::kInt16, VE::kInt16); + case UniOpVVV::kMaxU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt16, VE::kUInt16); + case UniOpVVV::kMaxI32 : return VecOpInfo::make(VE::kInt32, VE::kInt32, VE::kInt32); + case UniOpVVV::kMaxU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kMaxI64 : return VecOpInfo::make(VE::kInt64, VE::kInt64, VE::kInt64); + case UniOpVVV::kMaxU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kCmpEqU8 : return VecOpInfo::make(VE::kUInt8, VE::kUInt8, VE::kUInt8); + case UniOpVVV::kCmpEqU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt16, VE::kUInt16); + case UniOpVVV::kCmpEqU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kCmpEqU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kCmpGtI8 : return VecOpInfo::make(VE::kInt8, VE::kInt8, VE::kInt8); + case UniOpVVV::kCmpGtU8 : return VecOpInfo::make(VE::kUInt8, VE::kUInt8, VE::kUInt8); + case UniOpVVV::kCmpGtI16 : return VecOpInfo::make(VE::kInt16, VE::kInt16, VE::kInt16); + case UniOpVVV::kCmpGtU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt16, VE::kUInt16); + case UniOpVVV::kCmpGtI32 : return VecOpInfo::make(VE::kInt32, VE::kInt32, VE::kInt32); + case UniOpVVV::kCmpGtU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kCmpGtI64 : return VecOpInfo::make(VE::kInt64, VE::kInt64, VE::kInt64); + case UniOpVVV::kCmpGtU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kCmpGeI8 : return VecOpInfo::make(VE::kInt8, VE::kInt8, VE::kInt8); + case UniOpVVV::kCmpGeU8 : return VecOpInfo::make(VE::kUInt8, VE::kUInt8, VE::kUInt8); + case UniOpVVV::kCmpGeI16 : return VecOpInfo::make(VE::kInt16, VE::kInt16, VE::kInt16); + case UniOpVVV::kCmpGeU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt16, VE::kUInt16); + case UniOpVVV::kCmpGeI32 : return VecOpInfo::make(VE::kInt32, VE::kInt32, VE::kInt32); + case UniOpVVV::kCmpGeU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kCmpGeI64 : return VecOpInfo::make(VE::kInt64, VE::kInt64, VE::kInt64); + case UniOpVVV::kCmpGeU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kCmpLtI8 : return VecOpInfo::make(VE::kInt8, VE::kInt8, VE::kInt8); + case UniOpVVV::kCmpLtU8 : return VecOpInfo::make(VE::kUInt8, VE::kUInt8, VE::kUInt8); + case UniOpVVV::kCmpLtI16 : return VecOpInfo::make(VE::kInt16, VE::kInt16, VE::kInt16); + case UniOpVVV::kCmpLtU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt16, VE::kUInt16); + case UniOpVVV::kCmpLtI32 : return VecOpInfo::make(VE::kInt32, VE::kInt32, VE::kInt32); + case UniOpVVV::kCmpLtU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kCmpLtI64 : return VecOpInfo::make(VE::kInt64, VE::kInt64, VE::kInt64); + case UniOpVVV::kCmpLtU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kCmpLeI8 : return VecOpInfo::make(VE::kInt8, VE::kInt8, VE::kInt8); + case UniOpVVV::kCmpLeU8 : return VecOpInfo::make(VE::kUInt8, VE::kUInt8, VE::kUInt8); + case UniOpVVV::kCmpLeI16 : return VecOpInfo::make(VE::kInt16, VE::kInt16, VE::kInt16); + case UniOpVVV::kCmpLeU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt16, VE::kUInt16); + case UniOpVVV::kCmpLeI32 : return VecOpInfo::make(VE::kInt32, VE::kInt32, VE::kInt32); + case UniOpVVV::kCmpLeU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kCmpLeI64 : return VecOpInfo::make(VE::kInt64, VE::kInt64, VE::kInt64); + case UniOpVVV::kCmpLeU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kAndF32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kAndF64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kOrF32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kOrF64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kXorF32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kXorF64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kAndnF32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kAndnF64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kBicF32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kBicF64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kAddF32S : return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kAddF64S : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kAddF32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kAddF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kSubF32S : return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kSubF64S : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kSubF32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kSubF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kMulF32S : return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kMulF64S : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kMulF32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kMulF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kDivF32S : return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kDivF64S : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kDivF32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kDivF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kModF32S : return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kModF64S : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kModF32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kModF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kMinF32S : return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kMinF64S : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kMinF32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kMinF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kMaxF32S : return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kMaxF64S : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kMaxF32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kMaxF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kCmpEqF32S : return VecOpInfo::make(VE::kUInt32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kCmpEqF64S : return VecOpInfo::make(VE::kUInt64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kCmpEqF32 : return VecOpInfo::make(VE::kUInt32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kCmpEqF64 : return VecOpInfo::make(VE::kUInt64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kCmpNeF32S : return VecOpInfo::make(VE::kUInt32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kCmpNeF64S : return VecOpInfo::make(VE::kUInt64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kCmpNeF32 : return VecOpInfo::make(VE::kUInt32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kCmpNeF64 : return VecOpInfo::make(VE::kUInt64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kCmpGtF32S : return VecOpInfo::make(VE::kUInt32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kCmpGtF64S : return VecOpInfo::make(VE::kUInt64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kCmpGtF32 : return VecOpInfo::make(VE::kUInt32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kCmpGtF64 : return VecOpInfo::make(VE::kUInt64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kCmpGeF32S : return VecOpInfo::make(VE::kUInt32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kCmpGeF64S : return VecOpInfo::make(VE::kUInt64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kCmpGeF32 : return VecOpInfo::make(VE::kUInt32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kCmpGeF64 : return VecOpInfo::make(VE::kUInt64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kCmpLtF32S : return VecOpInfo::make(VE::kUInt32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kCmpLtF64S : return VecOpInfo::make(VE::kUInt64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kCmpLtF32 : return VecOpInfo::make(VE::kUInt32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kCmpLtF64 : return VecOpInfo::make(VE::kUInt64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kCmpLeF32S : return VecOpInfo::make(VE::kUInt32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kCmpLeF64S : return VecOpInfo::make(VE::kUInt64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kCmpLeF32 : return VecOpInfo::make(VE::kUInt32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kCmpLeF64 : return VecOpInfo::make(VE::kUInt64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kCmpOrdF32S : return VecOpInfo::make(VE::kUInt32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kCmpOrdF64S : return VecOpInfo::make(VE::kUInt64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kCmpOrdF32 : return VecOpInfo::make(VE::kUInt32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kCmpOrdF64 : return VecOpInfo::make(VE::kUInt64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kCmpUnordF32S : return VecOpInfo::make(VE::kUInt32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kCmpUnordF64S : return VecOpInfo::make(VE::kUInt64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kCmpUnordF32 : return VecOpInfo::make(VE::kUInt32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kCmpUnordF64 : return VecOpInfo::make(VE::kUInt64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kHAddF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kCombineLoHiU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kCombineLoHiF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kCombineHiLoU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kCombineHiLoF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kInterleaveLoU8 : return VecOpInfo::make(VE::kUInt8, VE::kUInt8, VE::kUInt8); + case UniOpVVV::kInterleaveHiU8 : return VecOpInfo::make(VE::kUInt8, VE::kUInt8, VE::kUInt8); + case UniOpVVV::kInterleaveLoU16: return VecOpInfo::make(VE::kUInt16, VE::kUInt16, VE::kUInt16); + case UniOpVVV::kInterleaveHiU16: return VecOpInfo::make(VE::kUInt16, VE::kUInt16, VE::kUInt16); + case UniOpVVV::kInterleaveLoU32: return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kInterleaveHiU32: return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kInterleaveLoU64: return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kInterleaveHiU64: return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVV::kInterleaveLoF32: return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kInterleaveHiF32: return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVV::kInterleaveLoF64: return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kInterleaveHiF64: return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVV::kPacksI16_I8 : return VecOpInfo::make(VE::kInt8, VE::kInt16, VE::kInt16); + case UniOpVVV::kPacksI16_U8 : return VecOpInfo::make(VE::kUInt8, VE::kInt16, VE::kInt16); + case UniOpVVV::kPacksI32_I16 : return VecOpInfo::make(VE::kInt16, VE::kInt32, VE::kInt32); + case UniOpVVV::kPacksI32_U16 : return VecOpInfo::make(VE::kUInt16, VE::kInt32, VE::kInt32); + case UniOpVVV::kSwizzlev_U8 : return VecOpInfo::make(VE::kUInt8, VE::kUInt8, VE::kUInt8); + +#if defined(ASMJIT_UJIT_AARCH64) + case UniOpVVV::kMulwLoI8 : return VecOpInfo::make(VE::kInt16, VE::kInt8, VE::kInt8); + case UniOpVVV::kMulwLoU8 : return VecOpInfo::make(VE::kUInt16, VE::kUInt8, VE::kUInt8); + case UniOpVVV::kMulwHiI8 : return VecOpInfo::make(VE::kInt16, VE::kInt8, VE::kInt8); + case UniOpVVV::kMulwHiU8 : return VecOpInfo::make(VE::kUInt16, VE::kUInt8, VE::kUInt8); + case UniOpVVV::kMulwLoI16 : return VecOpInfo::make(VE::kInt32, VE::kInt16, VE::kInt16); + case UniOpVVV::kMulwLoU16 : return VecOpInfo::make(VE::kUInt32, VE::kUInt16, VE::kUInt16); + case UniOpVVV::kMulwHiI16 : return VecOpInfo::make(VE::kInt32, VE::kInt16, VE::kInt16); + case UniOpVVV::kMulwHiU16 : return VecOpInfo::make(VE::kUInt32, VE::kUInt16, VE::kUInt16); + case UniOpVVV::kMulwLoI32 : return VecOpInfo::make(VE::kInt64, VE::kInt32, VE::kInt32); + case UniOpVVV::kMulwLoU32 : return VecOpInfo::make(VE::kUInt64, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kMulwHiI32 : return VecOpInfo::make(VE::kInt64, VE::kInt32, VE::kInt32); + case UniOpVVV::kMulwHiU32 : return VecOpInfo::make(VE::kUInt64, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kMAddwLoI8 : return VecOpInfo::make(VE::kInt16, VE::kInt8, VE::kInt8); + case UniOpVVV::kMAddwLoU8 : return VecOpInfo::make(VE::kUInt16, VE::kUInt8, VE::kUInt8); + case UniOpVVV::kMAddwHiI8 : return VecOpInfo::make(VE::kInt16, VE::kInt8, VE::kInt8); + case UniOpVVV::kMAddwHiU8 : return VecOpInfo::make(VE::kUInt16, VE::kUInt8, VE::kUInt8); + case UniOpVVV::kMAddwLoI16 : return VecOpInfo::make(VE::kInt32, VE::kInt16, VE::kInt16); + case UniOpVVV::kMAddwLoU16 : return VecOpInfo::make(VE::kUInt32, VE::kUInt16, VE::kUInt16); + case UniOpVVV::kMAddwHiI16 : return VecOpInfo::make(VE::kInt32, VE::kInt16, VE::kInt16); + case UniOpVVV::kMAddwHiU16 : return VecOpInfo::make(VE::kUInt32, VE::kUInt16, VE::kUInt16); + case UniOpVVV::kMAddwLoI32 : return VecOpInfo::make(VE::kInt64, VE::kInt32, VE::kInt32); + case UniOpVVV::kMAddwLoU32 : return VecOpInfo::make(VE::kUInt64, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kMAddwHiI32 : return VecOpInfo::make(VE::kInt64, VE::kInt32, VE::kInt32); + case UniOpVVV::kMAddwHiU32 : return VecOpInfo::make(VE::kUInt64, VE::kUInt32, VE::kUInt32); +#endif // ASMJIT_UJIT_AARCH64 + +#if defined(ASMJIT_UJIT_X86) + case UniOpVVV::kPermuteU8 : return VecOpInfo::make(VE::kUInt8, VE::kUInt8, VE::kUInt8); + case UniOpVVV::kPermuteU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt16, VE::kUInt16); + case UniOpVVV::kPermuteU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVV::kPermuteU64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); +#endif // ASMJIT_UJIT_X86 + } + + ASMJIT_NOT_REACHED(); +} + +static const char* vec_op_name_vvvi(UniOpVVVI op) noexcept { + switch (op) { + case UniOpVVVI::kAlignr_U128 : return "v_alignr_u128"; + case UniOpVVVI::kInterleaveShuffleU32x4: return "v_interleave_shuffle_u32x4"; + case UniOpVVVI::kInterleaveShuffleU64x2: return "v_interleave_shuffle_u64x2"; + case UniOpVVVI::kInterleaveShuffleF32x4: return "v_interleave_shuffle_f32x4"; + case UniOpVVVI::kInterleaveShuffleF64x2: return "v_interleave_shuffle_f64x2"; + case UniOpVVVI::kInsertV128_U32 : return "v_insert_v128_u32"; + case UniOpVVVI::kInsertV128_F32 : return "v_insert_v128_f32"; + case UniOpVVVI::kInsertV128_U64 : return "v_insert_v128_u64"; + case UniOpVVVI::kInsertV128_F64 : return "v_insert_v128_f64"; + case UniOpVVVI::kInsertV256_U32 : return "v_insert_v256_u32"; + case UniOpVVVI::kInsertV256_F32 : return "v_insert_v256_f32"; + case UniOpVVVI::kInsertV256_U64 : return "v_insert_v256_u64"; + case UniOpVVVI::kInsertV256_F64 : return "v_insert_v256_f64"; + } + + ASMJIT_NOT_REACHED(); +} + +static VecOpInfo vec_op_info_vvvi(UniOpVVVI op) noexcept { + using VE = VecElementType; + + switch (op) { + case UniOpVVVI::kAlignr_U128 : return VecOpInfo::make(VE::kUInt8, VE::kUInt8, VE::kUInt8); + case UniOpVVVI::kInterleaveShuffleU32x4: return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVVI::kInterleaveShuffleU64x2: return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVVI::kInterleaveShuffleF32x4: return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVVI::kInterleaveShuffleF64x2: return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVVI::kInsertV128_U32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVVI::kInsertV128_F32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVVI::kInsertV128_U64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVVI::kInsertV128_F64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVVI::kInsertV256_U32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVVI::kInsertV256_F32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVVI::kInsertV256_U64 : return VecOpInfo::make(VE::kUInt64, VE::kUInt64, VE::kUInt64); + case UniOpVVVI::kInsertV256_F64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64); + + default: + ASMJIT_NOT_REACHED(); + } +} + +static const char* vec_op_name_vvvv(UniOpVVVV op) noexcept { + switch (op) { + case UniOpVVVV::kBlendV_U8: return "v_blendv_u8"; + case UniOpVVVV::kMAddU16 : return "v_madd_u16"; + case UniOpVVVV::kMAddU32 : return "v_madd_u32"; + case UniOpVVVV::kMAddF32S : return "v_madd_f32s"; + case UniOpVVVV::kMAddF64S : return "v_madd_f64s"; + case UniOpVVVV::kMAddF32 : return "v_madd_f32"; + case UniOpVVVV::kMAddF64 : return "v_madd_f64"; + case UniOpVVVV::kMSubF32S : return "v_msub_f32s"; + case UniOpVVVV::kMSubF64S : return "v_msub_f64s"; + case UniOpVVVV::kMSubF32 : return "v_msub_f32"; + case UniOpVVVV::kMSubF64 : return "v_msub_f64"; + case UniOpVVVV::kNMAddF32S: return "v_nmadd_f32s"; + case UniOpVVVV::kNMAddF64S: return "v_nmadd_f64s"; + case UniOpVVVV::kNMAddF32 : return "v_nmadd_f32"; + case UniOpVVVV::kNMAddF64 : return "v_nmadd_f64"; + case UniOpVVVV::kNMSubF32S: return "v_nmsub_f32s"; + case UniOpVVVV::kNMSubF64S: return "v_nmsub_f64s"; + case UniOpVVVV::kNMSubF32 : return "v_nmsub_f32"; + case UniOpVVVV::kNMSubF64 : return "v_nmsub_f64"; + } + + ASMJIT_NOT_REACHED(); +} + +static VecOpInfo vec_op_info_vvvv(UniOpVVVV op) noexcept { + using VE = VecElementType; + + switch (op) { + case UniOpVVVV::kBlendV_U8: return VecOpInfo::make(VE::kUInt8, VE::kUInt8, VE::kUInt8, VE::kUInt8); + case UniOpVVVV::kMAddU16 : return VecOpInfo::make(VE::kUInt16, VE::kUInt16, VE::kUInt16, VE::kUInt16); + case UniOpVVVV::kMAddU32 : return VecOpInfo::make(VE::kUInt32, VE::kUInt32, VE::kUInt32, VE::kUInt32); + case UniOpVVVV::kMAddF32S : return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVVV::kMAddF64S : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVVV::kMAddF32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVVV::kMAddF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVVV::kMSubF32S : return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVVV::kMSubF64S : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVVV::kMSubF32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVVV::kMSubF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVVV::kNMAddF32S: return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVVV::kNMAddF64S: return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVVV::kNMAddF32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVVV::kNMAddF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVVV::kNMSubF32S: return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVVV::kNMSubF64S: return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64, VE::kFloat64); + case UniOpVVVV::kNMSubF32 : return VecOpInfo::make(VE::kFloat32, VE::kFloat32, VE::kFloat32, VE::kFloat32); + case UniOpVVVV::kNMSubF64 : return VecOpInfo::make(VE::kFloat64, VE::kFloat64, VE::kFloat64, VE::kFloat64); + } + + ASMJIT_NOT_REACHED(); +} + +// ujit::UniCompiler - Tests - SIMD - Float To Int - Machine Behavior +// ================================================================== + +template<FloatToIntOutsideRangeBehavior behavior, typename IntT, typename FloatT> +static ASMJIT_INLINE_NODEBUG int32_t cvt_float_to_int_trunc(const FloatT& x) noexcept { + constexpr IntT min_value = std::numeric_limits<IntT>::lowest(); + constexpr IntT max_value = std::numeric_limits<IntT>::max(); + constexpr IntT zero = IntT(0); + + if (std::isnan(x)) { + return behavior == FloatToIntOutsideRangeBehavior::kSmallestValue ? min_value : zero; + } + + if (x < FloatT(min_value)) { + return min_value; + } + + if (x > FloatT(max_value)) { + return behavior == FloatToIntOutsideRangeBehavior::kSmallestValue ? min_value : max_value; + } + + return IntT(x); +} + +template<FloatToIntOutsideRangeBehavior behavior, typename IntT, typename FloatT> +static ASMJIT_INLINE_NODEBUG int32_t cvt_float_to_int_round(const FloatT& x) noexcept { + constexpr IntT min_value = std::numeric_limits<IntT>::lowest(); + constexpr IntT max_value = std::numeric_limits<IntT>::max(); + constexpr IntT zero = IntT(0); + + if (std::isnan(x)) { + return behavior == FloatToIntOutsideRangeBehavior::kSmallestValue ? min_value : zero; + } + + if (x < FloatT(min_value)) { + return min_value; + } + + if (x > FloatT(max_value)) { + return behavior == FloatToIntOutsideRangeBehavior::kSmallestValue ? min_value : max_value; + } + + return IntT(std::nearbyint(x)); +} + +// ujit::UniCompiler - Tests - SIMD - Data Generators & Constraints +// ================================================================ + +// Data generator, which is used to fill the content of SIMD registers. +class DataGenInt { +public: + TestUtils::Random rng; + uint32_t step; + + struct Float32Data { + uint32_t u32; + float f32; + }; + + ASMJIT_INLINE explicit DataGenInt(uint64_t seed) noexcept + : rng(seed), + step(0) {} + + ASMJIT_NOINLINE uint64_t next_uint64() noexcept { + if (++step >= 256) { + step = 0; + } + + // NOTE: Nothing really elaborate - sometimes we want to test also numbers + // that random number generators won't return often, so we hardcode some. + switch (step) { + case 0: return 0u; + case 1: return 0u; + case 2: return 0u; + case 6: return 1u; + case 7: return 0u; + case 10: return 0u; + case 11: return 0xFFu; + case 15: return 0xFFFFu; + case 17: return 0xFFFFFFFFu; + case 21: return 0xFFFFFFFFFFFFFFFFu; + case 24: return 1u; + case 40: return 0xFFu; + case 55: return 0x8080808080808080u; + case 66: return 0x80000080u; + case 69: return 1u; + case 79: return 0x7F; + case 122: return 0xFFFFu; + case 123: return 0xFFFFu; + case 124: return 0xFFFFu; + case 127: return 1u; + case 130: return 0xFFu; + case 142: return 0x7FFFu; + case 143: return 0x7FFFu; + case 144: return 0u; + case 145: return 0x7FFFu; + default : return rng.next_uint64(); + } + } + + ASMJIT_NOINLINE float next_float32() noexcept { + if (++step >= 256) { + step = 0; + } + + switch (step) { + case 0: return 0.0f; + case 1: return 0.0f; + case 2: return 0.0f; + case 6: return 1.0f; + case 7: return 0.0f; + case 10: return 0.00001f; + case 11: return 2.0f; + case 12: return -std::numeric_limits<float>::infinity(); + case 15: return 3.0f; + case 17: return 256.0f; + case 21: return 0.5f; + case 23: return std::numeric_limits<float>::quiet_NaN(); + case 24: return 0.25f; + case 27: return std::numeric_limits<float>::quiet_NaN(); + case 29: return std::numeric_limits<float>::infinity(); + case 31: return std::numeric_limits<float>::quiet_NaN(); + case 35: return std::numeric_limits<float>::quiet_NaN(); + case 40: return 5.12323f; + case 45: return -std::numeric_limits<float>::infinity(); + case 55: return 100.5f; + case 66: return 0.1f; + case 69: return 0.2f; + case 79: return 0.3f; + case 89: return -13005961.0f; + case 99: return -std::numeric_limits<float>::infinity(); + case 100: + case 102: + case 104: + case 106: + case 108: return float(rng.next_double()); + case 110: + case 112: + case 114: + case 116: + case 118: return -float(rng.next_double()); + case 122: return 10.3f; + case 123: return 20.3f; + case 124: return -100.3f; + case 127: return 1.3f; + case 130: return std::numeric_limits<float>::quiet_NaN(); + case 135: return -std::numeric_limits<float>::infinity(); + case 142: return 1.0f; + case 143: return 1.5f; + case 144: return 2.0f; + case 145: return std::numeric_limits<float>::infinity(); + case 155: return -1.5f; + case 165: return -0.5f; + case 175: return -1.0f; + case 245: return 2.5f; + + default: { + float sign = rng.next_uint32() < 0x7FFFFFF ? 1.0f : -1.0f; + return float(rng.next_double() * double(rng.next_uint32() & 0xFFFFFFu)) * sign; + } + } + } + + ASMJIT_NOINLINE double next_float64() noexcept { + if (++step >= 256) { + step = 0; + } + + switch (step) { + case 0: return 0.0; + case 1: return 0.0; + case 2: return 0.0; + case 6: return 1.0; + case 7: return 0.0; + case 10: return 0.00001; + case 11: return 2.0; + case 12: return -std::numeric_limits<double>::infinity(); + case 15: return 3.0; + case 17: return 256.0; + case 21: return 0.5; + case 23: return std::numeric_limits<double>::quiet_NaN(); + case 24: return 0.25; + case 27: return std::numeric_limits<double>::quiet_NaN(); + case 29: return std::numeric_limits<double>::infinity(); + case 31: return std::numeric_limits<double>::quiet_NaN(); + case 35: return std::numeric_limits<double>::quiet_NaN(); + case 40: return 5.12323; + case 45: return -std::numeric_limits<double>::infinity(); + case 55: return 100.5; + case 66: return 0.1; + case 69: return 0.2; + case 79: return 0.3; + case 80: return 4503599627370495.5; + case 99: return -std::numeric_limits<double>::infinity(); + case 100: + case 102: + case 104: + case 106: + case 108: return rng.next_double(); + case 110: + case 112: + case 114: + case 116: + case 118: return -rng.next_double(); + case 122: return 10.3; + case 123: return 20.3; + case 124: return -100.3; + case 125: return 4503599627370496.0; + case 127: return 1.3; + case 130: return std::numeric_limits<double>::quiet_NaN(); + case 135: return -std::numeric_limits<double>::infinity(); + case 142: return 1.0; + case 143: return 1.5; + case 144: return 2.0; + case 145: return std::numeric_limits<double>::infinity(); + case 155: return -1.5; + case 165: return -0.5; + case 175: return -1.0; + case 245: return 2.5; + case 248: return -4503599627370495.5; + + default: { + double sign = rng.next_uint32() < 0x7FFFFFF ? 1.0 : -1.0; + return double(rng.next_double() * double(rng.next_uint32() & 0x3FFFFFFFu)) * sign; + } + } + } +}; + +template<typename T> +struct half_minus_1ulp_const; + +template<> struct half_minus_1ulp_const<float> { static inline constexpr float value = 0.49999997f; }; +template<> struct half_minus_1ulp_const<double> { static inline constexpr double value = 0.49999999999999994; }; + +// Some SIMD operations are constrained, especially those higher level. So, to successfully test these we +// have to model the constraints in a way that the SIMD instruction we test actually gets the correct input. +// Note that a constraint doesn't have to be always range based, it could be anything. +struct ConstraintNone { + template<uint32_t kW> + static ASMJIT_INLINE_NODEBUG void apply(VecOverlay<kW>& v) noexcept { Support::maybe_unused(v); } +}; + +template<typename ElementT, typename Derived> +struct ConstraintBase { + template<uint32_t kW> + static ASMJIT_INLINE void apply(VecOverlay<kW>& v) noexcept { + ElementT* elements = v.template data<ElementT>(); + for (size_t i = 0; i < kW / sizeof(ElementT); i++) { + elements[i] = Derived::apply_one(elements[i]); + } + } +}; + +template<uint8_t kMin, uint8_t kMax> +struct ConstraintRangeU8 : public ConstraintBase<uint16_t, ConstraintRangeU8<kMin, kMax>> { + static ASMJIT_INLINE_NODEBUG uint8_t apply_one(uint8_t x) noexcept { return std::clamp(x, kMin, kMax); } +}; + +template<uint16_t kMin, uint16_t kMax> +struct ConstraintRangeU16 : public ConstraintBase<uint16_t, ConstraintRangeU16<kMin, kMax>> { + static ASMJIT_INLINE_NODEBUG uint16_t apply_one(uint16_t x) noexcept { return std::clamp(x, kMin, kMax); } +}; + +template<uint32_t kMin, uint32_t kMax> +struct ConstraintRangeU32 : public ConstraintBase<uint32_t, ConstraintRangeU32<kMin, kMax>> { + static ASMJIT_INLINE_NODEBUG uint32_t apply_one(uint32_t x) noexcept { return std::clamp(x, kMin, kMax); } +}; + +// ujit::UniCompiler - Tests - Generic Operations +// ============================================== + +template<typename T> +static ASMJIT_INLINE_NODEBUG std::make_unsigned_t<T> cast_uint(const T& x) noexcept { + return static_cast<std::make_unsigned_t<T>>(x); +} + +template<typename T> +static ASMJIT_INLINE_NODEBUG std::make_signed_t<T> cast_int(const T& x) noexcept { + return static_cast<std::make_signed_t<T>>(x); +} + +static ASMJIT_INLINE_NODEBUG int8_t saturate_i16_to_i8(int16_t x) noexcept { + return x < int16_t(-128) ? int8_t(-128) : + x > int16_t( 127) ? int8_t( 127) : int8_t(x & 0xFF); +} + +static ASMJIT_INLINE_NODEBUG uint8_t saturate_i16_to_u8(int16_t x) noexcept { + return x < int16_t(0x00) ? uint8_t(0x00) : + x > int16_t(0xFF) ? uint8_t(0xFF) : uint8_t(x & 0xFF); +} + +static ASMJIT_INLINE_NODEBUG int16_t saturate_i32_to_i16(int32_t x) noexcept { + return x < int32_t(-32768) ? int16_t(-32768) : + x > int32_t( 32767) ? int16_t( 32767) : int16_t(x & 0xFFFF); +} + +static ASMJIT_INLINE_NODEBUG uint16_t saturate_i32_to_u16(int32_t x) noexcept { + return x < int32_t(0x0000) ? uint16_t(0x0000) : + x > int32_t(0xFFFF) ? uint16_t(0xFFFF) : uint16_t(x & 0xFFFF); +} + +template<typename T, typename Derived> struct op_each_vv { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + for (uint32_t i = 0; i < kW / sizeof(T); i++) { + out.set(i, Derived::apply_one(a.template get<T>(i))); + } + return out; + } +}; + +template<typename T, typename Derived> struct op_each_vvi { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, uint32_t imm) noexcept { + VecOverlay<kW> out{}; + for (uint32_t i = 0; i < kW / sizeof(T); i++) { + out.set(i, Derived::apply_one(a.template get<T>(i), imm)); + } + return out; + } +}; + +template<typename T, typename Derived> struct op_each_vvv { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b) noexcept { + VecOverlay<kW> out{}; + for (uint32_t i = 0; i < kW / sizeof(T); i++) { + out.set(i, Derived::apply_one(a.template get<T>(i), b.template get<T>(i))); + } + return out; + } +}; + +template<typename T, typename Derived> struct op_each_vvvi { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b, uint32_t imm) noexcept { + VecOverlay<kW> out{}; + for (uint32_t i = 0; i < kW / sizeof(T); i++) { + out.set(i, Derived::apply_one(a.template get<T>(i), b.template get<T>(i), imm)); + } + return out; + } +}; + +template<typename T, typename Derived> struct op_each_vvvv { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b, const VecOverlay<kW>& c) noexcept { + VecOverlay<kW> out{}; + for (uint32_t i = 0; i < kW / sizeof(T); i++) { + out.set(i, Derived::apply_one(a.template get<T>(i), b.template get<T>(i), c.template get<T>(i))); + } + return out; + } +}; + +template<ScalarOpBehavior kB, typename T, typename Derived> struct op_scalar_vv { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out {}; + if constexpr (kB == ScalarOpBehavior::kPreservingVec128) { + out.copy_16b_from(a); + } + out.set(0, Derived::apply_one(a.template get<T>(0))); + return out; + } +}; + +template<ScalarOpBehavior kB, typename T, typename Derived> struct op_scalar_vvi { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, uint32_t imm) noexcept { + VecOverlay<kW> out {}; + if constexpr (kB == ScalarOpBehavior::kPreservingVec128) { + out.copy_16b_from(a); + } + out.set(0, Derived::apply_one(a.template get<T>(0), imm)); + return out; + } +}; + +template<ScalarOpBehavior kB, typename T, typename Derived> struct op_scalar_vvv { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b) noexcept { + VecOverlay<kW> out {}; + if constexpr (kB == ScalarOpBehavior::kPreservingVec128) { + out.copy_16b_from(a); + } + out.set(0, Derived::apply_one(a.template get<T>(0), b.template get<T>(0))); + return out; + } +}; + +template<ScalarOpBehavior kB, typename T, typename Derived> struct op_scalar_vvvv { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b, const VecOverlay<kW>& c) noexcept { + VecOverlay<kW> out {}; + if constexpr (kB == ScalarOpBehavior::kPreservingVec128) { + out.copy_16b_from(a); + } + out.set(0, Derived::apply_one(a.template get<T>(0), b.template get<T>(0), c.template get<T>(0))); + return out; + } +}; + +// ujit::UniCompiler - Tests - Generic Operations - VV +// =================================================== + +struct vec_op_mov : public op_each_vv<uint32_t, vec_op_mov> { + static ASMJIT_INLINE_NODEBUG uint32_t apply_one(const uint32_t& a) noexcept { return a; } +}; + +struct vec_op_mov_u64 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + out.data_u64[0] = a.data_u64[0]; + return out; + } +}; + +struct vec_op_broadcast_u8 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + for (uint32_t i = 0; i < kW; i++) + out.data_u8[i] = a.data_u8[0]; + return out; + } +}; + +struct vec_op_broadcast_u16 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + for (uint32_t i = 0; i < kW / 2u; i++) { + out.data_u16[i] = a.data_u16[0]; + } + return out; + } +}; + +struct vec_op_broadcast_u32 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + for (uint32_t i = 0; i < kW / 4u; i++) { + out.data_u32[i] = a.data_u32[0]; + } + return out; + } +}; + +struct vec_op_broadcast_u64 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + for (uint32_t i = 0; i < kW / 8u; i++) { + out.data_u64[i] = a.data_u64[0]; + } + return out; + } +}; + +struct vec_op_broadcast_u128 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + + for (uint32_t i = 0; i < kW / 8u; i += 2) { + out.data_u64[i + 0] = a.data_u64[0]; + out.data_u64[i + 1] = a.data_u64[1]; + } + return out; + } +}; + +struct vec_op_broadcast_u256 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + if constexpr (kW < 32) { + out = a; + } + else { + for (uint32_t i = 0; i < kW / 8u; i += 4) { + out.data_u64[i + 0] = a.data_u64[0]; + out.data_u64[i + 1] = a.data_u64[1]; + out.data_u64[i + 2] = a.data_u64[2]; + out.data_u64[i + 3] = a.data_u64[3]; + } + } + return out; + } +}; + +template<typename T> struct vec_op_abs : public op_each_vv<T, vec_op_abs<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a) noexcept { return a < 0 ? T(cast_uint(T(0)) - cast_uint(a)) : a; } +}; + +template<typename T> struct vec_op_neg : public op_each_vv<T, vec_op_neg<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a) noexcept { return T(cast_uint(T(0)) - cast_uint(a)); } +}; + +template<typename T> struct vec_op_not : public op_each_vv<T, vec_op_not<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a) noexcept { return T(~a); } +}; + +struct vec_op_cvt_i8_lo_to_i16 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_i16[off / 2 + 0] = a.data_i8[off / 2 + 0]; + out.data_i16[off / 2 + 1] = a.data_i8[off / 2 + 1]; + out.data_i16[off / 2 + 2] = a.data_i8[off / 2 + 2]; + out.data_i16[off / 2 + 3] = a.data_i8[off / 2 + 3]; + out.data_i16[off / 2 + 4] = a.data_i8[off / 2 + 4]; + out.data_i16[off / 2 + 5] = a.data_i8[off / 2 + 5]; + out.data_i16[off / 2 + 6] = a.data_i8[off / 2 + 6]; + out.data_i16[off / 2 + 7] = a.data_i8[off / 2 + 7]; + } + return out; + } +}; + +struct vec_op_cvt_i8_hi_to_i16 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_i16[off / 2 + 0] = a.data_i8[kW / 2 + off / 2 + 0]; + out.data_i16[off / 2 + 1] = a.data_i8[kW / 2 + off / 2 + 1]; + out.data_i16[off / 2 + 2] = a.data_i8[kW / 2 + off / 2 + 2]; + out.data_i16[off / 2 + 3] = a.data_i8[kW / 2 + off / 2 + 3]; + out.data_i16[off / 2 + 4] = a.data_i8[kW / 2 + off / 2 + 4]; + out.data_i16[off / 2 + 5] = a.data_i8[kW / 2 + off / 2 + 5]; + out.data_i16[off / 2 + 6] = a.data_i8[kW / 2 + off / 2 + 6]; + out.data_i16[off / 2 + 7] = a.data_i8[kW / 2 + off / 2 + 7]; + } + return out; + } +}; + +struct vec_op_cvt_u8_lo_to_u16 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_u16[off / 2 + 0] = a.data_u8[off / 2 + 0]; + out.data_u16[off / 2 + 1] = a.data_u8[off / 2 + 1]; + out.data_u16[off / 2 + 2] = a.data_u8[off / 2 + 2]; + out.data_u16[off / 2 + 3] = a.data_u8[off / 2 + 3]; + out.data_u16[off / 2 + 4] = a.data_u8[off / 2 + 4]; + out.data_u16[off / 2 + 5] = a.data_u8[off / 2 + 5]; + out.data_u16[off / 2 + 6] = a.data_u8[off / 2 + 6]; + out.data_u16[off / 2 + 7] = a.data_u8[off / 2 + 7]; + } + return out; + } +}; + +struct vec_op_cvt_u8_hi_to_u16 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_u16[off / 2 + 0] = a.data_u8[kW / 2 + off / 2 + 0]; + out.data_u16[off / 2 + 1] = a.data_u8[kW / 2 + off / 2 + 1]; + out.data_u16[off / 2 + 2] = a.data_u8[kW / 2 + off / 2 + 2]; + out.data_u16[off / 2 + 3] = a.data_u8[kW / 2 + off / 2 + 3]; + out.data_u16[off / 2 + 4] = a.data_u8[kW / 2 + off / 2 + 4]; + out.data_u16[off / 2 + 5] = a.data_u8[kW / 2 + off / 2 + 5]; + out.data_u16[off / 2 + 6] = a.data_u8[kW / 2 + off / 2 + 6]; + out.data_u16[off / 2 + 7] = a.data_u8[kW / 2 + off / 2 + 7]; + } + return out; + } +}; + +struct vec_op_cvt_i8_to_i32 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_i32[off / 4 + 0] = a.data_i8[off / 4 + 0]; + out.data_i32[off / 4 + 1] = a.data_i8[off / 4 + 1]; + out.data_i32[off / 4 + 2] = a.data_i8[off / 4 + 2]; + out.data_i32[off / 4 + 3] = a.data_i8[off / 4 + 3]; + } + return out; + } +}; + +struct vec_op_cvt_u8_to_u32 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_u32[off / 4 + 0] = a.data_u8[off / 4 + 0]; + out.data_u32[off / 4 + 1] = a.data_u8[off / 4 + 1]; + out.data_u32[off / 4 + 2] = a.data_u8[off / 4 + 2]; + out.data_u32[off / 4 + 3] = a.data_u8[off / 4 + 3]; + } + return out; + } +}; + +struct vec_op_cvt_i16_lo_to_i32 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_i32[off / 4 + 0] = a.data_i16[off / 4 + 0]; + out.data_i32[off / 4 + 1] = a.data_i16[off / 4 + 1]; + out.data_i32[off / 4 + 2] = a.data_i16[off / 4 + 2]; + out.data_i32[off / 4 + 3] = a.data_i16[off / 4 + 3]; + } + return out; + } +}; + +struct vec_op_cvt_i16_hi_to_i32 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_i32[off / 4 + 0] = a.data_i16[kW / 4 + off / 4 + 0]; + out.data_i32[off / 4 + 1] = a.data_i16[kW / 4 + off / 4 + 1]; + out.data_i32[off / 4 + 2] = a.data_i16[kW / 4 + off / 4 + 2]; + out.data_i32[off / 4 + 3] = a.data_i16[kW / 4 + off / 4 + 3]; + } + return out; + } +}; + +struct vec_op_cvt_u16_lo_to_u32 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_u32[off / 4 + 0] = a.data_u16[off / 4 + 0]; + out.data_u32[off / 4 + 1] = a.data_u16[off / 4 + 1]; + out.data_u32[off / 4 + 2] = a.data_u16[off / 4 + 2]; + out.data_u32[off / 4 + 3] = a.data_u16[off / 4 + 3]; + } + return out; + } +}; + +struct vec_op_cvt_u16_hi_to_u32 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_u32[off / 4 + 0] = a.data_u16[kW / 4 + off / 4 + 0]; + out.data_u32[off / 4 + 1] = a.data_u16[kW / 4 + off / 4 + 1]; + out.data_u32[off / 4 + 2] = a.data_u16[kW / 4 + off / 4 + 2]; + out.data_u32[off / 4 + 3] = a.data_u16[kW / 4 + off / 4 + 3]; + } + return out; + } +}; + +struct vec_op_cvt_i32_lo_to_i64 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_i64[off / 8 + 0] = a.data_i32[off / 8 + 0]; + out.data_i64[off / 8 + 1] = a.data_i32[off / 8 + 1]; + } + return out; + } +}; + +struct vec_op_cvt_i32_hi_to_i64 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_i64[off / 8 + 0] = a.data_i32[kW / 8 + off / 8 + 0]; + out.data_i64[off / 8 + 1] = a.data_i32[kW / 8 + off / 8 + 1]; + } + return out; + } +}; + +struct vec_op_cvt_u32_lo_to_u64 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_u64[off / 8 + 0] = a.data_u32[off / 8 + 0]; + out.data_u64[off / 8 + 1] = a.data_u32[off / 8 + 1]; + } + return out; + } +}; + +struct vec_op_cvt_u32_hi_to_u64 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_u64[off / 8 + 0] = a.data_u32[kW / 8 + off / 8 + 0]; + out.data_u64[off / 8 + 1] = a.data_u32[kW / 8 + off / 8 + 1]; + } + return out; + } +}; + +template<typename T> struct vec_op_fabs : public op_each_vv<T, vec_op_fabs<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a) noexcept { return std::fabs(a); } +}; + +template<typename T> struct vec_op_trunc : public op_each_vv<T, vec_op_trunc<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a) noexcept { return std::trunc(a); } +}; + +template<typename T> struct vec_op_floor : public op_each_vv<T, vec_op_floor<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a) noexcept { return std::floor(a); } +}; + +template<typename T> struct vec_op_ceil : public op_each_vv<T, vec_op_ceil<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a) noexcept { return std::ceil(a); } +}; + +template<typename T> struct vec_op_round_even : public op_each_vv<T, vec_op_round_even<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a) noexcept { return std::nearbyint(a); } +}; + +template<typename T> struct vec_op_round_half_away : public op_each_vv<T, vec_op_round_half_away<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a) noexcept { return std::trunc(fadd(a, fxor(half_minus_1ulp_const<T>::value, fsign(a)))); } +}; + +template<typename T> struct vec_op_round_half_up : public op_each_vv<T, vec_op_round_half_up<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a) noexcept { return std::floor(fadd(a, half_minus_1ulp_const<T>::value)); } +}; + +template<typename T> struct vec_op_sqrt : public op_each_vv<T, vec_op_sqrt<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a) noexcept { return fsqrt(a); } +}; + +template<typename T> struct vec_op_rcp : public op_each_vv<T, vec_op_rcp<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a) noexcept { return fdiv(T(1), a); } +}; + +struct vec_op_cvt_i32_to_f32 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_f32[off / 4 + 0] = float(a.data_i32[off / 4 + 0]); + out.data_f32[off / 4 + 1] = float(a.data_i32[off / 4 + 1]); + out.data_f32[off / 4 + 2] = float(a.data_i32[off / 4 + 2]); + out.data_f32[off / 4 + 3] = float(a.data_i32[off / 4 + 3]); + } + return out; + } +}; + +template<bool kHi> +struct vec_op_cvt_f32_to_f64_impl { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + uint32_t adj = kHi ? kW / 8 : 0u; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_f64[off / 8 + 0] = double(a.data_f32[off / 8 + adj + 0]); + out.data_f64[off / 8 + 1] = double(a.data_f32[off / 8 + adj + 1]); + } + return out; + } +}; + +struct vec_op_cvt_f32_lo_to_f64 : public vec_op_cvt_f32_to_f64_impl<false> {}; +struct vec_op_cvt_f32_hi_to_f64 : public vec_op_cvt_f32_to_f64_impl<true> {}; + +template<bool kHi> +struct vec_op_cvt_f64_to_f32_impl { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + uint32_t adj = kHi ? kW / 8 : 0u; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_f32[off / 8 + adj + 0] = float(a.data_f64[off / 8 + 0]); + out.data_f32[off / 8 + adj + 1] = float(a.data_f64[off / 8 + 1]); + } + return out; + } +}; + +struct vec_op_cvt_f64_to_f32_lo : public vec_op_cvt_f64_to_f32_impl<false> {}; +struct vec_op_cvt_f64_to_f32_hi : public vec_op_cvt_f64_to_f32_impl<true> {}; + +template<bool kHi> +struct vec_op_cvt_i32_to_f64_impl { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + uint32_t adj = kHi ? kW / 8 : 0u; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_f64[off / 8 + 0] = double(a.data_i32[off / 8 + adj + 0]); + out.data_f64[off / 8 + 1] = double(a.data_i32[off / 8 + adj + 1]); + } + return out; + } +}; + +struct vec_op_cvt_i32_lo_to_f64 : public vec_op_cvt_i32_to_f64_impl<false> {}; +struct vec_op_cvt_i32_hi_to_f64 : public vec_op_cvt_i32_to_f64_impl<true> {}; + +template<FloatToIntOutsideRangeBehavior behavior> +struct vec_op_cvt_trunc_f32_to_i32 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_i32[off / 4 + 0] = cvt_float_to_int_trunc<behavior, int32_t>(a.data_f32[off / 4 + 0]); + out.data_i32[off / 4 + 1] = cvt_float_to_int_trunc<behavior, int32_t>(a.data_f32[off / 4 + 1]); + out.data_i32[off / 4 + 2] = cvt_float_to_int_trunc<behavior, int32_t>(a.data_f32[off / 4 + 2]); + out.data_i32[off / 4 + 3] = cvt_float_to_int_trunc<behavior, int32_t>(a.data_f32[off / 4 + 3]); + } + return out; + } +}; + +template<FloatToIntOutsideRangeBehavior behavior, bool kHi> +struct vec_op_cvt_trunc_f64_to_i32_impl { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + uint32_t adj = kHi ? kW / 8 : 0u; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_i32[off / 8 + adj + 0] = cvt_float_to_int_trunc<behavior, int32_t>(a.data_f64[off / 8 + 0]); + out.data_i32[off / 8 + adj + 1] = cvt_float_to_int_trunc<behavior, int32_t>(a.data_f64[off / 8 + 1]); + } + return out; + } +}; + +template<FloatToIntOutsideRangeBehavior behavior> +struct vec_op_cvt_trunc_f64_to_i32_lo : vec_op_cvt_trunc_f64_to_i32_impl<behavior, false> {}; + +template<FloatToIntOutsideRangeBehavior behavior> +struct vec_op_cvt_trunc_f64_to_i32_hi : vec_op_cvt_trunc_f64_to_i32_impl<behavior, true> {}; + +template<FloatToIntOutsideRangeBehavior behavior> +struct vec_op_cvt_round_f32_to_i32 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_i32[off / 4 + 0] = cvt_float_to_int_round<behavior, int32_t>(a.data_f32[off / 4 + 0]); + out.data_i32[off / 4 + 1] = cvt_float_to_int_round<behavior, int32_t>(a.data_f32[off / 4 + 1]); + out.data_i32[off / 4 + 2] = cvt_float_to_int_round<behavior, int32_t>(a.data_f32[off / 4 + 2]); + out.data_i32[off / 4 + 3] = cvt_float_to_int_round<behavior, int32_t>(a.data_f32[off / 4 + 3]); + } + return out; + } +}; + +template<FloatToIntOutsideRangeBehavior behavior, bool kHi> +struct vec_op_cvt_round_f64_to_i32_impl { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + uint32_t adj = kHi ? kW / 8 : 0u; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_i32[off / 8 + adj + 0] = cvt_float_to_int_round<behavior, int32_t>(a.data_f64[off / 8 + 0]); + out.data_i32[off / 8 + adj + 1] = cvt_float_to_int_round<behavior, int32_t>(a.data_f64[off / 8 + 1]); + } + return out; + } +}; + +template<FloatToIntOutsideRangeBehavior behavior> +struct vec_op_cvt_round_f64_to_i32_lo : vec_op_cvt_round_f64_to_i32_impl<behavior, false> {}; +template<FloatToIntOutsideRangeBehavior behavior> +struct vec_op_cvt_round_f64_to_i32_hi : vec_op_cvt_round_f64_to_i32_impl<behavior, true> {}; + +struct scalar_op_cvt_f32_to_f64 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + out.data_f64[0] = a.data_f32[0]; + return out; + } +}; + +struct scalar_op_cvt_f64_to_f32 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a) noexcept { + VecOverlay<kW> out{}; + out.data_f32[0] = a.data_f64[0]; + return out; + } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_trunc : public op_scalar_vv<kB, T, scalar_op_trunc<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a) noexcept { return std::trunc(a); } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_floor : public op_scalar_vv<kB, T, scalar_op_floor<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a) noexcept { return std::floor(a); } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_ceil : public op_scalar_vv<kB, T, scalar_op_ceil<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a) noexcept { return std::ceil(a); } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_round_even : public op_scalar_vv<kB, T, scalar_op_round_even<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a) noexcept { return std::nearbyint(a); } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_round_half_away : public op_scalar_vv<kB, T, scalar_op_round_half_away<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a) noexcept { return std::trunc(fadd(a, fxor(half_minus_1ulp_const<T>::value, fsign(a)))); } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_round_half_up : public op_scalar_vv<kB, T, scalar_op_round_half_up<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a) noexcept { return std::floor(fadd(a, half_minus_1ulp_const<T>::value)); } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_sqrt : public op_scalar_vv<kB, T, scalar_op_sqrt<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a) noexcept { return fsqrt(a); } +}; + +// ujit::UniCompiler - Tests - Generic Operations - VVI +// ==================================================== + +template<typename T> struct vec_op_slli : public op_each_vvi<T, vec_op_slli<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, uint32_t imm) noexcept { return T(cast_uint(a) << imm); } +}; + +template<typename T> struct vec_op_srli : public op_each_vvi<T, vec_op_srli<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, uint32_t imm) noexcept { return T(cast_uint(a) >> imm); } +}; + +template<typename T> struct vec_op_rsrli : public op_each_vvi<T, vec_op_rsrli<T>> { + static ASMJIT_INLINE T apply_one(const T& a, uint32_t imm) noexcept { + T add = T((a & (T(1) << (imm - 1))) != 0); + return T((cast_uint(a) >> imm) + cast_uint(add)); + } +}; + +template<typename T> struct vec_op_srai : public op_each_vvi<T, vec_op_srai<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, uint32_t imm) noexcept { return T(cast_int(a) >> imm); } +}; + +struct vec_op_sllb_u128 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, uint32_t imm) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + for (uint32_t i = 0; i < 16; i++) { + out.data_u8[off + i] = i < imm ? uint8_t(0) : a.data_u8[off + i - imm]; + } + } + return out; + } +}; + +struct vec_op_srlb_u128 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, uint32_t imm) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + for (uint32_t i = 0; i < 16; i++) { + out.data_u8[off + i] = i + imm < 16u ? a.data_u8[off + i + imm] : uint8_t(0); + } + } + return out; + } +}; + +struct vec_op_swizzle_u16 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, uint32_t imm) noexcept { + uint32_t D = (imm >> 24) & 0x3; + uint32_t C = (imm >> 16) & 0x3; + uint32_t B = (imm >> 8) & 0x3; + uint32_t A = (imm >> 0) & 0x3; + + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_u16[off / 2 + 0] = a.data_u16[off / 2 + 0 + A]; + out.data_u16[off / 2 + 1] = a.data_u16[off / 2 + 0 + B]; + out.data_u16[off / 2 + 2] = a.data_u16[off / 2 + 0 + C]; + out.data_u16[off / 2 + 3] = a.data_u16[off / 2 + 0 + D]; + out.data_u16[off / 2 + 4] = a.data_u16[off / 2 + 4 + A]; + out.data_u16[off / 2 + 5] = a.data_u16[off / 2 + 4 + B]; + out.data_u16[off / 2 + 6] = a.data_u16[off / 2 + 4 + C]; + out.data_u16[off / 2 + 7] = a.data_u16[off / 2 + 4 + D]; + } + return out; + } +}; + +struct vec_op_swizzle_lo_u16x4 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, uint32_t imm) noexcept { + uint32_t D = (imm >> 24) & 0x3; + uint32_t C = (imm >> 16) & 0x3; + uint32_t B = (imm >> 8) & 0x3; + uint32_t A = (imm >> 0) & 0x3; + + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_u16[off / 2 + 0] = a.data_u16[off / 2 + A]; + out.data_u16[off / 2 + 1] = a.data_u16[off / 2 + B]; + out.data_u16[off / 2 + 2] = a.data_u16[off / 2 + C]; + out.data_u16[off / 2 + 3] = a.data_u16[off / 2 + D]; + memcpy(out.data_u8 + off + 8, a.data_u8 + off + 8, 8); + } + return out; + } +}; + +struct vec_op_swizzle_hi_u16x4 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, uint32_t imm) noexcept { + uint32_t D = (imm >> 24) & 0x3; + uint32_t C = (imm >> 16) & 0x3; + uint32_t B = (imm >> 8) & 0x3; + uint32_t A = (imm >> 0) & 0x3; + + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + memcpy(out.data_u8 + off, a.data_u8 + off, 8); + out.data_u16[off / 2 + 4] = a.data_u16[off / 2 + 4 + A]; + out.data_u16[off / 2 + 5] = a.data_u16[off / 2 + 4 + B]; + out.data_u16[off / 2 + 6] = a.data_u16[off / 2 + 4 + C]; + out.data_u16[off / 2 + 7] = a.data_u16[off / 2 + 4 + D]; + } + return out; + } +}; + +struct vec_op_swizzle_u32x4 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, uint32_t imm) noexcept { + uint32_t D = (imm >> 24) & 0x3; + uint32_t C = (imm >> 16) & 0x3; + uint32_t B = (imm >> 8) & 0x3; + uint32_t A = (imm >> 0) & 0x3; + + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_u32[off / 4 + 0] = a.data_u32[off / 4 + A]; + out.data_u32[off / 4 + 1] = a.data_u32[off / 4 + B]; + out.data_u32[off / 4 + 2] = a.data_u32[off / 4 + C]; + out.data_u32[off / 4 + 3] = a.data_u32[off / 4 + D]; + } + return out; + } +}; + +struct vec_op_swizzle_u64x2 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, uint32_t imm) noexcept { + uint32_t B = (imm >> 8) & 0x1; + uint32_t A = (imm >> 0) & 0x1; + + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_u64[off / 8 + 0] = a.data_u64[off / 8 + A]; + out.data_u64[off / 8 + 1] = a.data_u64[off / 8 + B]; + } + return out; + } +}; + +// ujit::UniCompiler - Tests - SIMD - Generic Operations - VVV +// =========================================================== + +template<typename T> struct vec_op_and : public op_each_vvv<T, vec_op_and<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return T(a & b); } +}; + +template<typename T> struct vec_op_or : public op_each_vvv<T, vec_op_or<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return T(a | b); } +}; + +template<typename T> struct vec_op_xor : public op_each_vvv<T, vec_op_xor<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return T(a ^ b); } +}; + +template<typename T> struct vec_op_andn : public op_each_vvv<T, vec_op_andn<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return T(~a & b); } +}; + +template<typename T> struct vec_op_bic : public op_each_vvv<T, vec_op_bic<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return T(a & ~b); } +}; + +template<typename T> struct vec_op_add : public op_each_vvv<T, vec_op_add<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return T(cast_uint(a) + cast_uint(b)); } +}; + +template<typename T> struct vec_op_adds : public op_each_vvv<T, vec_op_adds<T>> { + static ASMJIT_INLINE T apply_one(const T& a, const T& b) noexcept { + Support::FastUInt8 of{}; + T result = Support::add_overflow(a, b, &of); + + if (!of) { + return result; + } + + if constexpr (std::is_unsigned_v<T>) { + return std::numeric_limits<T>::max(); + } + else { + return b > T(0) ? std::numeric_limits<T>::max() : std::numeric_limits<T>::lowest(); + } + } +}; + +template<typename T> struct vec_op_sub : public op_each_vvv<T, vec_op_sub<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return T(cast_uint(a) - cast_uint(b)); } +}; + +template<typename T> struct vec_op_subs : public op_each_vvv<T, vec_op_subs<T>> { + static ASMJIT_INLINE T apply_one(const T& a, const T& b) noexcept { + Support::FastUInt8 of{}; + T result = Support::sub_overflow(a, b, &of); + + if (!of) { + return result; + } + + if constexpr (std::is_unsigned_v<T>) { + return std::numeric_limits<T>::lowest(); + } + else { + return b > T(0) ? std::numeric_limits<T>::lowest() : std::numeric_limits<T>::max(); + } + } +}; + +template<typename T> struct vec_op_mul : public op_each_vvv<T, vec_op_mul<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return T((uint64_t(a) * uint64_t(b)) & uint64_t(~T(0))); } +}; + +template<typename T> struct vec_op_mulhi : public op_each_vvv<T, vec_op_mulhi<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { + uint64_t result = uint64_t(int64_t(cast_int(a))) * uint64_t(int64_t(cast_int(b))); + return T(T(result >> (sizeof(T) * 8u)) & T(~T(0))); + } +}; + +template<typename T> struct vec_op_mulhu : public op_each_vvv<T, vec_op_mulhu<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { + uint64_t result = uint64_t(a) * uint64_t(b); + return T(result >> (sizeof(T) * 8u)) & uint64_t(~T(0)); + } +}; + +struct vec_op_mul_u64_lo_u32 : public op_each_vvv<uint64_t, vec_op_mul_u64_lo_u32> { + static ASMJIT_INLINE_NODEBUG uint64_t apply_one(const uint64_t& a, const uint64_t& b) noexcept { + return uint64_t(a) * uint64_t(b & 0xFFFFFFFFu); + } +}; + +struct vec_op_mhadd_i16_i32 : public op_each_vvv<uint32_t, vec_op_mhadd_i16_i32> { + static ASMJIT_INLINE_NODEBUG uint32_t apply_one(const uint32_t& a, const uint32_t& b) noexcept { + uint32_t al = uint32_t(int32_t(int16_t(a & 0xFFFF))); + uint32_t ah = uint32_t(int32_t(int16_t(a >> 16))); + + uint32_t bl = uint32_t(int32_t(int16_t(b & 0xFFFF))); + uint32_t bh = uint32_t(int32_t(int16_t(b >> 16))); + + return al * bl + ah * bh; + } +}; + +template<typename T> struct vec_op_madd : public op_each_vvvv<T, vec_op_madd<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b, const T& c) noexcept { return T((uint64_t(a) * uint64_t(b) + uint64_t(c)) & uint64_t(~T(0))); } +}; + +template<typename T> struct vec_op_min : public op_each_vvv<T, vec_op_min<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return a < b ? a : b; } +}; + +template<typename T> struct vec_op_max : public op_each_vvv<T, vec_op_max<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return a > b ? a : b; } +}; + +template<typename T> struct vec_op_cmp_eq : public op_each_vvv<T, vec_op_cmp_eq<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return a == b ? Support::bit_ones<T> : T(0); } +}; + +template<typename T> struct vec_op_cmp_ne : public op_each_vvv<T, vec_op_cmp_ne<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return a != b ? Support::bit_ones<T> : T(0); } +}; + +template<typename T> struct vec_op_cmp_gt : public op_each_vvv<T, vec_op_cmp_gt<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return a > b ? Support::bit_ones<T> : T(0); } +}; + +template<typename T> struct vec_op_cmp_ge : public op_each_vvv<T, vec_op_cmp_ge<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return a >= b ? Support::bit_ones<T> : T(0); } +}; + +template<typename T> struct vec_op_cmp_lt : public op_each_vvv<T, vec_op_cmp_lt<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return a < b ? Support::bit_ones<T> : T(0); } +}; + +template<typename T> struct vec_op_cmp_le : public op_each_vvv<T, vec_op_cmp_le<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return a <= b ? Support::bit_ones<T> : T(0); } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_fadd : public op_scalar_vvv<kB, T, scalar_op_fadd<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return fadd(a, b); } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_fsub : public op_scalar_vvv<kB, T, scalar_op_fsub<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return fsub(a, b); } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_fmul : public op_scalar_vvv<kB, T, scalar_op_fmul<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return fmul(a, b); } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_fdiv : public op_scalar_vvv<kB, T, scalar_op_fdiv<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return fdiv(a, b); } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_fmin_ternary : public op_scalar_vvv<kB, T, scalar_op_fmin_ternary<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return a < b ? a : b; } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_fmax_ternary : public op_scalar_vvv<kB, T, scalar_op_fmax_ternary<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return a > b ? a : b; } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_fmin_finite : public op_scalar_vvv<kB, T, scalar_op_fmin_finite<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return std::isnan(a) ? b : std::isnan(b) ? a : Support::min(a, b); } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_fmax_finite : public op_scalar_vvv<kB, T, scalar_op_fmax_finite<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return std::isnan(a) ? b : std::isnan(b) ? a : Support::max(a, b); } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_fmadd_nofma : public op_scalar_vvvv<kB, T, scalar_op_fmadd_nofma<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b, const T& c) noexcept { return fmadd_nofma_ref(a, b, c); } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_fmsub_nofma : public op_scalar_vvvv<kB, T, scalar_op_fmsub_nofma<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b, const T& c) noexcept { return fmadd_nofma_ref(a, b, -c); } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_fnmadd_nofma : public op_scalar_vvvv<kB, T, scalar_op_fnmadd_nofma<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b, const T& c) noexcept { return fmadd_nofma_ref(-a, b, c); } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_fnmsub_nofma : public op_scalar_vvvv<kB, T, scalar_op_fnmsub_nofma<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b, const T& c) noexcept { return fmadd_nofma_ref(-a, b, -c); } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_fmadd_fma : public op_scalar_vvvv<kB, T, scalar_op_fmadd_fma<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b, const T& c) noexcept { return fmadd_fma_ref(a, b, c); } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_fmsub_fma : public op_scalar_vvvv<kB, T, scalar_op_fmsub_fma<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b, const T& c) noexcept { return fmadd_fma_ref(a, b, -c); } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_fnmadd_fma : public op_scalar_vvvv<kB, T, scalar_op_fnmadd_fma<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b, const T& c) noexcept { return fmadd_fma_ref(-a, b, c); } +}; + +template<ScalarOpBehavior kB, typename T> struct scalar_op_fnmsub_fma : public op_scalar_vvvv<kB, T, scalar_op_fnmsub_fma<kB, T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b, const T& c) noexcept { return fmadd_fma_ref(-a, b, -c); } +}; + +template<typename T> struct vec_op_fadd : public op_each_vvv<T, vec_op_fadd<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return fadd(a, b); } +}; + +template<typename T> struct vec_op_fsub : public op_each_vvv<T, vec_op_fsub<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return fsub(a, b); } +}; + +template<typename T> struct vec_op_fmul : public op_each_vvv<T, vec_op_fmul<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return fmul(a, b); } +}; + +template<typename T> struct vec_op_fdiv : public op_each_vvv<T, vec_op_fdiv<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return fdiv(a, b); } +}; + +template<typename T> struct vec_op_fmin_ternary : public op_each_vvv<T, vec_op_fmin_ternary<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return a < b ? a : b; } +}; + +template<typename T> struct vec_op_fmax_ternary : public op_each_vvv<T, vec_op_fmax_ternary<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return a > b ? a : b; } +}; + +template<typename T> struct vec_op_fmin_finite : public op_each_vvv<T, vec_op_fmin_finite<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return std::isnan(a) ? b : std::isnan(b) ? a : Support::min(a, b); } +}; + +template<typename T> struct vec_op_fmax_finite : public op_each_vvv<T, vec_op_fmax_finite<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b) noexcept { return std::isnan(a) ? b : std::isnan(b) ? a : Support::max(a, b); } +}; + +template<typename T> struct vec_op_fmadd_nofma : public op_each_vvvv<T, vec_op_fmadd_nofma<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b, const T& c) noexcept { return fmadd_nofma_ref(a, b, c); } +}; + +template<typename T> struct vec_op_fmsub_nofma : public op_each_vvvv<T, vec_op_fmsub_nofma<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b, const T& c) noexcept { return fmadd_nofma_ref(a, b, -c); } +}; + +template<typename T> struct vec_op_fnmadd_nofma : public op_each_vvvv<T, vec_op_fnmadd_nofma<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b, const T& c) noexcept { return fmadd_nofma_ref(-a, b, c); } +}; + +template<typename T> struct vec_op_fnmsub_nofma : public op_each_vvvv<T, vec_op_fnmsub_nofma<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b, const T& c) noexcept { return fmadd_nofma_ref(-a, b, -c); } +}; + +template<typename T> struct vec_op_fmadd_fma : public op_each_vvvv<T, vec_op_fmadd_fma<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b, const T& c) noexcept { return fmadd_fma_ref(a, b, c); } +}; + +template<typename T> struct vec_op_fmsub_fma : public op_each_vvvv<T, vec_op_fmsub_fma<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b, const T& c) noexcept { return fmadd_fma_ref(a, b, -c); } +}; + +template<typename T> struct vec_op_fnmadd_fma : public op_each_vvvv<T, vec_op_fnmadd_fma<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b, const T& c) noexcept { return fmadd_fma_ref(-a, b, c); } +}; + +template<typename T> struct vec_op_fnmsub_fma : public op_each_vvvv<T, vec_op_fnmsub_fma<T>> { + static ASMJIT_INLINE_NODEBUG T apply_one(const T& a, const T& b, const T& c) noexcept { return fmadd_fma_ref(-a, b, -c); } +}; + +template<typename T> +struct cmp_result { + typedef T Result; + + static ASMJIT_INLINE_NODEBUG Result make(bool result) noexcept { return result ? Result(~Result(0)) : Result(0); } +}; + +template<> +struct cmp_result<float> { + typedef uint32_t Result; + + static ASMJIT_INLINE_NODEBUG Result make(bool result) noexcept { return result ? ~Result(0) : Result(0); } +}; + +template<> +struct cmp_result<double> { + typedef uint64_t Result; + + static ASMJIT_INLINE_NODEBUG Result make(bool result) noexcept { return result ? ~Result(0) : Result(0); } +}; + +template<typename T> struct vec_op_fcmpo_eq : public op_each_vvv<T, vec_op_fcmpo_eq<T>> { + static ASMJIT_INLINE_NODEBUG typename cmp_result<T>::Result apply_one(const T& a, const T& b) noexcept { return cmp_result<T>::make(a == b); } +}; + +template<typename T> struct vec_op_fcmpu_ne : public op_each_vvv<T, vec_op_fcmpu_ne<T>> { + static ASMJIT_INLINE_NODEBUG typename cmp_result<T>::Result apply_one(const T& a, const T& b) noexcept { return cmp_result<T>::make(!(a == b)); } +}; + +template<typename T> struct vec_op_fcmpo_gt : public op_each_vvv<T, vec_op_fcmpo_gt<T>> { + static ASMJIT_INLINE_NODEBUG typename cmp_result<T>::Result apply_one(const T& a, const T& b) noexcept { return cmp_result<T>::make(a > b); } +}; + +template<typename T> struct vec_op_fcmpo_ge : public op_each_vvv<T, vec_op_fcmpo_ge<T>> { + static ASMJIT_INLINE_NODEBUG typename cmp_result<T>::Result apply_one(const T& a, const T& b) noexcept { return cmp_result<T>::make(a >= b); } +}; + +template<typename T> struct vec_op_fcmpo_lt : public op_each_vvv<T, vec_op_fcmpo_lt<T>> { + static ASMJIT_INLINE_NODEBUG typename cmp_result<T>::Result apply_one(const T& a, const T& b) noexcept { return cmp_result<T>::make(a < b); } +}; + +template<typename T> struct vec_op_fcmpo_le : public op_each_vvv<T, vec_op_fcmpo_le<T>> { + static ASMJIT_INLINE_NODEBUG typename cmp_result<T>::Result apply_one(const T& a, const T& b) noexcept { return cmp_result<T>::make(a <= b); } +}; + +template<typename T> struct vec_op_fcmp_ord : public op_each_vvv<T, vec_op_fcmp_ord<T>> { + static ASMJIT_INLINE_NODEBUG typename cmp_result<T>::Result apply_one(const T& a, const T& b) noexcept { return cmp_result<T>::make(!std::isnan(a) && !std::isnan(b)); } +}; + +template<typename T> struct vec_op_fcmp_unord : public op_each_vvv<T, vec_op_fcmp_unord<T>> { + static ASMJIT_INLINE_NODEBUG typename cmp_result<T>::Result apply_one(const T& a, const T& b) noexcept { return cmp_result<T>::make(std::isnan(a) || std::isnan(b)); } +}; + +struct vec_op_hadd_f64 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_f64[off / 8 + 0] = a.data_f64[off / 8 + 0] + a.data_f64[off / 8 + 1]; + out.data_f64[off / 8 + 1] = b.data_f64[off / 8 + 0] + b.data_f64[off / 8 + 1]; + } + return out; + } +}; + +struct vec_op_combine_lo_hi_u64 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_u64[off / 8 + 0] = b.data_u64[off / 8 + 1]; + out.data_u64[off / 8 + 1] = a.data_u64[off / 8 + 0]; + } + return out; + } +}; + +struct vec_op_combine_hi_lo_u64 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_u64[off / 8 + 0] = b.data_u64[off / 8 + 0]; + out.data_u64[off / 8 + 1] = a.data_u64[off / 8 + 1]; + } + return out; + } +}; + +struct vec_op_interleave_lo_u8 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + for (uint32_t i = 0; i < 8; i++) { + out.data_u8[off + i * 2 + 0] = a.data_u8[off + i]; + out.data_u8[off + i * 2 + 1] = b.data_u8[off + i]; + } + } + return out; + } +}; + +struct vec_op_interleave_hi_u8 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + for (uint32_t i = 0; i < 8; i++) { + out.data_u8[off + i * 2 + 0] = a.data_u8[off + 8 + i]; + out.data_u8[off + i * 2 + 1] = b.data_u8[off + 8 + i]; + } + } + return out; + } +}; + +struct vec_op_interleave_lo_u16 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + for (uint32_t i = 0; i < 4; i++) { + out.data_u16[off / 2 + i * 2 + 0] = a.data_u16[off / 2 + i]; + out.data_u16[off / 2 + i * 2 + 1] = b.data_u16[off / 2 + i]; + } + } + return out; + } +}; + +struct vec_op_interleave_hi_u16 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + for (uint32_t i = 0; i < 4; i++) { + out.data_u16[off / 2 + i * 2 + 0] = a.data_u16[off / 2 + 4 + i]; + out.data_u16[off / 2 + i * 2 + 1] = b.data_u16[off / 2 + 4 + i]; + } + } + return out; + } +}; + +struct vec_op_interleave_lo_u32 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + for (uint32_t i = 0; i < 2; i++) { + out.data_u32[off / 4 + i * 2 + 0] = a.data_u32[off / 4 + i]; + out.data_u32[off / 4 + i * 2 + 1] = b.data_u32[off / 4 + i]; + } + } + return out; + } +}; + +struct vec_op_interleave_hi_u32 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + for (uint32_t i = 0; i < 2; i++) { + out.data_u32[off / 4 + i * 2 + 0] = a.data_u32[off / 4 + 2 + i]; + out.data_u32[off / 4 + i * 2 + 1] = b.data_u32[off / 4 + 2 + i]; + } + } + return out; + } +}; + +struct vec_op_interleave_lo_u64 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_u64[off / 8 + 0] = a.data_u64[off / 8 + 0]; + out.data_u64[off / 8 + 1] = b.data_u64[off / 8 + 0]; + } + return out; + } +}; + +struct vec_op_interleave_hi_u64 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_u64[off / 8 + 0] = a.data_u64[off / 8 + 1]; + out.data_u64[off / 8 + 1] = b.data_u64[off / 8 + 1]; + } + return out; + } +}; + +// ujit::UniCompiler - Tests - SIMD - Generic Operations - VVVI +// ============================================================ + +struct vec_op_alignr_u128 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b, uint32_t imm) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + for (uint32_t i = 0; i < 16; i++) { + out.data_u8[off + i] = i + imm < 16 ? b.data_u8[off + i + imm] : a.data_u8[off + i + imm - 16]; + } + } + return out; + } +}; + +struct vec_op_interleave_shuffle_u32x4 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b, uint32_t imm) noexcept { + uint32_t D = (imm >> 24) & 0x3; + uint32_t C = (imm >> 16) & 0x3; + uint32_t B = (imm >> 8) & 0x3; + uint32_t A = (imm >> 0) & 0x3; + + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_u32[off / 4 + 0] = a.data_u32[off / 4 + A]; + out.data_u32[off / 4 + 1] = a.data_u32[off / 4 + B]; + out.data_u32[off / 4 + 2] = b.data_u32[off / 4 + C]; + out.data_u32[off / 4 + 3] = b.data_u32[off / 4 + D]; + } + return out; + } +}; + +struct vec_op_interleave_shuffle_u64x2 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b, uint32_t imm) noexcept { + uint32_t B = (imm >> 8) & 0x1; + uint32_t A = (imm >> 0) & 0x1; + + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_u64[off / 8 + 0] = a.data_u64[off / 8 + A]; + out.data_u64[off / 8 + 1] = b.data_u64[off / 8 + B]; + } + return out; + } +}; + +struct vec_op_packs_i16_i8 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_i8[off + 0] = saturate_i16_to_i8(a.data_i16[off / 2 + 0]); + out.data_i8[off + 1] = saturate_i16_to_i8(a.data_i16[off / 2 + 1]); + out.data_i8[off + 2] = saturate_i16_to_i8(a.data_i16[off / 2 + 2]); + out.data_i8[off + 3] = saturate_i16_to_i8(a.data_i16[off / 2 + 3]); + out.data_i8[off + 4] = saturate_i16_to_i8(a.data_i16[off / 2 + 4]); + out.data_i8[off + 5] = saturate_i16_to_i8(a.data_i16[off / 2 + 5]); + out.data_i8[off + 6] = saturate_i16_to_i8(a.data_i16[off / 2 + 6]); + out.data_i8[off + 7] = saturate_i16_to_i8(a.data_i16[off / 2 + 7]); + out.data_i8[off + 8] = saturate_i16_to_i8(b.data_i16[off / 2 + 0]); + out.data_i8[off + 9] = saturate_i16_to_i8(b.data_i16[off / 2 + 1]); + out.data_i8[off + 10] = saturate_i16_to_i8(b.data_i16[off / 2 + 2]); + out.data_i8[off + 11] = saturate_i16_to_i8(b.data_i16[off / 2 + 3]); + out.data_i8[off + 12] = saturate_i16_to_i8(b.data_i16[off / 2 + 4]); + out.data_i8[off + 13] = saturate_i16_to_i8(b.data_i16[off / 2 + 5]); + out.data_i8[off + 14] = saturate_i16_to_i8(b.data_i16[off / 2 + 6]); + out.data_i8[off + 15] = saturate_i16_to_i8(b.data_i16[off / 2 + 7]); + } + return out; + } +}; + +struct vec_op_packs_i16_u8 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_u8[off + 0] = saturate_i16_to_u8(a.data_i16[off / 2 + 0]); + out.data_u8[off + 1] = saturate_i16_to_u8(a.data_i16[off / 2 + 1]); + out.data_u8[off + 2] = saturate_i16_to_u8(a.data_i16[off / 2 + 2]); + out.data_u8[off + 3] = saturate_i16_to_u8(a.data_i16[off / 2 + 3]); + out.data_u8[off + 4] = saturate_i16_to_u8(a.data_i16[off / 2 + 4]); + out.data_u8[off + 5] = saturate_i16_to_u8(a.data_i16[off / 2 + 5]); + out.data_u8[off + 6] = saturate_i16_to_u8(a.data_i16[off / 2 + 6]); + out.data_u8[off + 7] = saturate_i16_to_u8(a.data_i16[off / 2 + 7]); + out.data_u8[off + 8] = saturate_i16_to_u8(b.data_i16[off / 2 + 0]); + out.data_u8[off + 9] = saturate_i16_to_u8(b.data_i16[off / 2 + 1]); + out.data_u8[off + 10] = saturate_i16_to_u8(b.data_i16[off / 2 + 2]); + out.data_u8[off + 11] = saturate_i16_to_u8(b.data_i16[off / 2 + 3]); + out.data_u8[off + 12] = saturate_i16_to_u8(b.data_i16[off / 2 + 4]); + out.data_u8[off + 13] = saturate_i16_to_u8(b.data_i16[off / 2 + 5]); + out.data_u8[off + 14] = saturate_i16_to_u8(b.data_i16[off / 2 + 6]); + out.data_u8[off + 15] = saturate_i16_to_u8(b.data_i16[off / 2 + 7]); + } + return out; + } +}; + +struct vec_op_packs_i32_i16 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_i16[off / 2 + 0] = saturate_i32_to_i16(a.data_i32[off / 4 + 0]); + out.data_i16[off / 2 + 1] = saturate_i32_to_i16(a.data_i32[off / 4 + 1]); + out.data_i16[off / 2 + 2] = saturate_i32_to_i16(a.data_i32[off / 4 + 2]); + out.data_i16[off / 2 + 3] = saturate_i32_to_i16(a.data_i32[off / 4 + 3]); + out.data_i16[off / 2 + 4] = saturate_i32_to_i16(b.data_i32[off / 4 + 0]); + out.data_i16[off / 2 + 5] = saturate_i32_to_i16(b.data_i32[off / 4 + 1]); + out.data_i16[off / 2 + 6] = saturate_i32_to_i16(b.data_i32[off / 4 + 2]); + out.data_i16[off / 2 + 7] = saturate_i32_to_i16(b.data_i32[off / 4 + 3]); + } + return out; + } +}; + +struct vec_op_packs_i32_u16 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + out.data_u16[off / 2 + 0] = saturate_i32_to_u16(a.data_i32[off / 4 + 0]); + out.data_u16[off / 2 + 1] = saturate_i32_to_u16(a.data_i32[off / 4 + 1]); + out.data_u16[off / 2 + 2] = saturate_i32_to_u16(a.data_i32[off / 4 + 2]); + out.data_u16[off / 2 + 3] = saturate_i32_to_u16(a.data_i32[off / 4 + 3]); + out.data_u16[off / 2 + 4] = saturate_i32_to_u16(b.data_i32[off / 4 + 0]); + out.data_u16[off / 2 + 5] = saturate_i32_to_u16(b.data_i32[off / 4 + 1]); + out.data_u16[off / 2 + 6] = saturate_i32_to_u16(b.data_i32[off / 4 + 2]); + out.data_u16[off / 2 + 7] = saturate_i32_to_u16(b.data_i32[off / 4 + 3]); + } + return out; + } +}; + +// ujit::UniCompiler - Tests - SIMD - Generic Operations - VVVV +// ============================================================ + +struct vec_op_blendv_bits : public op_each_vvvv<uint32_t, vec_op_blendv_bits> { + static ASMJIT_INLINE_NODEBUG uint32_t apply_one(const uint32_t& a, const uint32_t& b, const uint32_t& c) noexcept { return ((a & ~c) | (b & c)); } +}; + +struct vec_op_swizzlev_u8 { + template<uint32_t kW> + static ASMJIT_INLINE VecOverlay<kW> apply(const VecOverlay<kW>& a, const VecOverlay<kW>& b) noexcept { + VecOverlay<kW> out{}; + for (uint32_t off = 0; off < kW; off += 16) { + for (uint32_t i = 0; i < 16; i++) { + size_t sel = b.data_u8[off + i] & (0x8F); // 3 bits ignored. + out.data_u8[off + i] = sel & 0x80 ? uint8_t(0) : a.data_u8[off + sel]; + } + } + return out; + } +}; + +struct vec_op_div255_u16 : public op_each_vv<uint16_t, vec_op_div255_u16> { + static ASMJIT_INLINE uint16_t apply_one(const uint16_t& a) noexcept { + uint32_t x = a + 0x80u; + return uint16_t((x + (x >> 8)) >> 8); + } +}; + +struct vec_op_div65535_u32 : public op_each_vv<uint32_t, vec_op_div65535_u32> { + static ASMJIT_INLINE uint32_t apply_one(const uint32_t& a) noexcept { + uint32_t x = a + 0x8000u; + return uint32_t((x + (x >> 16)) >> 16); + } +}; + +// ujit::UniCompiler - Tests - SIMD - Utilities +// ============================================ + +template<uint32_t kW> +static void fill_random_bytes(DataGenInt& dg, VecOverlay<kW>& dst) noexcept { + for (uint32_t i = 0; i < kW / 8u; i++) { + dst.data_u64[i] = dg.next_uint64(); + } +} + +template<uint32_t kW> +static void fill_random_f32(DataGenInt& dg, VecOverlay<kW>& dst) noexcept { + for (uint32_t i = 0; i < kW / 4u; i++) { + dst.data_f32[i] = dg.next_float32(); + } +} + +template<uint32_t kW> +static void fill_random_f64(DataGenInt& dg, VecOverlay<kW>& dst) noexcept { + for (uint32_t i = 0; i < kW / 8u; i++) { + dst.data_f64[i] = dg.next_float64(); + } +} + +template<uint32_t kW> +static void fill_random_data(DataGenInt& dg, VecOverlay<kW>& dst, VecElementType element_type) noexcept { + switch (element_type) { + case VecElementType::kFloat32: + fill_random_f32(dg, dst); + break; + + case VecElementType::kFloat64: + fill_random_f64(dg, dst); + break; + + default: + fill_random_bytes(dg, dst); + break; + } +} + +// ujit::UniCompiler - Tests - SIMD - Verification +// =============================================== + +template<uint32_t kW> +static ASMJIT_NOINLINE void test_vecop_vv_failed(UniOpVV op, Variation variation, const VecOverlay<kW>& arg0, const VecOverlay<kW>& observed, const VecOverlay<kW>& expected, const char* assembly) noexcept { + VecOpInfo op_info = vec_op_info_vv(op); + + String arg0_str = vec_stringify(arg0, op_info.arg(0)); + String observed_str = vec_stringify(observed, op_info.ret()); + String expected_str = vec_stringify(expected, op_info.ret()); + + EXPECT(false) + .message("Operation '%s' (variation %u) failed:\n" + " Input #0: %s\n" + " Expected: %s\n" + " Observed: %s\n" + "Assembly:\n%s", + vec_op_name_vv(op), + variation.value, + arg0_str.data(), + expected_str.data(), + observed_str.data(), + assembly); +} + +template<uint32_t kW> +static ASMJIT_NOINLINE void test_vecop_vvi_failed(UniOpVVI op, Variation variation, const VecOverlay<kW>& arg0, const VecOverlay<kW>& observed, const VecOverlay<kW>& expected, uint32_t imm, const char* assembly) noexcept { + VecOpInfo op_info = vec_op_info_vvi(op); + + String arg0_str = vec_stringify(arg0, op_info.arg(0)); + String observed_str = vec_stringify(observed, op_info.ret()); + String expected_str = vec_stringify(expected, op_info.ret()); + + EXPECT(false) + .message("Operation '%s' (variation %u) failed:\n" + " Input #0: %s\n" + " ImmValue: %u (0x%08X)\n" + " Expected: %s\n" + " Observed: %s\n" + "Assembly:\n%s", + vec_op_name_vvi(op), + variation.value, + arg0_str.data(), + imm, imm, + expected_str.data(), + observed_str.data(), + assembly); +} + +template<uint32_t kW> +static ASMJIT_NOINLINE void test_vecop_vvv_failed(UniOpVVV op, Variation variation, const VecOverlay<kW>& arg0, const VecOverlay<kW>& arg1, const VecOverlay<kW>& observed, const VecOverlay<kW>& expected, const char* assembly) noexcept { + VecOpInfo op_info = vec_op_info_vvv(op); + + String arg0_str = vec_stringify(arg0, op_info.arg(0)); + String arg1_str = vec_stringify(arg1, op_info.arg(1)); + String observed_str = vec_stringify(observed, op_info.ret()); + String expected_str = vec_stringify(expected, op_info.ret()); + + EXPECT(false) + .message("Operation '%s' (variation %u) failed:\n" + " Input #0: %s\n" + " Input #1: %s\n" + " Expected: %s\n" + " Observed: %s\n" + "Assembly:\n%s", + vec_op_name_vvv(op), + variation.value, + arg0_str.data(), + arg1_str.data(), + expected_str.data(), + observed_str.data(), + assembly); +} + +template<uint32_t kW> +static ASMJIT_NOINLINE void test_vecop_vvvi_failed(UniOpVVVI op, Variation variation, const VecOverlay<kW>& arg0, const VecOverlay<kW>& arg1, const VecOverlay<kW>& observed, const VecOverlay<kW>& expected, uint32_t imm, const char* assembly) noexcept { + VecOpInfo op_info = vec_op_info_vvvi(op); + + String arg0_str = vec_stringify(arg0, op_info.arg(0)); + String arg1_str = vec_stringify(arg1, op_info.arg(1)); + String observed_str = vec_stringify(observed, op_info.ret()); + String expected_str = vec_stringify(expected, op_info.ret()); + + EXPECT(false) + .message("Operation '%s' (variation %u) failed:\n" + " Input #1: %s\n" + " Input #2: %s\n" + " ImmValue: %u (0x%08X)\n" + " Expected: %s\n" + " Observed: %s\n" + "Assembly:\n%s", + vec_op_name_vvvi(op), + variation.value, + arg0_str.data(), + arg1_str.data(), + imm, imm, + expected_str.data(), + observed_str.data(), + assembly); +} + +template<uint32_t kW> +static ASMJIT_NOINLINE void test_vecop_vvvv_failed(UniOpVVVV op, Variation variation, const VecOverlay<kW>& arg0, const VecOverlay<kW>& arg1, const VecOverlay<kW>& arg2, const VecOverlay<kW>& observed, const VecOverlay<kW>& expected, const char* assembly) noexcept { + VecOpInfo op_info = vec_op_info_vvvv(op); + + String arg0_str = vec_stringify(arg0, op_info.arg(0)); + String arg1_str = vec_stringify(arg1, op_info.arg(1)); + String arg2_str = vec_stringify(arg2, op_info.arg(2)); + String observed_str = vec_stringify(observed, op_info.ret()); + String expected_str = vec_stringify(expected, op_info.ret()); + + EXPECT(false) + .message("Operation '%s' (variation %u) failed\n" + " Input #1: %s\n" + " Input #2: %s\n" + " Input #3: %s\n" + " Expected: %s\n" + " Observed: %s\n" + "Assembly:\n%s", + vec_op_name_vvvv(op), + variation.value, + arg0_str.data(), + arg1_str.data(), + arg2_str.data(), + expected_str.data(), + observed_str.data(), + assembly); +} + +// ujit::UniCompiler - Tests - Integer Operations - VV +// =================================================== + +template<VecWidth kVecWidth, UniOpVV kOp, typename GenericOp, typename Constraint> +static ASMJIT_NOINLINE void test_vecop_vv_constraint(JitContext& ctx, Variation variation = Variation{0}) { + constexpr uint32_t kW = byte_width_from_vec_width(kVecWidth); + + TestVVFunc compiled_apply = create_func_vv(ctx, kVecWidth, kOp, variation); + DataGenInt dg(kRandomSeed); + + VecOpInfo op_info = vec_op_info_vv(kOp); + + for (uint32_t iter = 0; iter < kTestIterCount; iter++) { + VecOverlay<kW> a {}; + VecOverlay<kW> observed {}; + VecOverlay<kW> expected {}; + + fill_random_data(dg, a, op_info.arg(0)); + Constraint::apply(a); + + compiled_apply(&observed, &a); + expected = GenericOp::apply(a); + + if (!vec_eq(observed, expected, op_info.ret())) { + test_vecop_vv_failed(kOp, variation, a, observed, expected, ctx.logger_content()); + } + } + + ctx.rt.release(compiled_apply); +} + +template<VecWidth kVecWidth, UniOpVV kOp, typename GenericOp> +static void test_vecop_vv(JitContext& ctx, Variation variation = Variation{0}) { + return test_vecop_vv_constraint<kVecWidth, kOp, GenericOp, ConstraintNone>(ctx, variation); +} + +// ujit::UniCompiler - Tests - SIMD - Integer Operations - VVI +// =========================================================== + +template<VecWidth kVecWidth, UniOpVVI kOp, typename GenericOp, typename Constraint> +static ASMJIT_NOINLINE void test_vecop_vvi_constraint(JitContext& ctx, uint32_t imm, Variation variation = Variation{0}) { + constexpr uint32_t kW = byte_width_from_vec_width(kVecWidth); + + TestVVFunc compiled_apply = create_func_vvi(ctx, kVecWidth, kOp, imm, variation); + DataGenInt dg(kRandomSeed); + + VecOpInfo op_info = vec_op_info_vvi(kOp); + + for (uint32_t iter = 0; iter < kTestIterCount; iter++) { + VecOverlay<kW> a {}; + VecOverlay<kW> observed {}; + VecOverlay<kW> expected {}; + + fill_random_data(dg, a, op_info.arg(0)); + Constraint::apply(a); + + compiled_apply(&observed, &a); + expected = GenericOp::apply(a, imm); + + if (!vec_eq(observed, expected, op_info.ret())) { + test_vecop_vvi_failed(kOp, variation, a, observed, expected, imm, ctx.logger_content()); + } + } + + ctx.rt.release(compiled_apply); +} + +template<VecWidth kVecWidth, UniOpVVI kOp, typename GenericOp> +static void test_vecop_vvi(JitContext& ctx, uint32_t imm, Variation variation = Variation{0}) { + return test_vecop_vvi_constraint<kVecWidth, kOp, GenericOp, ConstraintNone>(ctx, imm, variation); +} + +// ujit::UniCompiler - Tests - SIMD - Integer Operations - VVV +// =========================================================== + +template<VecWidth kVecWidth, UniOpVVV kOp, typename GenericOp, typename Constraint> +static ASMJIT_NOINLINE void test_vecop_vvv_constraint(JitContext& ctx, Variation variation = Variation{0}) { + constexpr uint32_t kW = byte_width_from_vec_width(kVecWidth); + + TestVVVFunc compiled_apply = create_func_vvv(ctx, kVecWidth, kOp, variation); + DataGenInt dg(kRandomSeed); + + VecOpInfo op_info = vec_op_info_vvv(kOp); + + for (uint32_t iter = 0; iter < kTestIterCount; iter++) { + VecOverlay<kW> a {}; + VecOverlay<kW> b {}; + VecOverlay<kW> observed {}; + VecOverlay<kW> expected {}; + + fill_random_data(dg, a, op_info.arg(0)); + fill_random_data(dg, b, op_info.arg(1)); + Constraint::apply(a); + Constraint::apply(b); + + compiled_apply(&observed, &a, &b); + expected = GenericOp::apply(a, b); + + if (!vec_eq(observed, expected, op_info.ret())) { + test_vecop_vvv_failed(kOp, variation, a, b, observed, expected, ctx.logger_content()); + } + } + + ctx.rt.release(compiled_apply); +} + +template<VecWidth kVecWidth, UniOpVVV kOp, typename GenericOp> +static void test_vecop_vvv(JitContext& ctx, Variation variation = Variation{0}) { + return test_vecop_vvv_constraint<kVecWidth, kOp, GenericOp, ConstraintNone>(ctx, variation); +} + +// ujit::UniCompiler - Tests - SIMD - Integer Operations - VVVI +// ============================================================ + +template<VecWidth kVecWidth, UniOpVVVI kOp, typename GenericOp, typename Constraint> +static ASMJIT_NOINLINE void test_vecop_vvvi_constraint(JitContext& ctx, uint32_t imm, Variation variation = Variation{0}) { + constexpr uint32_t kW = byte_width_from_vec_width(kVecWidth); + + TestVVVFunc compiled_apply = create_func_vvvi(ctx, kVecWidth, kOp, imm, variation); + DataGenInt dg(kRandomSeed); + + VecOpInfo op_info = vec_op_info_vvvi(kOp); + + for (uint32_t iter = 0; iter < kTestIterCount; iter++) { + VecOverlay<kW> a {}; + VecOverlay<kW> b {}; + VecOverlay<kW> observed {}; + VecOverlay<kW> expected {}; + + fill_random_data(dg, a, op_info.arg(0)); + fill_random_data(dg, b, op_info.arg(1)); + Constraint::apply(a); + Constraint::apply(b); + + compiled_apply(&observed, &a, &b); + expected = GenericOp::apply(a, b, imm); + + if (!vec_eq(observed, expected, op_info.ret())) { + test_vecop_vvvi_failed(kOp, variation, a, b, observed, expected, imm, ctx.logger_content()); + } + } + + ctx.rt.release(compiled_apply); +} + +template<VecWidth kVecWidth, UniOpVVVI kOp, typename GenericOp> +static void test_vecop_vvvi(JitContext& ctx, uint32_t imm, Variation variation = Variation{0}) { + return test_vecop_vvvi_constraint<kVecWidth, kOp, GenericOp, ConstraintNone>(ctx, imm, variation); +} + +// ujit::UniCompiler - Tests - SIMD - Integer Operations - VVVV +// ============================================================ + +template<VecWidth kVecWidth, UniOpVVVV kOp, typename GenericOp, typename Constraint> +static ASMJIT_NOINLINE void test_vecop_vvvv_constraint(JitContext& ctx, Variation variation = Variation{0}) { + constexpr uint32_t kW = byte_width_from_vec_width(kVecWidth); + + TestVVVVFunc compiled_apply = create_func_vvvv(ctx, kVecWidth, kOp, variation); + DataGenInt dg(kRandomSeed); + + VecOpInfo op_info = vec_op_info_vvvv(kOp); + + for (uint32_t iter = 0; iter < kTestIterCount; iter++) { + VecOverlay<kW> a {}; + VecOverlay<kW> b {}; + VecOverlay<kW> c {}; + VecOverlay<kW> observed {}; + VecOverlay<kW> expected {}; + + fill_random_data(dg, a, op_info.arg(0)); + fill_random_data(dg, b, op_info.arg(1)); + fill_random_data(dg, c, op_info.arg(2)); + Constraint::apply(a); + Constraint::apply(b); + Constraint::apply(c); + + compiled_apply(&observed, &a, &b, &c); + expected = GenericOp::apply(a, b, c); + + if (!vec_eq(observed, expected, op_info.ret())) { + test_vecop_vvvv_failed(kOp, variation, a, b, c, observed, expected, ctx.logger_content()); + } + } +} + +template<VecWidth kVecWidth, UniOpVVVV kOp, typename GenericOp> +static void test_vecop_vvvv(JitContext& ctx, Variation variation = Variation{0}) { + return test_vecop_vvvv_constraint<kVecWidth, kOp, GenericOp, ConstraintNone>(ctx, variation); +} + +// ujit::UniCompiler - Tests - SIMD - Runner +// ========================================= + +template<VecWidth kVecWidth> +static ASMJIT_NOINLINE void test_simd_ops(JitContext& ctx) { + // We need to know some behaviors in advance so we can select the right test function, + // so create a dummy compiler and extract the necessary information from it. + ScalarOpBehavior scalar_op_behavior {}; + FMAddOpBehavior fmadd_op_behavior {}; + FloatToIntOutsideRangeBehavior float_to_int_behavior {}; + + { + ctx.prepare(); + UniCompiler uc(&ctx.cc, ctx.features, ctx.cpu_hints); + + scalar_op_behavior = uc.scalar_op_behavior(); + fmadd_op_behavior = uc.fmadd_op_behavior(); + float_to_int_behavior = uc.float_to_int_outside_range_behavior(); + } + + bool valgrind_fma_bug = false; + +#if defined(ASMJIT_UJIT_X86) + // When running under valgrind there is a bug in its instrumentation of FMA SS/SD instructions. + // Instead of keeping the unaffected elements in the destination register they are cleared instead, + // which would cause test failures. So, detect whether we are running under Valgind that has this + // bug and avoid scalar FMA tests in that case. + if (fmadd_op_behavior != FMAddOpBehavior::kNoFMA) { + float a[4] = { 1, 2, 3, 4 }; + float b[4] = { 2, 4, 8, 1 }; + float c[4] = { 4, 7, 3, 9 }; + + float d[4] {}; + madd_fma_check_valgrind_bug(a, b, c, d); + + valgrind_fma_bug = d[1] == 0.0f; + } +#endif // ASMJIT_UJIT_X86 + + INFO(" Testing mov"); + { + test_vecop_vv<kVecWidth, UniOpVV::kMov, vec_op_mov>(ctx); + test_vecop_vv<kVecWidth, UniOpVV::kMovU64, vec_op_mov_u64>(ctx); + } + + INFO(" Testing broadcast"); + { + // Test all broadcasts - vector based, GP to vector, and memory to vector. + for (uint32_t v = 0; v < kNumVariationsVV_Broadcast; v++) { + test_vecop_vv<kVecWidth, UniOpVV::kBroadcastU8Z, vec_op_broadcast_u8>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kBroadcastU16Z, vec_op_broadcast_u16>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kBroadcastU8, vec_op_broadcast_u8>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kBroadcastU16, vec_op_broadcast_u16>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kBroadcastU32, vec_op_broadcast_u32>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kBroadcastU64, vec_op_broadcast_u64>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kBroadcastF32, vec_op_broadcast_u32>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kBroadcastF64, vec_op_broadcast_u64>(ctx, Variation{v}); + + test_vecop_vv<kVecWidth, UniOpVV::kBroadcastV128_U32, vec_op_broadcast_u128>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kBroadcastV128_U64, vec_op_broadcast_u128>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kBroadcastV128_F32, vec_op_broadcast_u128>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kBroadcastV128_F64, vec_op_broadcast_u128>(ctx, Variation{v}); + + if constexpr (kVecWidth > VecWidth::k256) { + test_vecop_vv<kVecWidth, UniOpVV::kBroadcastV256_U32, vec_op_broadcast_u256>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kBroadcastV256_U64, vec_op_broadcast_u256>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kBroadcastV256_F32, vec_op_broadcast_u256>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kBroadcastV256_F64, vec_op_broadcast_u256>(ctx, Variation{v}); + } + } + } + + INFO(" Testing abs (int)"); + { + for (uint32_t v = 0; v < kNumVariationsVV; v++) { + test_vecop_vv<kVecWidth, UniOpVV::kAbsI8, vec_op_abs<int8_t>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kAbsI16, vec_op_abs<int16_t>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kAbsI32, vec_op_abs<int32_t>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kAbsI64, vec_op_abs<int64_t>>(ctx, Variation{v}); + } + } + + INFO(" Testing not (int)"); + { + for (uint32_t v = 0; v < kNumVariationsVV; v++) { + test_vecop_vv<kVecWidth, UniOpVV::kNotU32, vec_op_not<uint32_t>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kNotU64, vec_op_not<uint64_t>>(ctx, Variation{v}); + } + } + + INFO(" Testing cvt (int)"); + { + for (uint32_t v = 0; v < kNumVariationsVV; v++) { + test_vecop_vv<kVecWidth, UniOpVV::kCvtI8LoToI16, vec_op_cvt_i8_lo_to_i16>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtI8HiToI16, vec_op_cvt_i8_hi_to_i16>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtU8LoToU16, vec_op_cvt_u8_lo_to_u16>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtU8HiToU16, vec_op_cvt_u8_hi_to_u16>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtI8ToI32, vec_op_cvt_i8_to_i32>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtU8ToU32, vec_op_cvt_u8_to_u32>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtI16LoToI32, vec_op_cvt_i16_lo_to_i32>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtI16HiToI32, vec_op_cvt_i16_hi_to_i32>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtU16LoToU32, vec_op_cvt_u16_lo_to_u32>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtU16HiToU32, vec_op_cvt_u16_hi_to_u32>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtI32LoToI64, vec_op_cvt_i32_lo_to_i64>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtI32HiToI64, vec_op_cvt_i32_hi_to_i64>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtU32LoToU64, vec_op_cvt_u32_lo_to_u64>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtU32HiToU64, vec_op_cvt_u32_hi_to_u64>(ctx, Variation{v}); + } + } + + INFO(" Testing abs (float)"); + { + for (uint32_t v = 0; v < kNumVariationsVV; v++) { + test_vecop_vv<kVecWidth, UniOpVV::kAbsF32, vec_op_fabs<float>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kAbsF64, vec_op_fabs<double>>(ctx, Variation{v}); + } + } + + INFO(" Testing not (float)"); + { + for (uint32_t v = 0; v < kNumVariationsVV; v++) { + test_vecop_vv<kVecWidth, UniOpVV::kNotF32, vec_op_not<uint32_t>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kNotF64, vec_op_not<uint64_t>>(ctx, Variation{v}); + } + } + + INFO(" Testing rounding (float)"); + { + for (uint32_t v = 0; v < kNumVariationsVV; v++) { + // Variation 2 means that the source operand is memory, which would ALWAYS zero the rest of the register. + if (scalar_op_behavior == ScalarOpBehavior::kZeroing || v == 2u) { + test_vecop_vv<kVecWidth, UniOpVV::kTruncF32S, scalar_op_trunc<ScalarOpBehavior::kZeroing, float>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kTruncF64S, scalar_op_trunc<ScalarOpBehavior::kZeroing, double>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kFloorF32S, scalar_op_floor<ScalarOpBehavior::kZeroing, float>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kFloorF64S, scalar_op_floor<ScalarOpBehavior::kZeroing, double>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCeilF32S, scalar_op_ceil<ScalarOpBehavior::kZeroing, float>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCeilF64S, scalar_op_ceil<ScalarOpBehavior::kZeroing, double>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kRoundEvenF32S, scalar_op_round_even<ScalarOpBehavior::kZeroing, float>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kRoundEvenF64S, scalar_op_round_even<ScalarOpBehavior::kZeroing, double>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kRoundHalfAwayF32S, scalar_op_round_half_away<ScalarOpBehavior::kZeroing, float>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kRoundHalfAwayF64S, scalar_op_round_half_away<ScalarOpBehavior::kZeroing, double>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kRoundHalfUpF32S, scalar_op_round_half_up<ScalarOpBehavior::kZeroing, float>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kRoundHalfUpF64S, scalar_op_round_half_up<ScalarOpBehavior::kZeroing, double>>(ctx, Variation{v}); + } + else { + test_vecop_vv<kVecWidth, UniOpVV::kTruncF32S, scalar_op_trunc<ScalarOpBehavior::kPreservingVec128, float>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kTruncF64S, scalar_op_trunc<ScalarOpBehavior::kPreservingVec128, double>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kFloorF32S, scalar_op_floor<ScalarOpBehavior::kPreservingVec128, float>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kFloorF64S, scalar_op_floor<ScalarOpBehavior::kPreservingVec128, double>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCeilF32S, scalar_op_ceil<ScalarOpBehavior::kPreservingVec128, float>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCeilF64S, scalar_op_ceil<ScalarOpBehavior::kPreservingVec128, double>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kRoundEvenF32S, scalar_op_round_even<ScalarOpBehavior::kPreservingVec128, float>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kRoundEvenF64S, scalar_op_round_even<ScalarOpBehavior::kPreservingVec128, double>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kRoundHalfAwayF32S, scalar_op_round_half_away<ScalarOpBehavior::kPreservingVec128, float>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kRoundHalfAwayF64S, scalar_op_round_half_away<ScalarOpBehavior::kPreservingVec128, double>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kRoundHalfUpF32S, scalar_op_round_half_up<ScalarOpBehavior::kPreservingVec128, float>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kRoundHalfUpF64S, scalar_op_round_half_up<ScalarOpBehavior::kPreservingVec128, double>>(ctx, Variation{v}); + } + + test_vecop_vv<kVecWidth, UniOpVV::kTruncF32, vec_op_trunc<float>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kTruncF64, vec_op_trunc<double>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kFloorF32, vec_op_floor<float>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kFloorF64, vec_op_floor<double>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCeilF32, vec_op_ceil<float>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCeilF64, vec_op_ceil<double>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kRoundEvenF32, vec_op_round_even<float>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kRoundEvenF64, vec_op_round_even<double>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kRoundHalfAwayF32, vec_op_round_half_away<float>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kRoundHalfAwayF64, vec_op_round_half_away<double>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kRoundHalfUpF32, vec_op_round_half_up<float>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kRoundHalfUpF64, vec_op_round_half_up<double>>(ctx, Variation{v}); + } + } + + INFO(" Testing rcp (float)"); + { + for (uint32_t v = 0; v < kNumVariationsVV; v++) { + test_vecop_vv<kVecWidth, UniOpVV::kRcpF32, vec_op_rcp<float>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kRcpF64, vec_op_rcp<double>>(ctx, Variation{v}); + } + } + + INFO(" Testing sqrt (float)"); + { + if (scalar_op_behavior == ScalarOpBehavior::kZeroing) { + test_vecop_vv<kVecWidth, UniOpVV::kSqrtF32S, scalar_op_sqrt<ScalarOpBehavior::kZeroing, float>>(ctx); + test_vecop_vv<kVecWidth, UniOpVV::kSqrtF64S, scalar_op_sqrt<ScalarOpBehavior::kZeroing, double>>(ctx); + } + else { + test_vecop_vv<kVecWidth, UniOpVV::kSqrtF32S, scalar_op_sqrt<ScalarOpBehavior::kPreservingVec128, float>>(ctx); + test_vecop_vv<kVecWidth, UniOpVV::kSqrtF64S, scalar_op_sqrt<ScalarOpBehavior::kPreservingVec128, double>>(ctx); + } + + for (uint32_t v = 0; v < kNumVariationsVV; v++) { + test_vecop_vv<kVecWidth, UniOpVV::kSqrtF32, vec_op_sqrt<float>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kSqrtF64, vec_op_sqrt<double>>(ctx, Variation{v}); + } + } + + INFO(" Testing cvt (float)"); + { + for (uint32_t v = 0; v < kNumVariationsVV; v++) { + // TODO: [JIT] Re-enable when the content of the remaining part of the register is formalized. + // test_vecop_vv<kVecWidth, UniOpVV::kCvtF32ToF64S, scalar_op_cvt_f32_to_f64>(ctx); + // test_vecop_vv<kVecWidth, UniOpVV::kCvtF64ToF32S, scalar_op_cvt_f64_to_f32>(ctx); + + test_vecop_vv<kVecWidth, UniOpVV::kCvtI32ToF32, vec_op_cvt_i32_to_f32>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtF32LoToF64, vec_op_cvt_f32_lo_to_f64>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtF32HiToF64, vec_op_cvt_f32_hi_to_f64>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtF64ToF32Lo, vec_op_cvt_f64_to_f32_lo>(ctx, Variation{0}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtF64ToF32Hi, vec_op_cvt_f64_to_f32_hi>(ctx, Variation{0}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtI32LoToF64, vec_op_cvt_i32_lo_to_f64>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtI32HiToF64, vec_op_cvt_i32_hi_to_f64>(ctx, Variation{v}); + + if (float_to_int_behavior == FloatToIntOutsideRangeBehavior::kSmallestValue) { + constexpr FloatToIntOutsideRangeBehavior behavior = FloatToIntOutsideRangeBehavior::kSmallestValue; + test_vecop_vv<kVecWidth, UniOpVV::kCvtTruncF32ToI32, vec_op_cvt_trunc_f32_to_i32<behavior>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtTruncF64ToI32Lo, vec_op_cvt_trunc_f64_to_i32_lo<behavior>>(ctx, Variation{0}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtTruncF64ToI32Hi, vec_op_cvt_trunc_f64_to_i32_hi<behavior>>(ctx, Variation{0}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtRoundF32ToI32, vec_op_cvt_round_f32_to_i32<behavior>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtRoundF64ToI32Lo, vec_op_cvt_round_f64_to_i32_lo<behavior>>(ctx, Variation{0}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtRoundF64ToI32Hi, vec_op_cvt_round_f64_to_i32_hi<behavior>>(ctx, Variation{0}); + } + else { + constexpr FloatToIntOutsideRangeBehavior behavior = FloatToIntOutsideRangeBehavior::kSaturatedValue; + test_vecop_vv<kVecWidth, UniOpVV::kCvtTruncF32ToI32, vec_op_cvt_trunc_f32_to_i32<behavior>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtTruncF64ToI32Lo, vec_op_cvt_trunc_f64_to_i32_lo<behavior>>(ctx, Variation{0}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtTruncF64ToI32Hi, vec_op_cvt_trunc_f64_to_i32_hi<behavior>>(ctx, Variation{0}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtRoundF32ToI32, vec_op_cvt_round_f32_to_i32<behavior>>(ctx, Variation{v}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtRoundF64ToI32Lo, vec_op_cvt_round_f64_to_i32_lo<behavior>>(ctx, Variation{0}); + test_vecop_vv<kVecWidth, UniOpVV::kCvtRoundF64ToI32Hi, vec_op_cvt_round_f64_to_i32_hi<behavior>>(ctx, Variation{0}); + } + } + } + + INFO(" Testing bit shift"); + { + for (uint32_t v = 0; v < kNumVariationsVVI; v++) { +/* + for (uint32_t i = 1; i < 8; i++) { + test_vecop_vvi<kVecWidth, UniOpVVI::kSllU8 , vec_op_slli<uint8_t>>(ctx, i, Variation{v}); + test_vecop_vvi<kVecWidth, UniOpVVI::kSrlU8 , vec_op_srli<uint8_t>>(ctx, i, Variation{v}); + test_vecop_vvi<kVecWidth, UniOpVVI::kSraI8 , vec_op_srai<int8_t>>(ctx, i, Variation{v}); + } +*/ + for (uint32_t i = 1; i < 16; i++) { + test_vecop_vvi<kVecWidth, UniOpVVI::kSllU16, vec_op_slli<uint16_t>>(ctx, i, Variation{v}); + test_vecop_vvi<kVecWidth, UniOpVVI::kSrlU16, vec_op_srli<uint16_t>>(ctx, i, Variation{v}); + test_vecop_vvi<kVecWidth, UniOpVVI::kSraI16, vec_op_srai<int16_t>>(ctx, i, Variation{v}); + } + + for (uint32_t i = 1; i < 32; i++) { + test_vecop_vvi<kVecWidth, UniOpVVI::kSllU32, vec_op_slli<uint32_t>>(ctx, i, Variation{v}); + test_vecop_vvi<kVecWidth, UniOpVVI::kSrlU32, vec_op_srli<uint32_t>>(ctx, i, Variation{v}); + test_vecop_vvi<kVecWidth, UniOpVVI::kSraI32, vec_op_srai<int32_t>>(ctx, i, Variation{v}); + } + + for (uint32_t i = 1; i < 64; i++) { + test_vecop_vvi<kVecWidth, UniOpVVI::kSllU64, vec_op_slli<uint64_t>>(ctx, i, Variation{v}); + test_vecop_vvi<kVecWidth, UniOpVVI::kSrlU64, vec_op_srli<uint64_t>>(ctx, i, Variation{v}); + test_vecop_vvi<kVecWidth, UniOpVVI::kSraI64, vec_op_srai<int64_t>>(ctx, i, Variation{v}); + } + } + } + + INFO(" Testing sllb_u128 & srlb_u128"); + { + for (uint32_t v = 0; v < kNumVariationsVVI; v++) { + for (uint32_t i = 1; i < 16; i++) { + test_vecop_vvi<kVecWidth, UniOpVVI::kSllbU128, vec_op_sllb_u128>(ctx, i, Variation{v}); + test_vecop_vvi<kVecWidth, UniOpVVI::kSrlbU128, vec_op_srlb_u128>(ctx, i, Variation{v}); + } + } + } + + INFO(" Testing swizzle_[lo|hi]_u16x4"); + { + for (uint32_t v = 0; v < kNumVariationsVVI; v++) { + for (uint32_t i = 0; i < 256; i++) { + uint32_t imm = swizzle((i >> 6) & 3, (i >> 4) & 3, (i >> 2) & 3, i & 3).value; + + test_vecop_vvi<kVecWidth, UniOpVVI::kSwizzleLoU16x4, vec_op_swizzle_lo_u16x4>(ctx, imm, Variation{v}); + test_vecop_vvi<kVecWidth, UniOpVVI::kSwizzleHiU16x4, vec_op_swizzle_hi_u16x4>(ctx, imm, Variation{v}); + test_vecop_vvi<kVecWidth, UniOpVVI::kSwizzleU16x4, vec_op_swizzle_u16>(ctx, imm, Variation{v}); + } + } + } + + INFO(" Testing swizzle_u32x4"); + { + for (uint32_t v = 0; v < kNumVariationsVVI; v++) { + for (uint32_t i = 0; i < 256; i++) { + uint32_t imm = swizzle((i >> 6) & 3, (i >> 4) & 3, (i >> 2) & 3, i & 3).value; + + test_vecop_vvi<kVecWidth, UniOpVVI::kSwizzleU32x4, vec_op_swizzle_u32x4>(ctx, imm, Variation{v}); + test_vecop_vvi<kVecWidth, UniOpVVI::kSwizzleF32x4, vec_op_swizzle_u32x4>(ctx, imm, Variation{v}); + } + } + } + + INFO(" Testing swizzle_u64x2"); + { + for (uint32_t v = 0; v < kNumVariationsVVI; v++) { + for (uint32_t i = 0; i < 4; i++) { + uint32_t imm = swizzle((i >> 1) & 1, i & 1).value; + test_vecop_vvi<kVecWidth, UniOpVVI::kSwizzleU64x2, vec_op_swizzle_u64x2>(ctx, imm, Variation{v}); + test_vecop_vvi<kVecWidth, UniOpVVI::kSwizzleF64x2, vec_op_swizzle_u64x2>(ctx, imm, Variation{v}); + } + } + } + + INFO(" Testing logical (int)"); + { + for (uint32_t v = 0; v < kNumVariationsVVV; v++) { + test_vecop_vvv<kVecWidth, UniOpVVV::kAndU32, vec_op_and<uint32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kAndU64, vec_op_and<uint64_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kOrU32, vec_op_or<uint32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kOrU64, vec_op_or<uint64_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kXorU32, vec_op_xor<uint32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kXorU64, vec_op_xor<uint64_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kAndnU32, vec_op_andn<uint32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kAndnU64, vec_op_andn<uint64_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kBicU32, vec_op_bic<uint32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kBicU64, vec_op_bic<uint64_t>>(ctx, Variation{v}); + } + } + + INFO(" Testing add / adds (int)"); + { + for (uint32_t v = 0; v < kNumVariationsVVV; v++) { + test_vecop_vvv<kVecWidth, UniOpVVV::kAddU8, vec_op_add<uint8_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kAddU16, vec_op_add<uint16_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kAddU32, vec_op_add<uint32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kAddU64, vec_op_add<uint64_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kAddsI8, vec_op_adds<int8_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kAddsI16, vec_op_adds<int16_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kAddsU8, vec_op_adds<uint8_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kAddsU16, vec_op_adds<uint16_t>>(ctx, Variation{v}); + } + } + + INFO(" Testing sub / subs (int)"); + { + for (uint32_t v = 0; v < kNumVariationsVVV; v++) { + test_vecop_vvv<kVecWidth, UniOpVVV::kSubU8, vec_op_sub<uint8_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kSubU16, vec_op_sub<uint16_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kSubU32, vec_op_sub<uint32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kSubU64, vec_op_sub<uint64_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kSubsI8, vec_op_subs<int8_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kSubsI16, vec_op_subs<int16_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kSubsU8, vec_op_subs<uint8_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kSubsU16, vec_op_subs<uint16_t>>(ctx, Variation{v}); + } + } + + INFO(" Testing mul (int)"); + { + for (uint32_t v = 0; v < kNumVariationsVVV; v++) { + test_vecop_vvv<kVecWidth, UniOpVVV::kMulU16, vec_op_mul<uint16_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMulU32, vec_op_mul<uint32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMulU64, vec_op_mul<uint64_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMulhI16, vec_op_mulhi<int16_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMulhU16, vec_op_mulhu<uint16_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMulU64_LoU32, vec_op_mul_u64_lo_u32>(ctx, Variation{v}); + } + } + + INFO(" Testing mhadd (int)"); + { + for (uint32_t v = 0; v < kNumVariationsVVV; v++) { + test_vecop_vvv<kVecWidth, UniOpVVV::kMHAddI16_I32, vec_op_mhadd_i16_i32>(ctx, Variation{v}); + } + } + + INFO(" Testing madd (int)"); + { + for (uint32_t v = 0; v < kNumVariationsVVVV; v++) { + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMAddU16, vec_op_madd<uint16_t>>(ctx, Variation{v}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMAddU32, vec_op_madd<uint32_t>>(ctx, Variation{v}); + } + } + + INFO(" Testing min / max (int)"); + { + for (uint32_t v = 0; v < kNumVariationsVVV; v++) { + test_vecop_vvv<kVecWidth, UniOpVVV::kMinI8, vec_op_min<int8_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMinI16, vec_op_min<int16_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMinI32, vec_op_min<int32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMinI64, vec_op_min<int64_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMinU8, vec_op_min<uint8_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMinU16, vec_op_min<uint16_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMinU32, vec_op_min<uint32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMinU64, vec_op_min<uint64_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMaxI8, vec_op_max<int8_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMaxI16, vec_op_max<int16_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMaxI32, vec_op_max<int32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMaxI64, vec_op_max<int64_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMaxU8, vec_op_max<uint8_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMaxU16, vec_op_max<uint16_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMaxU32, vec_op_max<uint32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMaxU64, vec_op_max<uint64_t>>(ctx, Variation{v}); + } + } + + INFO(" Testing cmp (int)"); + { + for (uint32_t v = 0; v < kNumVariationsVVV; v++) { + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpEqU8, vec_op_cmp_eq<uint8_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpEqU16, vec_op_cmp_eq<uint16_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpEqU32, vec_op_cmp_eq<uint32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpEqU64, vec_op_cmp_eq<uint64_t>>(ctx, Variation{v}); +/* + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpNeU8, vec_op_cmp_ne<uint8_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpNeU16, vec_op_cmp_ne<uint16_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpNeU32, vec_op_cmp_ne<uint32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpNeU64, vec_op_cmp_ne<uint64_t>>(ctx, Variation{v}); +*/ + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpGtI8, vec_op_cmp_gt<int8_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpGtI16, vec_op_cmp_gt<int16_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpGtI32, vec_op_cmp_gt<int32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpGtI64, vec_op_cmp_gt<int64_t>>(ctx, Variation{v}); + + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpGtU8, vec_op_cmp_gt<uint8_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpGtU16, vec_op_cmp_gt<uint16_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpGtU32, vec_op_cmp_gt<uint32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpGtU64, vec_op_cmp_gt<uint64_t>>(ctx, Variation{v}); + + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpGeI8, vec_op_cmp_ge<int8_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpGeI16, vec_op_cmp_ge<int16_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpGeI32, vec_op_cmp_ge<int32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpGeI64, vec_op_cmp_ge<int64_t>>(ctx, Variation{v}); + + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpGeU8, vec_op_cmp_ge<uint8_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpGeU16, vec_op_cmp_ge<uint16_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpGeU32, vec_op_cmp_ge<uint32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpGeU64, vec_op_cmp_ge<uint64_t>>(ctx, Variation{v}); + + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpLtI8, vec_op_cmp_lt<int8_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpLtI16, vec_op_cmp_lt<int16_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpLtI32, vec_op_cmp_lt<int32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpLtI64, vec_op_cmp_lt<int64_t>>(ctx, Variation{v}); + + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpLtU8, vec_op_cmp_lt<uint8_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpLtU16, vec_op_cmp_lt<uint16_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpLtU32, vec_op_cmp_lt<uint32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpLtU64, vec_op_cmp_lt<uint64_t>>(ctx, Variation{v}); + + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpLeI8, vec_op_cmp_le<int8_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpLeI16, vec_op_cmp_le<int16_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpLeI32, vec_op_cmp_le<int32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpLeI64, vec_op_cmp_le<int64_t>>(ctx, Variation{v}); + + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpLeU8, vec_op_cmp_le<uint8_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpLeU16, vec_op_cmp_le<uint16_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpLeU32, vec_op_cmp_le<uint32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpLeU64, vec_op_cmp_le<uint64_t>>(ctx, Variation{v}); + } + } + + INFO(" Testing logical (float)"); + { + for (uint32_t v = 0; v < kNumVariationsVVV; v++) { + test_vecop_vvv<kVecWidth, UniOpVVV::kAndF32, vec_op_and<uint32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kAndF64, vec_op_and<uint64_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kOrF32, vec_op_or<uint32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kOrF64, vec_op_or<uint64_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kXorF32, vec_op_xor<uint32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kXorF64, vec_op_xor<uint64_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kAndnF32, vec_op_andn<uint32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kAndnF64, vec_op_andn<uint64_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kBicF32, vec_op_bic<uint32_t>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kBicF64, vec_op_bic<uint64_t>>(ctx, Variation{v}); + } + } + + INFO(" Testing arithmetic (float)"); + { + if (scalar_op_behavior == ScalarOpBehavior::kZeroing) { + test_vecop_vvv<kVecWidth, UniOpVVV::kAddF32S, scalar_op_fadd<ScalarOpBehavior::kZeroing, float>>(ctx); + test_vecop_vvv<kVecWidth, UniOpVVV::kAddF64S, scalar_op_fadd<ScalarOpBehavior::kZeroing, double>>(ctx); + test_vecop_vvv<kVecWidth, UniOpVVV::kSubF32S, scalar_op_fsub<ScalarOpBehavior::kZeroing, float>>(ctx); + test_vecop_vvv<kVecWidth, UniOpVVV::kSubF64S, scalar_op_fsub<ScalarOpBehavior::kZeroing, double>>(ctx); + test_vecop_vvv<kVecWidth, UniOpVVV::kMulF32S, scalar_op_fmul<ScalarOpBehavior::kZeroing, float>>(ctx); + test_vecop_vvv<kVecWidth, UniOpVVV::kMulF64S, scalar_op_fmul<ScalarOpBehavior::kZeroing, double>>(ctx); + test_vecop_vvv<kVecWidth, UniOpVVV::kDivF32S, scalar_op_fdiv<ScalarOpBehavior::kZeroing, float>>(ctx); + test_vecop_vvv<kVecWidth, UniOpVVV::kDivF64S, scalar_op_fdiv<ScalarOpBehavior::kZeroing, double>>(ctx); + } + else { + test_vecop_vvv<kVecWidth, UniOpVVV::kAddF32S, scalar_op_fadd<ScalarOpBehavior::kPreservingVec128, float>>(ctx); + test_vecop_vvv<kVecWidth, UniOpVVV::kAddF64S, scalar_op_fadd<ScalarOpBehavior::kPreservingVec128, double>>(ctx); + test_vecop_vvv<kVecWidth, UniOpVVV::kSubF32S, scalar_op_fsub<ScalarOpBehavior::kPreservingVec128, float>>(ctx); + test_vecop_vvv<kVecWidth, UniOpVVV::kSubF64S, scalar_op_fsub<ScalarOpBehavior::kPreservingVec128, double>>(ctx); + test_vecop_vvv<kVecWidth, UniOpVVV::kMulF32S, scalar_op_fmul<ScalarOpBehavior::kPreservingVec128, float>>(ctx); + test_vecop_vvv<kVecWidth, UniOpVVV::kMulF64S, scalar_op_fmul<ScalarOpBehavior::kPreservingVec128, double>>(ctx); + test_vecop_vvv<kVecWidth, UniOpVVV::kDivF32S, scalar_op_fdiv<ScalarOpBehavior::kPreservingVec128, float>>(ctx); + test_vecop_vvv<kVecWidth, UniOpVVV::kDivF64S, scalar_op_fdiv<ScalarOpBehavior::kPreservingVec128, double>>(ctx); + } + + for (uint32_t v = 0; v < kNumVariationsVVV; v++) { + test_vecop_vvv<kVecWidth, UniOpVVV::kAddF32, vec_op_fadd<float>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kAddF64, vec_op_fadd<double>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kSubF32, vec_op_fsub<float>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kSubF64, vec_op_fsub<double>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMulF32, vec_op_fmul<float>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMulF64, vec_op_fmul<double>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kDivF32, vec_op_fdiv<float>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kDivF64, vec_op_fdiv<double>>(ctx, Variation{v}); + } + } + + if (fmadd_op_behavior == FMAddOpBehavior::kNoFMA) { + INFO(" Testing madd (no-fma) (float)"); + { + if (scalar_op_behavior == ScalarOpBehavior::kZeroing) { + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMAddF32S, scalar_op_fmadd_nofma<ScalarOpBehavior::kZeroing, float>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMAddF64S, scalar_op_fmadd_nofma<ScalarOpBehavior::kZeroing, double>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMSubF32S, scalar_op_fmsub_nofma<ScalarOpBehavior::kZeroing, float>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMSubF64S, scalar_op_fmsub_nofma<ScalarOpBehavior::kZeroing, double>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMAddF32S, scalar_op_fnmadd_nofma<ScalarOpBehavior::kZeroing, float>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMAddF64S, scalar_op_fnmadd_nofma<ScalarOpBehavior::kZeroing, double>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMSubF32S, scalar_op_fnmsub_nofma<ScalarOpBehavior::kZeroing, float>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMSubF64S, scalar_op_fnmsub_nofma<ScalarOpBehavior::kZeroing, double>>(ctx, Variation{0}); + } + else { + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMAddF32S, scalar_op_fmadd_nofma<ScalarOpBehavior::kPreservingVec128, float>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMAddF64S, scalar_op_fmadd_nofma<ScalarOpBehavior::kPreservingVec128, double>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMSubF32S, scalar_op_fmsub_nofma<ScalarOpBehavior::kPreservingVec128, float>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMSubF64S, scalar_op_fmsub_nofma<ScalarOpBehavior::kPreservingVec128, double>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMAddF32S, scalar_op_fnmadd_nofma<ScalarOpBehavior::kPreservingVec128, float>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMAddF64S, scalar_op_fnmadd_nofma<ScalarOpBehavior::kPreservingVec128, double>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMSubF32S, scalar_op_fnmsub_nofma<ScalarOpBehavior::kPreservingVec128, float>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMSubF64S, scalar_op_fnmsub_nofma<ScalarOpBehavior::kPreservingVec128, double>>(ctx, Variation{0}); + } + + for (uint32_t v = 0; v < kNumVariationsVVVV; v++) { + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMAddF32, vec_op_fmadd_nofma<float>>(ctx, Variation{v}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMAddF64, vec_op_fmadd_nofma<double>>(ctx, Variation{v}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMSubF32, vec_op_fmsub_nofma<float>>(ctx, Variation{v}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMSubF64, vec_op_fmsub_nofma<double>>(ctx, Variation{v}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMAddF32, vec_op_fnmadd_nofma<float>>(ctx, Variation{v}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMAddF64, vec_op_fnmadd_nofma<double>>(ctx, Variation{v}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMSubF32, vec_op_fnmsub_nofma<float>>(ctx, Variation{v}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMSubF64, vec_op_fnmsub_nofma<double>>(ctx, Variation{v}); + } + } + } + else { + INFO(" Testing madd (fma) (float)"); + { + if (valgrind_fma_bug) { + INFO(" (scalar FMA tests ignored due to a Valgrind bug!)"); + } + else { + if (scalar_op_behavior == ScalarOpBehavior::kZeroing) { + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMAddF32S, scalar_op_fmadd_fma<ScalarOpBehavior::kZeroing, float>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMAddF64S, scalar_op_fmadd_fma<ScalarOpBehavior::kZeroing, double>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMSubF32S, scalar_op_fmsub_fma<ScalarOpBehavior::kZeroing, float>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMSubF64S, scalar_op_fmsub_fma<ScalarOpBehavior::kZeroing, double>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMAddF32S, scalar_op_fnmadd_fma<ScalarOpBehavior::kZeroing, float>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMAddF64S, scalar_op_fnmadd_fma<ScalarOpBehavior::kZeroing, double>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMSubF32S, scalar_op_fnmsub_fma<ScalarOpBehavior::kZeroing, float>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMSubF64S, scalar_op_fnmsub_fma<ScalarOpBehavior::kZeroing, double>>(ctx, Variation{0}); + } + else { + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMAddF32S, scalar_op_fmadd_fma<ScalarOpBehavior::kPreservingVec128, float>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMAddF64S, scalar_op_fmadd_fma<ScalarOpBehavior::kPreservingVec128, double>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMSubF32S, scalar_op_fmsub_fma<ScalarOpBehavior::kPreservingVec128, float>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMSubF64S, scalar_op_fmsub_fma<ScalarOpBehavior::kPreservingVec128, double>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMAddF32S, scalar_op_fnmadd_fma<ScalarOpBehavior::kPreservingVec128, float>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMAddF64S, scalar_op_fnmadd_fma<ScalarOpBehavior::kPreservingVec128, double>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMSubF32S, scalar_op_fnmsub_fma<ScalarOpBehavior::kPreservingVec128, float>>(ctx, Variation{0}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMSubF64S, scalar_op_fnmsub_fma<ScalarOpBehavior::kPreservingVec128, double>>(ctx, Variation{0}); + } + } + + for (uint32_t v = 0; v < kNumVariationsVVVV; v++) { + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMAddF32, vec_op_fmadd_fma<float>>(ctx, Variation{v}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMAddF64, vec_op_fmadd_fma<double>>(ctx, Variation{v}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMSubF32, vec_op_fmsub_fma<float>>(ctx, Variation{v}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kMSubF64, vec_op_fmsub_fma<double>>(ctx, Variation{v}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMAddF32, vec_op_fnmadd_fma<float>>(ctx, Variation{v}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMAddF64, vec_op_fnmadd_fma<double>>(ctx, Variation{v}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMSubF32, vec_op_fnmsub_fma<float>>(ctx, Variation{v}); + test_vecop_vvvv<kVecWidth, UniOpVVVV::kNMSubF64, vec_op_fnmsub_fma<double>>(ctx, Variation{v}); + } + } + } + + INFO(" Testing min / max (float)"); + { +#if defined(ASMJIT_UJIT_X86) + test_vecop_vvv<kVecWidth, UniOpVVV::kMinF32S, scalar_op_fmin_ternary<ScalarOpBehavior::kPreservingVec128, float>>(ctx); + test_vecop_vvv<kVecWidth, UniOpVVV::kMinF64S, scalar_op_fmin_ternary<ScalarOpBehavior::kPreservingVec128, double>>(ctx); + test_vecop_vvv<kVecWidth, UniOpVVV::kMaxF32S, scalar_op_fmax_ternary<ScalarOpBehavior::kPreservingVec128, float>>(ctx); + test_vecop_vvv<kVecWidth, UniOpVVV::kMaxF64S, scalar_op_fmax_ternary<ScalarOpBehavior::kPreservingVec128, double>>(ctx); + + for (uint32_t v = 0; v < kNumVariationsVVV; v++) { + test_vecop_vvv<kVecWidth, UniOpVVV::kMinF32, vec_op_fmin_ternary<float>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMinF64, vec_op_fmin_ternary<double>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMaxF32, vec_op_fmax_ternary<float>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMaxF64, vec_op_fmax_ternary<double>>(ctx, Variation{v}); + } +#endif + +#if defined(ASMJIT_UJIT_AARCH64) + test_vecop_vvv<kVecWidth, UniOpVVV::kMinF32S, scalar_op_fmin_finite<ScalarOpBehavior::kZeroing, float>>(ctx); + test_vecop_vvv<kVecWidth, UniOpVVV::kMinF64S, scalar_op_fmin_finite<ScalarOpBehavior::kZeroing, double>>(ctx); + test_vecop_vvv<kVecWidth, UniOpVVV::kMaxF32S, scalar_op_fmax_finite<ScalarOpBehavior::kZeroing, float>>(ctx); + test_vecop_vvv<kVecWidth, UniOpVVV::kMaxF64S, scalar_op_fmax_finite<ScalarOpBehavior::kZeroing, double>>(ctx); + + for (uint32_t v = 0; v < kNumVariationsVVV; v++) { + test_vecop_vvv<kVecWidth, UniOpVVV::kMinF32, vec_op_fmin_finite<float>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMinF64, vec_op_fmin_finite<double>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMaxF32, vec_op_fmax_finite<float>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kMaxF64, vec_op_fmax_finite<double>>(ctx, Variation{v}); + } +#endif + } + + INFO(" Testing cmp (float)"); + { + for (uint32_t v = 0; v < kNumVariationsVVV; v++) { + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpEqF32, vec_op_fcmpo_eq<float>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpEqF64, vec_op_fcmpo_eq<double>>(ctx, Variation{v}); + + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpNeF32, vec_op_fcmpu_ne<float>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpNeF64, vec_op_fcmpu_ne<double>>(ctx, Variation{v}); + + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpGtF32, vec_op_fcmpo_gt<float>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpGtF64, vec_op_fcmpo_gt<double>>(ctx, Variation{v}); + + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpGeF32, vec_op_fcmpo_ge<float>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpGeF64, vec_op_fcmpo_ge<double>>(ctx, Variation{v}); + + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpLtF32, vec_op_fcmpo_lt<float>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpLtF64, vec_op_fcmpo_lt<double>>(ctx, Variation{v}); + + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpLeF32, vec_op_fcmpo_le<float>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpLeF64, vec_op_fcmpo_le<double>>(ctx, Variation{v}); + + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpOrdF32, vec_op_fcmp_ord<float>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpOrdF64, vec_op_fcmp_ord<double>>(ctx, Variation{v}); + + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpUnordF32, vec_op_fcmp_unord<float>>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCmpUnordF64, vec_op_fcmp_unord<double>>(ctx, Variation{v}); + } + } + + INFO(" Testing hadd (float)"); + { + for (uint32_t v = 0; v < kNumVariationsVVV; v++) { + test_vecop_vvv<kVecWidth, UniOpVVV::kHAddF64, vec_op_hadd_f64>(ctx, Variation{v}); + } + } + + INFO(" Testing combine"); + { + for (uint32_t v = 0; v < kNumVariationsVVV; v++) { + test_vecop_vvv<kVecWidth, UniOpVVV::kCombineLoHiU64, vec_op_combine_lo_hi_u64>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCombineLoHiF64, vec_op_combine_lo_hi_u64>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCombineHiLoU64, vec_op_combine_hi_lo_u64>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kCombineHiLoF64, vec_op_combine_hi_lo_u64>(ctx, Variation{v}); + } + } + + INFO(" Testing interleave"); + { + for (uint32_t v = 0; v < kNumVariationsVVV; v++) { + test_vecop_vvv<kVecWidth, UniOpVVV::kInterleaveLoU8, vec_op_interleave_lo_u8>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kInterleaveHiU8, vec_op_interleave_hi_u8>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kInterleaveLoU16, vec_op_interleave_lo_u16>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kInterleaveHiU16, vec_op_interleave_hi_u16>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kInterleaveLoU32, vec_op_interleave_lo_u32>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kInterleaveHiU32, vec_op_interleave_hi_u32>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kInterleaveLoU64, vec_op_interleave_lo_u64>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kInterleaveHiU64, vec_op_interleave_hi_u64>(ctx, Variation{v}); + } + } + + INFO(" Testing packs"); + { + for (uint32_t v = 0; v < kNumVariationsVVV; v++) { + test_vecop_vvv<kVecWidth, UniOpVVV::kPacksI16_I8, vec_op_packs_i16_i8>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kPacksI16_U8, vec_op_packs_i16_u8>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kPacksI32_I16, vec_op_packs_i32_i16>(ctx, Variation{v}); + test_vecop_vvv<kVecWidth, UniOpVVV::kPacksI32_U16, vec_op_packs_i32_u16>(ctx, Variation{v}); + } + } + + INFO(" Testing alignr_u128"); + { + for (uint32_t v = 0; v < kNumVariationsVVVI; v++) { + for (uint32_t i = 1; i < 16; i++) { + test_vecop_vvvi<kVecWidth, UniOpVVVI::kAlignr_U128, vec_op_alignr_u128>(ctx, i, Variation{v}); + } + } + } + + INFO(" Testing interleave_shuffle"); + { + for (uint32_t v = 0; v < kNumVariationsVVVI; v++) { + for (uint32_t i = 0; i < 256; i++) { + uint32_t imm = swizzle((i >> 6) & 3, (i >> 4) & 3, (i >> 2) & 3, i & 3).value; + + test_vecop_vvvi<kVecWidth, UniOpVVVI::kInterleaveShuffleU32x4, vec_op_interleave_shuffle_u32x4>(ctx, imm, Variation{v}); + test_vecop_vvvi<kVecWidth, UniOpVVVI::kInterleaveShuffleF32x4, vec_op_interleave_shuffle_u32x4>(ctx, imm, Variation{v}); + } + + for (uint32_t i = 0; i < 4; i++) { + uint32_t imm = swizzle((i >> 1) & 1, i & 1).value; + + test_vecop_vvvi<kVecWidth, UniOpVVVI::kInterleaveShuffleU64x2, vec_op_interleave_shuffle_u64x2>(ctx, imm, Variation{v}); + test_vecop_vvvi<kVecWidth, UniOpVVVI::kInterleaveShuffleF64x2, vec_op_interleave_shuffle_u64x2>(ctx, imm, Variation{v}); + } + } + } +} + +static void test_gp_ops(JitContext& ctx) { + test_cond_ops(ctx); + test_m_ops(ctx); + test_rm_ops(ctx); + test_mr_ops(ctx); + test_rr_ops(ctx); + test_rrr_ops(ctx); +} + +#if defined(ASMJIT_UJIT_X86) +static void dump_feature_list(String& out, const CpuFeatures& features) { +#if !defined(ASMJIT_NO_LOGGING) + CpuFeatures::Iterator it = features.iterator(); + bool first = true; + while (it.has_next()) { + size_t feature_id = it.next(); + if (!first) { + out.append(' '); + } + Formatter::format_feature(out, Arch::kHost, uint32_t(feature_id)); + first = false; + } +#else + Support::maybe_unused(features); + out.append("<ASMJIT_NO_LOGGING>"); +#endif +} + +static void test_x86_ops(JitContext& ctx, const CpuFeatures& host_features) { + using Ext = CpuFeatures::X86; + + { + String s; + dump_feature_list(s, host_features); + INFO("Available CPU features: %s", s.data()); + } + + // Features that must always be available; + CpuFeatures base; + base.add(Ext::kI486, Ext::kCMOV, Ext::kCMPXCHG8B, Ext::kFPU, Ext::kSSE, Ext::kSSE2); + + // To verify that JIT implements ALL features with ALL possible CPU flags, we use profiles to select features + // that the JIT compiler will be allowed to use. The features are gradually increased similarly to how new CPU + // generations introduced them. We cannot cover ALL possible CPUs, but that's not even necessary as we test + // individual operations where instructions can be selected on the features available. + + // GP variations. + { + CpuFeatures profiles[4] {}; + profiles[0] = base; + + profiles[1] = profiles[0]; + profiles[1].add(Ext::kADX, Ext::kBMI); + + profiles[2] = profiles[1]; + profiles[2].add(Ext::kBMI2, Ext::kLZCNT, Ext::kMOVBE, Ext::kPOPCNT); + + profiles[3] = host_features; + + bool first = true; + CpuFeatures last_filtered; + + for (const CpuFeatures& profile : profiles) { + CpuFeatures filtered = profile; + + for (uint32_t i = 0; i < CpuFeatures::kNumBitWords; i++) { + filtered.data()._bits[i] &= host_features.data()._bits[i]; + } + + if (!first && filtered == last_filtered) { + continue; + } + + String s; + if (filtered == host_features) { + s.assign("[ALL]"); + } + else { + dump_feature_list(s, filtered); + } + + ctx.features = filtered; + + INFO("Testing JIT compiler GP ops with [%s]", s.data()); + test_gp_ops(ctx); + + first = false; + last_filtered = filtered; + } + } + + // SIMD variations covering SSE2+, AVX+, and AVX512+ cases. + { + CpuFeatures profiles[15] {}; + profiles[0] = base; + + profiles[1] = profiles[0]; + profiles[1].add(Ext::kSSE3); + + profiles[2] = profiles[1]; + profiles[2].add(Ext::kSSSE3); + + profiles[3] = profiles[2]; + profiles[3].add(Ext::kSSE4_1); + + profiles[4] = profiles[3]; + profiles[4].add(Ext::kSSE4_2, Ext::kADX, Ext::kBMI, Ext::kBMI2, Ext::kLZCNT, Ext::kMOVBE, Ext::kPOPCNT); + + profiles[5] = profiles[4]; + profiles[5].add(Ext::kPCLMULQDQ); + + profiles[6] = profiles[5]; + profiles[6].add(Ext::kAVX); + + profiles[7] = profiles[6]; + profiles[7].add(Ext::kAVX2); + + profiles[8] = profiles[7]; + profiles[8].add(Ext::kF16C, Ext::kFMA, Ext::kVAES, Ext::kVPCLMULQDQ); + + profiles[9] = profiles[8]; + profiles[9].add(Ext::kAVX_IFMA, Ext::kAVX_NE_CONVERT, Ext::kAVX_VNNI, Ext::kAVX_VNNI_INT8, Ext::kAVX_VNNI_INT16); + + // We start deliberately from a profile that doesn't contains AVX_xxx + // extensions as these didn't exist when the first AVX512 CPUs were shipped. + profiles[10] = profiles[8]; + profiles[10].add(Ext::kAVX512_F, Ext::kAVX512_BW, Ext::kAVX512_DQ, Ext::kAVX512_CD, Ext::kAVX512_VL); + + profiles[11] = profiles[10]; + profiles[11].add(Ext::kAVX512_IFMA, Ext::kAVX512_VBMI); + + profiles[12] = profiles[11]; + profiles[12].add(Ext::kAVX512_BITALG, Ext::kAVX512_VBMI2, Ext::kAVX512_VNNI, Ext::kAVX512_VPOPCNTDQ); + + profiles[13] = profiles[12]; + profiles[13].add(Ext::kAVX512_BF16, Ext::kAVX512_FP16); + + profiles[14] = host_features; + + bool first = true; + CpuFeatures last_filtered; + + for (const CpuFeatures& profile : profiles) { + CpuFeatures filtered = profile; + + for (uint32_t i = 0; i < CpuFeatures::kNumBitWords; i++) { + filtered.data()._bits[i] &= host_features.data()._bits[i]; + } + + if (!first && filtered == last_filtered) { + continue; + } + + String s; + if (filtered == host_features) { + s.assign("[ALL]"); + } + else { + dump_feature_list(s, filtered); + } + + ctx.features = filtered; + + INFO("Testing JIT compiler 128-bit SIMD ops with [%s]", s.data()); + test_simd_ops<VecWidth::k128>(ctx); + + if (filtered.x86().has_avx2()) { + INFO("Testing JIT compiler 256-bit SIMD ops with [%s]", s.data()); + test_simd_ops<VecWidth::k256>(ctx); + } + + if (filtered.x86().has_avx512_f()) { + INFO("Testing JIT compiler 512-bit SIMD ops with [%s]", s.data()); + test_simd_ops<VecWidth::k512>(ctx); + } + + first = false; + last_filtered = filtered; + } + } +} +#endif // ASMJIT_UJIT_X86 + +#if defined(ASMJIT_UJIT_AARCH64) +static void test_a64_ops(JitContext& ctx, const CpuFeatures& host_features) { + ctx.features = host_features; + + test_gp_ops(ctx); + test_simd_ops<VecWidth::k128>(ctx); +} +#endif // ASMJIT_UJIT_AARCH64 + +} // {UniCompilerTests} + +UNIT(unicompiler) { + UniCompilerTests::JitContext ctx; + asmjit::CpuFeatures host_features = asmjit::CpuInfo::host().features(); + +#if defined(ASMJIT_UJIT_X86) + UniCompilerTests::test_x86_ops(ctx, host_features); +#elif defined(ASMJIT_UJIT_AARCH64) + UniCompilerTests::test_a64_ops(ctx, host_features); +#endif +} + +int main(int argc, const char* argv[]) { + print_app_info(); + return BrokenAPI::run(argc, argv); +} + +#else + +int main() { + print_app_info(); + printf("!! This test is disabled: <ASMJIT_NO_[U]JIT> or unsuitable arvhitecture !!\n"); + return 0; +} + +#endif // !ASMJIT_NO_UJIT && !ASMJIT_NO_JIT |
