summaryrefslogtreecommitdiffstatshomepage
path: root/3rdparty/asmjit/src/asmjit/x86/x86assembler.cpp
diff options
context:
space:
mode:
Diffstat (limited to '3rdparty/asmjit/src/asmjit/x86/x86assembler.cpp')
-rw-r--r--3rdparty/asmjit/src/asmjit/x86/x86assembler.cpp1638
1 files changed, 991 insertions, 647 deletions
diff --git a/3rdparty/asmjit/src/asmjit/x86/x86assembler.cpp b/3rdparty/asmjit/src/asmjit/x86/x86assembler.cpp
index 879fb2f0151..f11fea00238 100644
--- a/3rdparty/asmjit/src/asmjit/x86/x86assembler.cpp
+++ b/3rdparty/asmjit/src/asmjit/x86/x86assembler.cpp
@@ -1,31 +1,13 @@
-// AsmJit - Machine code generation for C++
+// This file is part of AsmJit project <https://asmjit.com>
//
-// * Official AsmJit Home Page: https://asmjit.com
-// * Official Github Repository: https://github.com/asmjit/asmjit
-//
-// Copyright (c) 2008-2020 The AsmJit Authors
-//
-// This software is provided 'as-is', without any express or implied
-// warranty. In no event will the authors be held liable for any damages
-// arising from the use of this software.
-//
-// Permission is granted to anyone to use this software for any purpose,
-// including commercial applications, and to alter it and redistribute it
-// freely, subject to the following restrictions:
-//
-// 1. The origin of this software must not be misrepresented; you must not
-// claim that you wrote the original software. If you use this software
-// in a product, an acknowledgment in the product documentation would be
-// appreciated but is not required.
-// 2. Altered source versions must be plainly marked as such, and must not be
-// misrepresented as being the original software.
-// 3. This notice may not be removed or altered from any source distribution.
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
#include "../core/api-build_p.h"
-#ifdef ASMJIT_BUILD_X86
+#if !defined(ASMJIT_NO_X86)
#include "../core/assembler.h"
-#include "../core/codebufferwriter_p.h"
+#include "../core/codewriter_p.h"
#include "../core/cpuinfo.h"
#include "../core/emitterutils_p.h"
#include "../core/formatter.h"
@@ -33,21 +15,18 @@
#include "../core/misc_p.h"
#include "../core/support.h"
#include "../x86/x86assembler.h"
+#include "../x86/x86emithelper_p.h"
+#include "../x86/x86instapi_p.h"
#include "../x86/x86instdb_p.h"
#include "../x86/x86formatter_p.h"
#include "../x86/x86opcode_p.h"
ASMJIT_BEGIN_SUB_NAMESPACE(x86)
-// ============================================================================
-// [TypeDefs]
-// ============================================================================
-
typedef Support::FastUInt8 FastUInt8;
-// ============================================================================
-// [Constants]
-// ============================================================================
+// x86::Assembler - Constants
+// ==========================
//! X86 bytes used to encode important prefixes.
enum X86Byte : uint32_t {
@@ -78,13 +57,13 @@ enum X86Byte : uint32_t {
//! 4-byte EVEX prefix:
//! - `[0]` - `0x62`.
- //! - `[1]` - Payload0 or `P[ 7: 0]` - `[R X B R' 0 0 m m]`.
+ //! - `[1]` - Payload0 or `P[ 7: 0]` - `[R X B R' 0 m m m]`.
//! - `[2]` - Payload1 or `P[15: 8]` - `[W v v v v 1 p p]`.
//! - `[3]` - Payload2 or `P[23:16]` - `[z L' L b V' a a a]`.
//!
//! Payload:
- //! - `P[ 1: 0]` - OPCODE: EVEX.mmmmm, only lowest 2 bits [1:0] used.
- //! - `P[ 3: 2]` - ______: Must be 0.
+ //! - `P[ 2: 0]` - OPCODE: EVEX.mmmmm, only lowest 3 bits [2:0] used.
+ //! - `P[ 3]` - ______: Must be 0.
//! - `P[ 4]` - REG-ID: EVEX.R' - 5th bit of 'RRRRR'.
//! - `P[ 5]` - REG-ID: EVEX.B - 4th bit of 'BBBBB'.
//! - `P[ 6]` - REG-ID: EVEX.X - 5th bit of 'BBBBB' or 4th bit of 'XXXX' (with SIB).
@@ -166,15 +145,13 @@ static const uint32_t x86OpcodePopSReg[8] = {
Opcode::k000F00 | 0xA9 // Pop GS.
};
-// ============================================================================
-// [asmjit::X86MemInfo | X86VEXPrefix | X86LLByRegType | X86CDisp8Table]
+// x86::Assembler - X86MemInfo | X86VEXPrefix | X86LLByRegType | X86CDisp8Table
// ============================================================================
//! Memory operand's info bits.
//!
-//! A lookup table that contains various information based on the BASE and INDEX
-//! information of a memory operand. This is much better and safer than playing
-//! with IFs in the code and can check for errors must faster and better.
+//! A lookup table that contains various information based on the BASE and INDEX information of a memory operand. This
+//! is much better and safer than playing with IFs in the code and can check for errors must faster and better.
enum X86MemInfo_Enum {
kX86MemInfo_0 = 0x00,
@@ -191,79 +168,72 @@ enum X86MemInfo_Enum {
template<uint32_t X>
struct X86MemInfo_T {
- enum {
+ enum : uint32_t {
B = (X ) & 0x1F,
I = (X >> 5) & 0x1F,
- kBase = (B >= Reg::kTypeGpw && B <= Reg::kTypeGpq ) ? kX86MemInfo_BaseGp :
- (B == Reg::kTypeRip ) ? kX86MemInfo_BaseRip :
- (B == Label::kLabelTag ) ? kX86MemInfo_BaseLabel : 0,
-
- kIndex = (I >= Reg::kTypeGpw && I <= Reg::kTypeGpq ) ? kX86MemInfo_Index :
- (I >= Reg::kTypeXmm && I <= Reg::kTypeZmm ) ? kX86MemInfo_Index : 0,
-
- k67H = (B == Reg::kTypeGpw && I == Reg::kTypeNone) ? kX86MemInfo_67H_X86 :
- (B == Reg::kTypeGpd && I == Reg::kTypeNone) ? kX86MemInfo_67H_X64 :
- (B == Reg::kTypeNone && I == Reg::kTypeGpw ) ? kX86MemInfo_67H_X86 :
- (B == Reg::kTypeNone && I == Reg::kTypeGpd ) ? kX86MemInfo_67H_X64 :
- (B == Reg::kTypeGpw && I == Reg::kTypeGpw ) ? kX86MemInfo_67H_X86 :
- (B == Reg::kTypeGpd && I == Reg::kTypeGpd ) ? kX86MemInfo_67H_X64 :
- (B == Reg::kTypeGpw && I == Reg::kTypeXmm ) ? kX86MemInfo_67H_X86 :
- (B == Reg::kTypeGpd && I == Reg::kTypeXmm ) ? kX86MemInfo_67H_X64 :
- (B == Reg::kTypeGpw && I == Reg::kTypeYmm ) ? kX86MemInfo_67H_X86 :
- (B == Reg::kTypeGpd && I == Reg::kTypeYmm ) ? kX86MemInfo_67H_X64 :
- (B == Reg::kTypeGpw && I == Reg::kTypeZmm ) ? kX86MemInfo_67H_X86 :
- (B == Reg::kTypeGpd && I == Reg::kTypeZmm ) ? kX86MemInfo_67H_X64 :
- (B == Label::kLabelTag && I == Reg::kTypeGpw ) ? kX86MemInfo_67H_X86 :
- (B == Label::kLabelTag && I == Reg::kTypeGpd ) ? kX86MemInfo_67H_X64 : 0,
+ kBase = (B >= uint32_t(RegType::kX86_Gpw) && B <= uint32_t(RegType::kX86_Gpq)) ? kX86MemInfo_BaseGp :
+ (B == uint32_t(RegType::kX86_Rip) ) ? kX86MemInfo_BaseRip :
+ (B == uint32_t(RegType::kLabelTag) ) ? kX86MemInfo_BaseLabel : 0,
+
+ kIndex = (I >= uint32_t(RegType::kX86_Gpw) && I <= uint32_t(RegType::kX86_Gpq)) ? kX86MemInfo_Index :
+ (I >= uint32_t(RegType::kX86_Xmm) && I <= uint32_t(RegType::kX86_Zmm)) ? kX86MemInfo_Index : 0,
+
+ k67H = (B == uint32_t(RegType::kX86_Gpw) && I == uint32_t(RegType::kNone) ) ? kX86MemInfo_67H_X86 :
+ (B == uint32_t(RegType::kX86_Gpd) && I == uint32_t(RegType::kNone) ) ? kX86MemInfo_67H_X64 :
+ (B == uint32_t(RegType::kNone) && I == uint32_t(RegType::kX86_Gpw)) ? kX86MemInfo_67H_X86 :
+ (B == uint32_t(RegType::kNone) && I == uint32_t(RegType::kX86_Gpd)) ? kX86MemInfo_67H_X64 :
+ (B == uint32_t(RegType::kX86_Gpw) && I == uint32_t(RegType::kX86_Gpw)) ? kX86MemInfo_67H_X86 :
+ (B == uint32_t(RegType::kX86_Gpd) && I == uint32_t(RegType::kX86_Gpd)) ? kX86MemInfo_67H_X64 :
+ (B == uint32_t(RegType::kX86_Gpw) && I == uint32_t(RegType::kX86_Xmm)) ? kX86MemInfo_67H_X86 :
+ (B == uint32_t(RegType::kX86_Gpd) && I == uint32_t(RegType::kX86_Xmm)) ? kX86MemInfo_67H_X64 :
+ (B == uint32_t(RegType::kX86_Gpw) && I == uint32_t(RegType::kX86_Ymm)) ? kX86MemInfo_67H_X86 :
+ (B == uint32_t(RegType::kX86_Gpd) && I == uint32_t(RegType::kX86_Ymm)) ? kX86MemInfo_67H_X64 :
+ (B == uint32_t(RegType::kX86_Gpw) && I == uint32_t(RegType::kX86_Zmm)) ? kX86MemInfo_67H_X86 :
+ (B == uint32_t(RegType::kX86_Gpd) && I == uint32_t(RegType::kX86_Zmm)) ? kX86MemInfo_67H_X64 :
+ (B == uint32_t(RegType::kLabelTag) && I == uint32_t(RegType::kX86_Gpw)) ? kX86MemInfo_67H_X86 :
+ (B == uint32_t(RegType::kLabelTag) && I == uint32_t(RegType::kX86_Gpd)) ? kX86MemInfo_67H_X64 : 0,
kValue = kBase | kIndex | k67H | 0x04 | 0x08
};
};
// The result stored in the LUT is a combination of
-// - 67H - Address override prefix - depends on BASE+INDEX register types and
-// the target architecture.
-// - REX - A possible combination of REX.[B|X|R|W] bits in REX prefix where
-// REX.B and REX.X are possibly masked out, but REX.R and REX.W are
-// kept as is.
-#define VALUE(X) X86MemInfo_T<X>::kValue
+// - 67H - Address override prefix - depends on BASE+INDEX register types and the target architecture.
+// - REX - A possible combination of REX.[B|X|R|W] bits in REX prefix where REX.B and REX.X are possibly
+// masked out, but REX.R and REX.W are kept as is.
+#define VALUE(x) X86MemInfo_T<x>::kValue
static const uint8_t x86MemInfo[] = { ASMJIT_LOOKUP_TABLE_1024(VALUE, 0) };
#undef VALUE
-// VEX3 or XOP xor bits applied to the opcode before emitted. The index to this
-// table is 'mmmmm' value, which contains all we need. This is only used by a
-// 3 BYTE VEX and XOP prefixes, 2 BYTE VEX prefix is handled differently. The
-// idea is to minimize the difference between VEX3 vs XOP when encoding VEX
-// or XOP instruction. This should minimize the code required to emit such
-// instructions and should also make it faster as we don't need any branch to
+// VEX3 or XOP xor bits applied to the opcode before emitted. The index to this table is 'mmmmm' value, which
+// contains all we need. This is only used by a 3 BYTE VEX and XOP prefixes, 2 BYTE VEX prefix is handled differently.
+// The idea is to minimize the difference between VEX3 vs XOP when encoding VEX or XOP instruction. This should
+// minimize the code required to emit such instructions and should also make it faster as we don't need any branch to
// decide between VEX3 vs XOP.
// ____ ___
// [_OPCODE_|WvvvvLpp|RXBmmmmm|VEX3_XOP]
-#define VALUE(X) ((X & 0x08) ? kX86ByteXop3 : kX86ByteVex3) | (0xF << 19) | (0x7 << 13)
+#define VALUE(x) ((x & 0x08) ? kX86ByteXop3 : kX86ByteVex3) | (0xF << 19) | (0x7 << 13)
static const uint32_t x86VEXPrefix[] = { ASMJIT_LOOKUP_TABLE_16(VALUE, 0) };
#undef VALUE
-// Table that contains LL opcode field addressed by a register size / 16. It's
-// used to propagate L.256 or L.512 when YMM or ZMM registers are used,
-// respectively.
-#define VALUE(X) (X & (64 >> 4)) ? Opcode::kLL_2 : \
- (X & (32 >> 4)) ? Opcode::kLL_1 : Opcode::kLL_0
+// Table that contains LL opcode field addressed by a register size / 16. It's used to propagate L.256 or L.512 when
+// YMM or ZMM registers are used, respectively.
+#define VALUE(x) (x & (64 >> 4)) ? Opcode::kLL_2 : \
+ (x & (32 >> 4)) ? Opcode::kLL_1 : Opcode::kLL_0
static const uint32_t x86LLBySizeDiv16[] = { ASMJIT_LOOKUP_TABLE_16(VALUE, 0) };
#undef VALUE
-// Table that contains LL opcode field addressed by a register size / 16. It's
-// used to propagate L.256 or L.512 when YMM or ZMM registers are used,
-// respectively.
-#define VALUE(X) X == Reg::kTypeZmm ? Opcode::kLL_2 : \
- X == Reg::kTypeYmm ? Opcode::kLL_1 : Opcode::kLL_0
+// Table that contains LL opcode field addressed by a register size / 16. It's used to propagate L.256 or L.512 when
+// YMM or ZMM registers are used, respectively.
+#define VALUE(x) x == uint32_t(RegType::kX86_Zmm) ? Opcode::kLL_2 : \
+ x == uint32_t(RegType::kX86_Ymm) ? Opcode::kLL_1 : Opcode::kLL_0
static const uint32_t x86LLByRegType[] = { ASMJIT_LOOKUP_TABLE_16(VALUE, 0) };
#undef VALUE
-// Table that contains a scale (shift left) based on 'TTWLL' field and
-// the instruction's tuple-type (TT) field. The scale is then applied to
-// the BASE-N stored in each opcode to calculate the final compressed
-// displacement used by all EVEX encoded instructions.
+// Table that contains a scale (shift left) based on 'TTWLL' field and the instruction's tuple-type (TT) field. The
+// scale is then applied to the BASE-N stored in each opcode to calculate the final compressed displacement used by
+// all EVEX encoded instructions.
template<uint32_t X>
struct X86CDisp8SHL_T {
enum {
@@ -278,7 +248,7 @@ struct X86CDisp8SHL_T {
};
};
-#define VALUE(X) X86CDisp8SHL_T<X>::kValue
+#define VALUE(x) X86CDisp8SHL_T<x>::kValue
static const uint32_t x86CDisp8SHL[] = { ASMJIT_LOOKUP_TABLE_32(VALUE, 0) };
#undef VALUE
@@ -310,37 +280,36 @@ struct X86Mod16BaseIndexTable_T {
};
};
-#define VALUE(X) X86Mod16BaseIndexTable_T<X>::kValue
+#define VALUE(x) X86Mod16BaseIndexTable_T<x>::kValue
static const uint8_t x86Mod16BaseIndexTable[] = { ASMJIT_LOOKUP_TABLE_64(VALUE, 0) };
#undef VALUE
-// ============================================================================
-// [asmjit::x86::Assembler - Helpers]
-// ============================================================================
+// x86::Assembler - Helpers
+// ========================
-static ASMJIT_INLINE bool x86IsJmpOrCall(uint32_t instId) noexcept {
+static ASMJIT_FORCE_INLINE bool x86IsJmpOrCall(InstId instId) noexcept {
return instId == Inst::kIdJmp || instId == Inst::kIdCall;
}
-static ASMJIT_INLINE bool x86IsImplicitMem(const Operand_& op, uint32_t base) noexcept {
+static ASMJIT_FORCE_INLINE bool x86IsImplicitMem(const Operand_& op, uint32_t base) noexcept {
return op.isMem() && op.as<Mem>().baseId() == base && !op.as<Mem>().hasOffset();
}
//! Combine `regId` and `vvvvvId` into a single value (used by AVX and AVX-512).
-static ASMJIT_INLINE uint32_t x86PackRegAndVvvvv(uint32_t regId, uint32_t vvvvvId) noexcept {
+static ASMJIT_FORCE_INLINE uint32_t x86PackRegAndVvvvv(uint32_t regId, uint32_t vvvvvId) noexcept {
return regId + (vvvvvId << kVexVVVVVShift);
}
-static ASMJIT_INLINE uint32_t x86OpcodeLByVMem(const Operand_& op) noexcept {
- return x86LLByRegType[op.as<Mem>().indexType()];
+static ASMJIT_FORCE_INLINE uint32_t x86OpcodeLByVMem(const Operand_& op) noexcept {
+ return x86LLByRegType[size_t(op.as<Mem>().indexType())];
}
-static ASMJIT_INLINE uint32_t x86OpcodeLBySize(uint32_t size) noexcept {
+static ASMJIT_FORCE_INLINE uint32_t x86OpcodeLBySize(uint32_t size) noexcept {
return x86LLBySizeDiv16[size / 16];
}
//! Encode MOD byte.
-static ASMJIT_INLINE uint32_t x86EncodeMod(uint32_t m, uint32_t o, uint32_t rm) noexcept {
+static ASMJIT_FORCE_INLINE uint32_t x86EncodeMod(uint32_t m, uint32_t o, uint32_t rm) noexcept {
ASMJIT_ASSERT(m <= 3);
ASMJIT_ASSERT(o <= 7);
ASMJIT_ASSERT(rm <= 7);
@@ -348,14 +317,14 @@ static ASMJIT_INLINE uint32_t x86EncodeMod(uint32_t m, uint32_t o, uint32_t rm)
}
//! Encode SIB byte.
-static ASMJIT_INLINE uint32_t x86EncodeSib(uint32_t s, uint32_t i, uint32_t b) noexcept {
+static ASMJIT_FORCE_INLINE uint32_t x86EncodeSib(uint32_t s, uint32_t i, uint32_t b) noexcept {
ASMJIT_ASSERT(s <= 3);
ASMJIT_ASSERT(i <= 7);
ASMJIT_ASSERT(b <= 7);
return (s << 6) + (i << 3) + b;
}
-static ASMJIT_INLINE bool x86IsRexInvalid(uint32_t rex) noexcept {
+static ASMJIT_FORCE_INLINE bool x86IsRexInvalid(uint32_t rex) noexcept {
// Validates the following possibilities:
// REX == 0x00 -> OKAY (X86_32 / X86_64).
// REX == 0x40-0x4F -> OKAY (X86_64).
@@ -364,29 +333,33 @@ static ASMJIT_INLINE bool x86IsRexInvalid(uint32_t rex) noexcept {
return rex > kX86ByteInvalidRex;
}
+static ASMJIT_FORCE_INLINE uint32_t x86GetForceEvex3MaskInLastBit(InstOptions options) noexcept {
+ constexpr uint32_t kVex3Bit = Support::ConstCTZ<uint32_t(InstOptions::kX86_Vex3)>::value;
+ return uint32_t(options & InstOptions::kX86_Vex3) << (31 - kVex3Bit);
+}
+
template<typename T>
-static constexpr T x86SignExtendI32(T imm) noexcept { return T(int64_t(int32_t(imm & T(0xFFFFFFFF)))); }
+static ASMJIT_FORCE_INLINE constexpr T x86SignExtendI32(T imm) noexcept { return T(int64_t(int32_t(imm & T(0xFFFFFFFF)))); }
-static ASMJIT_INLINE uint32_t x86AltOpcodeOf(const InstDB::InstInfo* info) noexcept {
+static ASMJIT_FORCE_INLINE uint32_t x86AltOpcodeOf(const InstDB::InstInfo* info) noexcept {
return InstDB::_altOpcodeTable[info->_altOpcodeIndex];
}
-// ============================================================================
-// [asmjit::X86BufferWriter]
-// ============================================================================
+// x86::Assembler - X86BufferWriter
+// ================================
-class X86BufferWriter : public CodeBufferWriter {
+class X86BufferWriter : public CodeWriter {
public:
- ASMJIT_INLINE explicit X86BufferWriter(Assembler* a) noexcept
- : CodeBufferWriter(a) {}
+ ASMJIT_FORCE_INLINE explicit X86BufferWriter(Assembler* a) noexcept
+ : CodeWriter(a) {}
- ASMJIT_INLINE void emitPP(uint32_t opcode) noexcept {
+ ASMJIT_FORCE_INLINE void emitPP(uint32_t opcode) noexcept {
uint32_t ppIndex = (opcode >> Opcode::kPP_Shift) &
(Opcode::kPP_FPUMask >> Opcode::kPP_Shift) ;
emit8If(x86OpcodePP[ppIndex], ppIndex != 0);
}
- ASMJIT_INLINE void emitMMAndOpcode(uint32_t opcode) noexcept {
+ ASMJIT_FORCE_INLINE void emitMMAndOpcode(uint32_t opcode) noexcept {
uint32_t mmIndex = (opcode & Opcode::kMM_Mask) >> Opcode::kMM_Shift;
const X86OpcodeMM& mmCode = x86OpcodeMM[mmIndex];
@@ -395,7 +368,7 @@ public:
emit8(opcode);
}
- ASMJIT_INLINE void emitSegmentOverride(uint32_t segmentId) noexcept {
+ ASMJIT_FORCE_INLINE void emitSegmentOverride(uint32_t segmentId) noexcept {
ASMJIT_ASSERT(segmentId < ASMJIT_ARRAY_SIZE(x86SegmentPrefix));
FastUInt8 prefix = x86SegmentPrefix[segmentId];
@@ -403,11 +376,11 @@ public:
}
template<typename CondT>
- ASMJIT_INLINE void emitAddressOverride(CondT condition) noexcept {
+ ASMJIT_FORCE_INLINE void emitAddressOverride(CondT condition) noexcept {
emit8If(0x67, condition);
}
- ASMJIT_INLINE void emitImmByteOrDWord(uint64_t immValue, FastUInt8 immSize) noexcept {
+ ASMJIT_FORCE_INLINE void emitImmByteOrDWord(uint64_t immValue, FastUInt8 immSize) noexcept {
if (!immSize)
return;
@@ -431,41 +404,41 @@ public:
emit8(imm & 0xFFu);
}
- ASMJIT_INLINE void emitImmediate(uint64_t immValue, FastUInt8 immSize) noexcept {
- if (!immSize)
- return;
-
+ ASMJIT_FORCE_INLINE void emitImmediate(uint64_t immValue, FastUInt8 immSize) noexcept {
#if ASMJIT_ARCH_BITS >= 64
- uint64_t imm = uint64_t(immValue);
+ uint64_t imm = immValue;
+ if (immSize >= 4) {
+ emit32uLE(imm & 0xFFFFFFFFu);
+ imm >>= 32;
+ immSize = FastUInt8(immSize - 4u);
+ }
#else
uint32_t imm = uint32_t(immValue & 0xFFFFFFFFu);
+ if (immSize >= 4) {
+ emit32uLE(imm);
+ imm = uint32_t(immValue >> 32);
+ immSize = FastUInt8(immSize - 4u);
+ }
#endif
- // Many instructions just use a single byte immediate, so make it fast.
+ if (!immSize)
+ return;
emit8(imm & 0xFFu);
- if (--immSize == 0) return;
-
imm >>= 8;
- emit8(imm & 0xFFu);
- if (--immSize == 0) return;
- imm >>= 8;
+ if (--immSize == 0)
+ return;
emit8(imm & 0xFFu);
- if (--immSize == 0) return;
-
imm >>= 8;
- emit8(imm & 0xFFu);
- if (--immSize == 0) return;
-
- // Can be 1, 2, 4 or 8 bytes, this handles the remaining high DWORD of an 8-byte immediate.
- ASMJIT_ASSERT(immSize == 4);
-#if ASMJIT_ARCH_BITS >= 64
+ if (--immSize == 0)
+ return;
+ emit8(imm & 0xFFu);
imm >>= 8;
- emit32uLE(uint32_t(imm));
-#else
- emit32uLE(uint32_t((uint64_t(immValue) >> 32) & 0xFFFFFFFFu));
-#endif
+
+ if (--immSize == 0)
+ return;
+ emit8(imm & 0xFFu);
}
};
@@ -474,97 +447,115 @@ public:
// If the operand is AH|BH|CH|DH
// - patch its index from 0..3 to 4..7 as encoded by X86.
// - Disallow REX prefix.
-#define FIXUP_GPB(REG_OP, REG_ID) \
- do { \
- if (!static_cast<const Gp&>(REG_OP).isGpbHi()) { \
- options |= (REG_ID >= 4) ? uint32_t(Inst::kOptionRex) \
- : uint32_t(0); \
- } \
- else { \
- options |= Inst::_kOptionInvalidRex; \
- REG_ID += 4; \
- } \
+#define FIXUP_GPB(REG_OP, REG_ID) \
+ do { \
+ if (!static_cast<const Gp&>(REG_OP).isGpbHi()) { \
+ options |= (REG_ID) >= 4 ? InstOptions::kX86_Rex \
+ : InstOptions::kNone; \
+ } \
+ else { \
+ options |= InstOptions::kX86_InvalidRex; \
+ REG_ID += 4; \
+ } \
} while (0)
-#define ENC_OPS1(OP0) ((Operand::kOp##OP0))
-#define ENC_OPS2(OP0, OP1) ((Operand::kOp##OP0) + ((Operand::kOp##OP1) << 3))
-#define ENC_OPS3(OP0, OP1, OP2) ((Operand::kOp##OP0) + ((Operand::kOp##OP1) << 3) + ((Operand::kOp##OP2) << 6))
-#define ENC_OPS4(OP0, OP1, OP2, OP3) ((Operand::kOp##OP0) + ((Operand::kOp##OP1) << 3) + ((Operand::kOp##OP2) << 6) + ((Operand::kOp##OP3) << 9))
+#define ENC_OPS1(OP0) \
+ (uint32_t(OperandType::k##OP0))
-// ============================================================================
-// [asmjit::x86::Assembler - Movabs Heuristics]
-// ============================================================================
+#define ENC_OPS2(OP0, OP1) \
+ (uint32_t(OperandType::k##OP0) + \
+ (uint32_t(OperandType::k##OP1) << 3))
+
+#define ENC_OPS3(OP0, OP1, OP2) \
+ (uint32_t(OperandType::k##OP0) + \
+ (uint32_t(OperandType::k##OP1) << 3) + \
+ (uint32_t(OperandType::k##OP2) << 6))
-static ASMJIT_INLINE bool x86GetMovAbsInstSize64Bit(uint32_t regSize, uint32_t options, const Mem& rmRel) noexcept {
+#define ENC_OPS4(OP0, OP1, OP2, OP3) \
+ (uint32_t(OperandType::k##OP0) + \
+ (uint32_t(OperandType::k##OP1) << 3) + \
+ (uint32_t(OperandType::k##OP2) << 6) + \
+ (uint32_t(OperandType::k##OP3) << 9))
+
+// x86::Assembler - Movabs Heuristics
+// ==================================
+
+static ASMJIT_FORCE_INLINE uint32_t x86GetMovAbsInstSize64Bit(uint32_t regSize, InstOptions options, const Mem& rmRel) noexcept {
uint32_t segmentPrefixSize = rmRel.segmentId() != 0;
uint32_t _66hPrefixSize = regSize == 2;
- uint32_t rexPrefixSize = (regSize == 8) || ((options & Inst::kOptionRex) != 0);
+ uint32_t rexPrefixSize = regSize == 8 || Support::test(options, InstOptions::kX86_Rex);
uint32_t opCodeByteSize = 1;
uint32_t immediateSize = 8;
return segmentPrefixSize + _66hPrefixSize + rexPrefixSize + opCodeByteSize + immediateSize;
}
-static ASMJIT_INLINE uint32_t x86GetMovAbsAddrType(Assembler* self, X86BufferWriter& writer, uint32_t regSize, uint32_t options, const Mem& rmRel) noexcept {
- uint32_t addrType = rmRel.addrType();
- int64_t addrValue = rmRel.offset();
-
- if (addrType == BaseMem::kAddrTypeDefault && !(options & Inst::kOptionModMR)) {
- if (self->is64Bit()) {
- uint64_t baseAddress = self->code()->baseAddress();
- if (baseAddress != Globals::kNoBaseAddress && !rmRel.hasSegment()) {
- uint32_t instructionSize = x86GetMovAbsInstSize64Bit(regSize, options, rmRel);
- uint64_t virtualOffset = uint64_t(writer.offsetFrom(self->_bufferData));
- uint64_t rip64 = baseAddress + self->_section->offset() + virtualOffset + instructionSize;
- uint64_t rel64 = uint64_t(addrValue) - rip64;
-
- if (!Support::isInt32(int64_t(rel64)))
- addrType = BaseMem::kAddrTypeAbs;
- }
- else {
- if (!Support::isInt32(addrValue))
- addrType = BaseMem::kAddrTypeAbs;
- }
+static ASMJIT_FORCE_INLINE bool x86ShouldUseMovabs(Assembler* self, X86BufferWriter& writer, uint32_t regSize, InstOptions options, const Mem& rmRel) noexcept {
+ if (self->is32Bit()) {
+ // There is no relative addressing, just decide whether to use MOV encoded with MOD R/M or absolute.
+ return !Support::test(options, InstOptions::kX86_ModMR | InstOptions::kX86_ModMR);
+ }
+ else {
+ // If the addressing type is REL or MOD R/M was specified then absolute mov won't be used.
+ if (rmRel.addrType() == Mem::AddrType::kRel || Support::test(options, InstOptions::kX86_ModMR))
+ return false;
+
+ int64_t addrValue = rmRel.offset();
+ uint64_t baseAddress = self->code()->baseAddress();
+
+ // If the address type is default, it means to basically check whether relative addressing is possible. However,
+ // this is only possible when the base address is known - relative encoding uses RIP+N it has to be calculated.
+ if (rmRel.addrType() == Mem::AddrType::kDefault && baseAddress != Globals::kNoBaseAddress && !rmRel.hasSegment()) {
+ uint32_t instructionSize = x86GetMovAbsInstSize64Bit(regSize, options, rmRel);
+ uint64_t virtualOffset = uint64_t(writer.offsetFrom(self->_bufferData));
+ uint64_t rip64 = baseAddress + self->_section->offset() + virtualOffset + instructionSize;
+ uint64_t rel64 = uint64_t(addrValue) - rip64;
+
+ if (Support::isInt32(int64_t(rel64)))
+ return false;
}
else {
- addrType = BaseMem::kAddrTypeAbs;
+ if (Support::isInt32(addrValue))
+ return false;
}
- }
- return addrType;
+ return uint64_t(addrValue) > 0xFFFFFFFFu;
+ }
}
-// ============================================================================
-// [asmjit::x86::Assembler - Construction / Destruction]
-// ============================================================================
+// x86::Assembler - Construction & Destruction
+// ===========================================
Assembler::Assembler(CodeHolder* code) noexcept : BaseAssembler() {
+ _archMask = (uint64_t(1) << uint32_t(Arch::kX86)) |
+ (uint64_t(1) << uint32_t(Arch::kX64)) ;
+ assignEmitterFuncs(this);
+
if (code)
code->attach(this);
}
Assembler::~Assembler() noexcept {}
-// ============================================================================
-// [asmjit::x86::Assembler - Emit (Low-Level)]
-// ============================================================================
+// x86::Assembler - Emit (Low-Level)
+// =================================
-ASMJIT_FAVOR_SPEED Error Assembler::_emit(uint32_t instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_* opExt) {
+ASMJIT_FAVOR_SPEED Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_* opExt) {
constexpr uint32_t kVSHR_W = Opcode::kW_Shift - 23;
constexpr uint32_t kVSHR_PP = Opcode::kPP_Shift - 16;
constexpr uint32_t kVSHR_PP_EW = Opcode::kPP_Shift - 16;
- constexpr uint32_t kRequiresSpecialHandling =
- uint32_t(Inst::kOptionReserved) | // Logging/Validation/Error.
- uint32_t(Inst::kOptionRep ) | // REP/REPE prefix.
- uint32_t(Inst::kOptionRepne ) | // REPNE prefix.
- uint32_t(Inst::kOptionLock ) | // LOCK prefix.
- uint32_t(Inst::kOptionXAcquire) | // XACQUIRE prefix.
- uint32_t(Inst::kOptionXRelease) ; // XRELEASE prefix.
+ constexpr InstOptions kRequiresSpecialHandling =
+ InstOptions::kReserved | // Logging/Validation/Error.
+ InstOptions::kX86_Rep | // REP/REPE prefix.
+ InstOptions::kX86_Repne | // REPNE prefix.
+ InstOptions::kX86_Lock | // LOCK prefix.
+ InstOptions::kX86_XAcquire | // XACQUIRE prefix.
+ InstOptions::kX86_XRelease ; // XRELEASE prefix.
Error err;
Opcode opcode; // Instruction opcode.
- uint32_t options; // Instruction options.
+ InstOptions options; // Instruction options.
uint32_t isign3; // A combined signature of first 3 operands.
const Operand_* rmRel; // Memory operand or operand that holds Label|Imm.
@@ -591,17 +582,16 @@ ASMJIT_FAVOR_SPEED Error Assembler::_emit(uint32_t instId, const Operand_& o0, c
const InstDB::CommonInfo* commonInfo = &instInfo->commonInfo();
// Signature of the first 3 operands.
- isign3 = o0.opType() + (o1.opType() << 3) + (o2.opType() << 6);
+ isign3 = (uint32_t(o0.opType()) ) +
+ (uint32_t(o1.opType()) << 3) +
+ (uint32_t(o2.opType()) << 6);
- // Combine all instruction options and also check whether the instruction
- // is valid. All options that require special handling (including invalid
- // instruction) are handled by the next branch.
- options = uint32_t(instId == 0);
- options |= uint32_t((size_t)(_bufferEnd - writer.cursor()) < 16);
- options |= uint32_t(instOptions() | forcedInstOptions());
+ // Combine all instruction options and also check whether the instruction is valid. All options
+ // that require special handling (including invalid instruction) are handled by the next branch.
+ options = InstOptions((instId == 0) | ((size_t)(_bufferEnd - writer.cursor()) < 16)) | instOptions() | forcedInstOptions();
// Handle failure and rare cases first.
- if (ASMJIT_UNLIKELY(options & kRequiresSpecialHandling)) {
+ if (ASMJIT_UNLIKELY(Support::test(options, kRequiresSpecialHandling))) {
if (ASMJIT_UNLIKELY(!_code))
return reportError(DebugUtils::errored(kErrorNotInitialized));
@@ -616,87 +606,93 @@ ASMJIT_FAVOR_SPEED Error Assembler::_emit(uint32_t instId, const Operand_& o0, c
#ifndef ASMJIT_NO_VALIDATION
// Strict validation.
- if (hasValidationOption(kValidationOptionAssembler)) {
+ if (hasDiagnosticOption(DiagnosticOptions::kValidateAssembler)) {
Operand_ opArray[Globals::kMaxOpCount];
EmitterUtils::opArrayFromEmitArgs(opArray, o0, o1, o2, opExt);
- err = InstAPI::validate(arch(), BaseInst(instId, options, _extraReg), opArray, Globals::kMaxOpCount);
+ err = _funcs.validate(arch(), BaseInst(instId, options, _extraReg), opArray, Globals::kMaxOpCount, ValidationFlags::kNone);
if (ASMJIT_UNLIKELY(err))
goto Failed;
}
#endif
- uint32_t iFlags = instInfo->flags();
+ InstDB::InstFlags iFlags = instInfo->flags();
// LOCK, XACQUIRE, and XRELEASE prefixes.
- if (options & Inst::kOptionLock) {
- bool xAcqRel = (options & (Inst::kOptionXAcquire | Inst::kOptionXRelease)) != 0;
+ if (Support::test(options, InstOptions::kX86_Lock)) {
+ bool xAcqRel = Support::test(options, InstOptions::kX86_XAcquire | InstOptions::kX86_XRelease);
- if (ASMJIT_UNLIKELY(!(iFlags & (InstDB::kFlagLock)) && !xAcqRel))
+ if (ASMJIT_UNLIKELY(!Support::test(iFlags, InstDB::InstFlags::kLock) && !xAcqRel))
goto InvalidLockPrefix;
if (xAcqRel) {
- if (ASMJIT_UNLIKELY((options & Inst::kOptionXAcquire) && !(iFlags & InstDB::kFlagXAcquire)))
+ if (ASMJIT_UNLIKELY(Support::test(options, InstOptions::kX86_XAcquire) && !Support::test(iFlags, InstDB::InstFlags::kXAcquire)))
goto InvalidXAcquirePrefix;
- if (ASMJIT_UNLIKELY((options & Inst::kOptionXRelease) && !(iFlags & InstDB::kFlagXRelease)))
+ if (ASMJIT_UNLIKELY(Support::test(options, InstOptions::kX86_XRelease) && !Support::test(iFlags, InstDB::InstFlags::kXRelease)))
goto InvalidXReleasePrefix;
- writer.emit8((options & Inst::kOptionXAcquire) ? 0xF2 : 0xF3);
+ writer.emit8(Support::test(options, InstOptions::kX86_XAcquire) ? 0xF2 : 0xF3);
}
writer.emit8(0xF0);
}
// REP and REPNE prefixes.
- if (options & (Inst::kOptionRep | Inst::kOptionRepne)) {
- if (ASMJIT_UNLIKELY(!(iFlags & InstDB::kFlagRep)))
+ if (Support::test(options, InstOptions::kX86_Rep | InstOptions::kX86_Repne)) {
+ if (ASMJIT_UNLIKELY(!Support::test(iFlags, InstDB::InstFlags::kRep)))
goto InvalidRepPrefix;
- if (_extraReg.isReg() && ASMJIT_UNLIKELY(_extraReg.group() != Reg::kGroupGp || _extraReg.id() != Gp::kIdCx))
+ if (ASMJIT_UNLIKELY(_extraReg.isReg() && (_extraReg.group() != RegGroup::kGp || _extraReg.id() != Gp::kIdCx)))
goto InvalidRepPrefix;
- writer.emit8((options & Inst::kOptionRepne) ? 0xF2 : 0xF3);
+ writer.emit8(Support::test(options, InstOptions::kX86_Repne) ? 0xF2 : 0xF3);
}
}
// This sequence seems to be the fastest.
opcode = InstDB::_mainOpcodeTable[instInfo->_mainOpcodeIndex];
- opReg = opcode.extractO();
+ opReg = opcode.extractModO();
+ rbReg = 0;
opcode |= instInfo->_mainOpcodeValue;
- // --------------------------------------------------------------------------
- // [Encoding Scope]
- // --------------------------------------------------------------------------
+ // Encoding Scope
+ // --------------
+
+ // How it works? Each case here represents a unique encoding of a group of instructions, which is handled
+ // separately. The handlers check instruction signature, possibly register types, etc, and process this
+ // information by writing some bits to opcode, opReg/rbReg, immValue/immSize, etc, and then at the end of
+ // the sequence it uses goto to jump into a lower level handler, that actually encodes the instruction.
switch (instInfo->_encoding) {
case InstDB::kEncodingNone:
goto EmitDone;
- // ------------------------------------------------------------------------
- // [X86]
- // ------------------------------------------------------------------------
+ // Base Instructions
+ // -----------------
case InstDB::kEncodingX86Op:
goto EmitX86Op;
- case InstDB::kEncodingX86Op_O_I8:
- if (ASMJIT_UNLIKELY(isign3 != ENC_OPS1(Imm)))
+ case InstDB::kEncodingX86Op_Mod11RM:
+ rbReg = opcode.extractModRM();
+ goto EmitX86R;
+
+ case InstDB::kEncodingX86Op_Mod11RM_I8:
+ // The first operand must be immediate, we don't care of other operands as they could be implicit.
+ if (!o0.isImm())
goto InvalidInstruction;
+ rbReg = opcode.extractModRM();
immValue = o0.as<Imm>().valueAs<uint8_t>();
immSize = 1;
- ASMJIT_FALLTHROUGH;
-
- case InstDB::kEncodingX86Op_O:
- rbReg = 0;
goto EmitX86R;
case InstDB::kEncodingX86Op_xAddr:
if (ASMJIT_UNLIKELY(!o0.isReg()))
goto InvalidInstruction;
- rmInfo = x86MemInfo[o0.as<Reg>().type()];
+ rmInfo = x86MemInfo[size_t(o0.as<Reg>().type())];
writer.emitAddressOverride((rmInfo & _addressOverrideMask()) != 0);
goto EmitX86Op;
@@ -742,11 +738,17 @@ ASMJIT_FAVOR_SPEED Error Assembler::_emit(uint32_t instId, const Operand_& o0, c
}
break;
+ case InstDB::kEncodingX86M_NoMemSize:
+ if (o0.isReg())
+ opcode.addPrefixBySize(o0.size());
+ goto CaseX86M_NoSize;
+
case InstDB::kEncodingX86M:
opcode.addPrefixBySize(o0.size());
ASMJIT_FALLTHROUGH;
case InstDB::kEncodingX86M_NoSize:
+CaseX86M_NoSize:
rbReg = o0.id();
if (isign3 == ENC_OPS1(Reg))
goto EmitX86R;
@@ -826,6 +828,13 @@ CaseX86M_GPB_MulDiv:
}
break;
+ case InstDB::kEncodingX86M_Only_EDX_EAX:
+ if (isign3 == ENC_OPS3(Mem, Reg, Reg) && Reg::isGpd(o1, Gp::kIdDx) && Reg::isGpd(o2, Gp::kIdAx)) {
+ rmRel = &o0;
+ goto EmitX86M;
+ }
+ ASMJIT_FALLTHROUGH;
+
case InstDB::kEncodingX86M_Only:
if (isign3 == ENC_OPS1(Mem)) {
rmRel = &o0;
@@ -837,21 +846,61 @@ CaseX86M_GPB_MulDiv:
if (isign3 == ENC_OPS1(None))
goto EmitX86Op;
- // Multi-byte NOP instruction "0F 1F /0".
+ // Single operand NOP instruction "0F 1F /0".
opcode = Opcode::k000F00 | 0x1F;
opReg = 0;
if (isign3 == ENC_OPS1(Reg)) {
- opcode.add66hBySize(o0.size());
+ opcode.addPrefixBySize(o0.size());
rbReg = o0.id();
goto EmitX86R;
}
if (isign3 == ENC_OPS1(Mem)) {
- opcode.add66hBySize(o0.size());
+ opcode.addPrefixBySize(o0.size());
rmRel = &o0;
goto EmitX86M;
}
+
+ // Two operand NOP instruction "0F 1F /r".
+ opReg = o1.id();
+ opcode.addPrefixBySize(o1.size());
+
+ if (isign3 == ENC_OPS2(Reg, Reg)) {
+ rbReg = o0.id();
+ goto EmitX86R;
+ }
+
+ if (isign3 == ENC_OPS2(Mem, Reg)) {
+ rmRel = &o0;
+ goto EmitX86M;
+ }
+ break;
+
+ case InstDB::kEncodingX86R_FromM:
+ if (isign3 == ENC_OPS1(Mem)) {
+ rmRel = &o0;
+ rbReg = o0.id();
+ goto EmitX86RFromM;
+ }
+ break;
+
+ case InstDB::kEncodingX86R32_EDX_EAX:
+ // Explicit form: R32, EDX, EAX.
+ if (isign3 == ENC_OPS3(Reg, Reg, Reg)) {
+ if (!Reg::isGpd(o1, Gp::kIdDx) || !Reg::isGpd(o2, Gp::kIdAx))
+ goto InvalidInstruction;
+ rbReg = o0.id();
+ goto EmitX86R;
+ }
+
+ // Implicit form: R32.
+ if (isign3 == ENC_OPS1(Reg)) {
+ if (!Reg::isGpd(o0))
+ goto InvalidInstruction;
+ rbReg = o0.id();
+ goto EmitX86R;
+ }
break;
case InstDB::kEncodingX86R_Native:
@@ -924,34 +973,27 @@ CaseX86M_GPB_MulDiv:
case InstDB::kEncodingX86Arith:
if (isign3 == ENC_OPS2(Reg, Reg)) {
- opcode += 2;
opcode.addArithBySize(o0.size());
if (o0.size() != o1.size())
goto OperandSizeMismatch;
- opReg = o0.id();
- rbReg = o1.id();
+ rbReg = o0.id();
+ opReg = o1.id();
if (o0.size() == 1) {
- FIXUP_GPB(o0, opReg);
- FIXUP_GPB(o1, rbReg);
-
- if (!(options & Inst::kOptionModMR))
- goto EmitX86R;
-
- opcode -= 2;
- std::swap(opReg, rbReg);
- goto EmitX86R;
+ FIXUP_GPB(o0, rbReg);
+ FIXUP_GPB(o1, opReg);
}
- else {
- if (!(options & Inst::kOptionModMR))
- goto EmitX86R;
- opcode -= 2;
- std::swap(opReg, rbReg);
+ // MOD/MR: The default encoding used if not instructed otherwise..
+ if (!Support::test(options, InstOptions::kX86_ModRM))
goto EmitX86R;
- }
+
+ // MOD/RM: Alternative encoding selected via instruction options.
+ opcode += 2;
+ std::swap(opReg, rbReg);
+ goto EmitX86R;
}
if (isign3 == ENC_OPS2(Reg, Mem)) {
@@ -1013,7 +1055,7 @@ CaseX86M_GPB_MulDiv:
else
goto InvalidImmediate;
}
- else if (canTransformTo32Bit && hasEncodingOption(kEncodingOptionOptimizeForSize)) {
+ else if (canTransformTo32Bit && hasEncodingOption(EncodingOptions::kOptimizeForSize)) {
size = 4;
}
@@ -1021,12 +1063,12 @@ CaseX86M_GPB_MulDiv:
}
immSize = FastUInt8(Support::min<uint32_t>(size, 4));
- if (Support::isInt8(immValue) && !(options & Inst::kOptionLongForm))
+ if (Support::isInt8(immValue) && !Support::test(options, InstOptions::kLongForm))
immSize = 1;
}
// Short form - AL, AX, EAX, RAX.
- if (rbReg == 0 && (size == 1 || immSize != 1) && !(options & Inst::kOptionLongForm)) {
+ if (rbReg == 0 && (size == 1 || immSize != 1) && !Support::test(options, InstOptions::kLongForm)) {
opcode &= Opcode::kPP_66 | Opcode::kW;
opcode |= ((opReg << 3) | (0x04 + (size != 1)));
immSize = FastUInt8(Support::min<uint32_t>(size, 4));
@@ -1050,7 +1092,7 @@ CaseX86M_GPB_MulDiv:
if (memSize == 4)
immValue = x86SignExtendI32<int64_t>(immValue);
- if (Support::isInt8(immValue) && !(options & Inst::kOptionLongForm))
+ if (Support::isInt8(immValue) && !Support::test(options, InstOptions::kLongForm))
immSize = 1;
opcode += memSize != 1 ? (immSize != 1 ? 1 : 3) : 0;
@@ -1093,7 +1135,7 @@ CaseX86M_GPB_MulDiv:
opcode = x86AltOpcodeOf(instInfo);
opcode.addPrefixBySize(o0.size());
- opReg = opcode.extractO();
+ opReg = opcode.extractModO();
if (isign3 == ENC_OPS2(Reg, Imm)) {
rbReg = o0.id();
@@ -1119,9 +1161,8 @@ CaseX86M_GPB_MulDiv:
if (isign3 == ENC_OPS1(Mem))
goto EmitX86M;
- // Call with 32-bit displacement use 0xE8 opcode. Call with 8-bit
- // displacement is not encodable so the alternative opcode field
- // in X86DB must be zero.
+ // Call with 32-bit displacement use 0xE8 opcode. Call with 8-bit displacement is not encodable so the
+ // alternative opcode field in X86DB must be zero.
opcode = 0xE8;
opReg = 0;
goto EmitJmpCall;
@@ -1158,7 +1199,7 @@ CaseX86M_GPB_MulDiv:
if (o1.size() != 1)
goto EmitX86M;
- FIXUP_GPB(o0, opReg);
+ FIXUP_GPB(o1, opReg);
goto EmitX86M;
}
break;
@@ -1235,7 +1276,7 @@ CaseX86M_GPB_MulDiv:
immValue = o2.as<Imm>().value();
immSize = 1;
- if (!Support::isInt8(immValue) || (options & Inst::kOptionLongForm)) {
+ if (!Support::isInt8(immValue) || Support::test(options, InstOptions::kLongForm)) {
opcode -= 2;
immSize = o0.size() == 2 ? 2 : 4;
}
@@ -1257,7 +1298,7 @@ CaseX86M_GPB_MulDiv:
if (o0.size() == 4)
immValue = x86SignExtendI32<int64_t>(immValue);
- if (!Support::isInt8(immValue) || (options & Inst::kOptionLongForm)) {
+ if (!Support::isInt8(immValue) || Support::test(options, InstOptions::kLongForm)) {
opcode -= 2;
immSize = o0.size() == 2 ? 2 : 4;
}
@@ -1309,7 +1350,7 @@ CaseX86M_GPB_MulDiv:
if (o0.size() == 4)
immValue = x86SignExtendI32<int64_t>(immValue);
- if (!Support::isInt8(immValue) || (options & Inst::kOptionLongForm)) {
+ if (!Support::isInt8(immValue) || Support::test(options, InstOptions::kLongForm)) {
opcode -= 2;
immSize = o0.size() == 2 ? 2 : 4;
}
@@ -1383,6 +1424,8 @@ CaseX86M_GPB_MulDiv:
}
if (isign3 == ENC_OPS1(Mem)) {
+ if (!o0.size())
+ goto AmbiguousOperandSize;
opcode.addArithBySize(o0.size());
rmRel = &o0;
goto EmitX86M;
@@ -1398,8 +1441,8 @@ CaseX86M_GPB_MulDiv:
break;
case InstDB::kEncodingX86Jcc:
- if ((options & (Inst::kOptionTaken | Inst::kOptionNotTaken)) && hasEncodingOption(kEncodingOptionPredictedJumps)) {
- uint8_t prefix = (options & Inst::kOptionTaken) ? uint8_t(0x3E) : uint8_t(0x2E);
+ if (Support::test(options, InstOptions::kTaken | InstOptions::kNotTaken) && hasEncodingOption(EncodingOptions::kPredictedJumps)) {
+ uint8_t prefix = Support::test(options, InstOptions::kTaken) ? uint8_t(0x3E) : uint8_t(0x2E);
writer.emit8(prefix);
}
@@ -1431,8 +1474,8 @@ CaseX86M_GPB_MulDiv:
if (isign3 == ENC_OPS1(Mem))
goto EmitX86M;
- // Jump encoded with 32-bit displacement use 0xE9 opcode. Jump encoded
- // with 8-bit displacement's opcode is stored as an alternative opcode.
+ // Jump encoded with 32-bit displacement use 0xE9 opcode. Jump encoded with 8-bit displacement's opcode is
+ // stored as an alternative opcode.
opcode = 0xE9;
opReg = 0;
goto EmitJmpCall;
@@ -1441,6 +1484,39 @@ CaseX86M_GPB_MulDiv:
rmRel = &o0;
goto EmitJmpCall;
+ case InstDB::kEncodingX86LcallLjmp:
+ if (isign3 == ENC_OPS1(Mem)) {
+ rmRel = &o0;
+ uint32_t mSize = rmRel->size();
+ if (mSize == 0) {
+ mSize = registerSize();
+ }
+ else {
+ mSize -= 2;
+ if (mSize != 2 && mSize != 4 && mSize != registerSize())
+ goto InvalidAddress;
+ }
+ opcode.addPrefixBySize(mSize);
+ goto EmitX86M;
+ }
+
+ if (isign3 == ENC_OPS2(Imm, Imm)) {
+ if (!is32Bit())
+ goto InvalidInstruction;
+
+ const Imm& imm0 = o0.as<Imm>();
+ const Imm& imm1 = o1.as<Imm>();
+
+ if (imm0.value() > 0xFFFFu || imm1.value() > 0xFFFFFFFFu)
+ goto InvalidImmediate;
+
+ opcode = x86AltOpcodeOf(instInfo);
+ immValue = imm1.value() | (imm0.value() << 32);
+ immSize = 6;
+ goto EmitX86Op;
+ }
+ break;
+
case InstDB::kEncodingX86Lea:
if (isign3 == ENC_OPS2(Reg, Mem)) {
opcode.addPrefixBySize(o0.size());
@@ -1453,58 +1529,46 @@ CaseX86M_GPB_MulDiv:
case InstDB::kEncodingX86Mov:
// Reg <- Reg
if (isign3 == ENC_OPS2(Reg, Reg)) {
- opReg = o0.id();
- rbReg = o1.id();
-
- // Asmjit uses segment registers indexed from 1 to 6, leaving zero as
- // "no segment register used". We have to fix this (decrement the index
- // of the register) when emitting MOV instructions which move to/from
- // a segment register. The segment register is always `opReg`, because
- // the MOV instruction uses either RM or MR encoding.
+ // Asmjit uses segment registers indexed from 1 to 6, leaving zero as "no segment register used". We have to
+ // fix this (decrement the index of the register) when emitting MOV instructions which move to/from a segment
+ // register. The segment register is always `opReg`, because the MOV instruction uses either RM or MR encoding.
// GP <- ??
if (Reg::isGp(o0)) {
+ rbReg = o0.id();
+ opReg = o1.id();
+
// GP <- GP
if (Reg::isGp(o1)) {
- uint32_t size0 = o0.size();
- uint32_t size1 = o1.size();
+ uint32_t opSize = o0.size();
+ if (opSize != o1.size())
+ goto InvalidInstruction;
- if (size0 != size1) {
- // We allow 'mov r64, r32' as it's basically zero-extend.
- if (size0 == 8 && size1 == 4)
- size0 = 4; // Zero extend, don't promote to 64-bit.
- else
- goto InvalidInstruction;
- }
+ if (opSize == 1) {
+ FIXUP_GPB(o0, rbReg);
+ FIXUP_GPB(o1, opReg);
+ opcode = 0x88;
- if (size0 == 1) {
- FIXUP_GPB(o0, opReg);
- FIXUP_GPB(o1, rbReg);
- opcode = 0x8A;
-
- if (!(options & Inst::kOptionModMR))
+ if (!Support::test(options, InstOptions::kX86_ModRM))
goto EmitX86R;
- opcode -= 2;
+ opcode += 2;
std::swap(opReg, rbReg);
goto EmitX86R;
}
else {
- opcode = 0x8B;
- opcode.addPrefixBySize(size0);
+ opcode = 0x89;
+ opcode.addPrefixBySize(opSize);
- if (!(options & Inst::kOptionModMR))
+ if (!Support::test(options, InstOptions::kX86_ModRM))
goto EmitX86R;
- opcode -= 2;
+ opcode += 2;
std::swap(opReg, rbReg);
goto EmitX86R;
}
}
- opReg = rbReg;
- rbReg = o0.id();
-
// GP <- SReg
if (Reg::isSReg(o1)) {
opcode = 0x8C;
@@ -1532,6 +1596,9 @@ CaseX86M_GPB_MulDiv:
}
}
else {
+ opReg = o0.id();
+ rbReg = o1.id();
+
// ?? <- GP
if (!Reg::isGp(o1))
goto InvalidInstruction;
@@ -1582,18 +1649,18 @@ CaseX86M_GPB_MulDiv:
opcode = 0;
opcode.addArithBySize(o0.size());
- if (o0.size() == 1)
- FIXUP_GPB(o0, opReg);
-
// Handle a special form of `mov al|ax|eax|rax, [ptr64]` that doesn't use MOD.
if (opReg == Gp::kIdAx && !rmRel->as<Mem>().hasBaseOrIndex()) {
- immValue = rmRel->as<Mem>().offset();
- if (x86GetMovAbsAddrType(this, writer, o0.size(), options, rmRel->as<Mem>()) == BaseMem::kAddrTypeAbs) {
+ if (x86ShouldUseMovabs(this, writer, o0.size(), options, rmRel->as<Mem>())) {
opcode += 0xA0;
+ immValue = rmRel->as<Mem>().offset();
goto EmitX86OpMovAbs;
}
}
+ if (o0.size() == 1)
+ FIXUP_GPB(o0, opReg);
+
opcode += 0x8A;
goto EmitX86M;
}
@@ -1615,18 +1682,18 @@ CaseX86M_GPB_MulDiv:
opcode = 0;
opcode.addArithBySize(o1.size());
- if (o1.size() == 1)
- FIXUP_GPB(o1, opReg);
-
// Handle a special form of `mov [ptr64], al|ax|eax|rax` that doesn't use MOD.
if (opReg == Gp::kIdAx && !rmRel->as<Mem>().hasBaseOrIndex()) {
- immValue = rmRel->as<Mem>().offset();
- if (x86GetMovAbsAddrType(this, writer, o1.size(), options, rmRel->as<Mem>()) == BaseMem::kAddrTypeAbs) {
+ if (x86ShouldUseMovabs(this, writer, o1.size(), options, rmRel->as<Mem>())) {
opcode += 0xA2;
+ immValue = rmRel->as<Mem>().offset();
goto EmitX86OpMovAbs;
}
}
+ if (o1.size() == 1)
+ FIXUP_GPB(o1, opReg);
+
opcode += 0x88;
goto EmitX86M;
}
@@ -1648,8 +1715,8 @@ CaseX86M_GPB_MulDiv:
immValue = o1.as<Imm>().value();
// Optimize the instruction size by using a 32-bit immediate if possible.
- if (immSize == 8 && !(options & Inst::kOptionLongForm)) {
- if (Support::isUInt32(immValue) && hasEncodingOption(kEncodingOptionOptimizeForSize)) {
+ if (immSize == 8 && !Support::test(options, InstOptions::kLongForm)) {
+ if (Support::isUInt32(immValue) && hasEncodingOption(EncodingOptions::kOptimizeForSize)) {
// Zero-extend by using a 32-bit GPD destination instead of a 64-bit GPQ.
immSize = 4;
}
@@ -1687,6 +1754,62 @@ CaseX86M_GPB_MulDiv:
}
break;
+ case InstDB::kEncodingX86Movabs:
+ // Reg <- Mem
+ if (isign3 == ENC_OPS2(Reg, Mem)) {
+ opReg = o0.id();
+ rmRel = &o1;
+
+ opcode = 0xA0;
+ opcode.addArithBySize(o0.size());
+
+ if (ASMJIT_UNLIKELY(!o0.as<Reg>().isGp()) || opReg != Gp::kIdAx)
+ goto InvalidInstruction;
+
+ if (ASMJIT_UNLIKELY(rmRel->as<Mem>().hasBaseOrIndex()))
+ goto InvalidAddress;
+
+ if (ASMJIT_UNLIKELY(rmRel->as<Mem>().addrType() == Mem::AddrType::kRel))
+ goto InvalidAddress;
+
+ immValue = rmRel->as<Mem>().offset();
+ goto EmitX86OpMovAbs;
+ }
+
+ // Mem <- Reg
+ if (isign3 == ENC_OPS2(Mem, Reg)) {
+ opReg = o1.id();
+ rmRel = &o0;
+
+ opcode = 0xA2;
+ opcode.addArithBySize(o1.size());
+
+ if (ASMJIT_UNLIKELY(!o1.as<Reg>().isGp()) || opReg != Gp::kIdAx)
+ goto InvalidInstruction;
+
+ if (ASMJIT_UNLIKELY(rmRel->as<Mem>().hasBaseOrIndex()))
+ goto InvalidAddress;
+
+ immValue = rmRel->as<Mem>().offset();
+ goto EmitX86OpMovAbs;
+ }
+
+ // Reg <- Imm.
+ if (isign3 == ENC_OPS2(Reg, Imm)) {
+ if (ASMJIT_UNLIKELY(!o0.as<Reg>().isGpq()))
+ goto InvalidInstruction;
+
+ opReg = o0.id();
+ opcode = 0xB8;
+
+ immSize = 8;
+ immValue = o1.as<Imm>().value();
+
+ opcode.addPrefixBySize(8);
+ goto EmitX86OpReg;
+ }
+ break;
+
case InstDB::kEncodingX86MovsxMovzx:
opcode.add(o1.size() != 1);
opcode.addPrefixBySize(o0.size());
@@ -1794,7 +1917,7 @@ CaseX86M_GPB_MulDiv:
immValue = o0.as<Imm>().value();
immSize = 4;
- if (Support::isInt8(immValue) && !(options & Inst::kOptionLongForm))
+ if (Support::isInt8(immValue) && !Support::test(options, InstOptions::kLongForm))
immSize = 1;
opcode = immSize == 1 ? 0x6A : 0x68;
@@ -1810,13 +1933,12 @@ CaseX86M_GPB_MulDiv:
goto InvalidSegment;
opcode = x86OpcodePopSReg[segment];
- goto EmitDone;
+ goto EmitX86Op;
}
else {
CaseX86PushPop_Gp:
- // We allow 2 byte, 4 byte, and 8 byte register sizes, although PUSH
- // and POP only allow 2 bytes or native size. On 64-bit we simply
- // PUSH/POP 64-bit register even if 32-bit register was given.
+ // We allow 2 byte, 4 byte, and 8 byte register sizes, although PUSH and POP only allow 2 bytes or
+ // native size. On 64-bit we simply PUSH/POP 64-bit register even if 32-bit register was given.
if (ASMJIT_UNLIKELY(o0.size() < 2))
goto InvalidInstruction;
@@ -1849,7 +1971,7 @@ CaseX86PushPop_Gp:
if (isign3 == ENC_OPS1(Imm)) {
immValue = o0.as<Imm>().value();
- if (immValue == 0 && !(options & Inst::kOptionLongForm)) {
+ if (immValue == 0 && !Support::test(options, InstOptions::kLongForm)) {
// 'ret' without immediate, change C2 to C3.
opcode.add(1);
goto EmitX86Op;
@@ -1881,7 +2003,7 @@ CaseX86PushPop_Gp:
immValue = o1.as<Imm>().value() & 0xFF;
immSize = 0;
- if (immValue == 1 && !(options & Inst::kOptionLongForm))
+ if (immValue == 1 && !Support::test(options, InstOptions::kLongForm))
goto EmitX86R;
opcode -= 0x10;
@@ -1890,6 +2012,8 @@ CaseX86PushPop_Gp:
}
}
else {
+ if (ASMJIT_UNLIKELY(o0.size() == 0))
+ goto AmbiguousOperandSize;
opcode.addArithBySize(o0.size());
if (isign3 == ENC_OPS2(Mem, Reg)) {
@@ -1902,14 +2026,11 @@ CaseX86PushPop_Gp:
}
if (isign3 == ENC_OPS2(Mem, Imm)) {
- if (ASMJIT_UNLIKELY(o0.size() == 0))
- goto AmbiguousOperandSize;
-
rmRel = &o0;
immValue = o1.as<Imm>().value() & 0xFF;
immSize = 0;
- if (immValue == 1 && !(options & Inst::kOptionLongForm))
+ if (immValue == 1 && !Support::test(options, InstOptions::kLongForm))
goto EmitX86M;
opcode -= 0x10;
@@ -2060,7 +2181,7 @@ CaseX86PushPop_Gp:
// The following instructions use the secondary opcode.
opcode = x86AltOpcodeOf(instInfo);
- opReg = opcode.extractO();
+ opReg = opcode.extractModO();
if (isign3 == ENC_OPS2(Reg, Imm)) {
opcode.addArithBySize(o0.size());
@@ -2077,7 +2198,7 @@ CaseX86PushPop_Gp:
}
// Short form - AL, AX, EAX, RAX.
- if (rbReg == 0 && !(options & Inst::kOptionLongForm)) {
+ if (rbReg == 0 && !Support::test(options, InstOptions::kLongForm)) {
opcode &= Opcode::kPP_66 | Opcode::kW;
opcode |= 0xA8 + (o0.size() != 1);
goto EmitX86Op;
@@ -2115,30 +2236,45 @@ CaseX86PushPop_Gp:
case InstDB::kEncodingX86Xadd:
if (isign3 == ENC_OPS2(Reg, Reg)) {
- opcode.addArithBySize(o0.size());
rbReg = o0.id();
opReg = o1.id();
- if (o0.size() != o1.size())
+ uint32_t opSize = o0.size();
+ if (opSize != o1.size())
goto OperandSizeMismatch;
- if (o0.size() == 1) {
+ if (opSize == 1) {
FIXUP_GPB(o0, rbReg);
FIXUP_GPB(o1, opReg);
goto EmitX86R;
}
- // Special opcode for 'xchg ?ax, reg'.
+ // Special cases for 'xchg ?ax, reg'.
if (instId == Inst::kIdXchg && (opReg == 0 || rbReg == 0)) {
- opcode &= Opcode::kPP_66 | Opcode::kW;
- opcode |= 0x90;
- // One of `xchg a, b` or `xchg b, a` is AX/EAX/RAX.
- opReg += rbReg;
- goto EmitX86OpReg;
- }
- else {
- goto EmitX86R;
+ if (is64Bit() && opReg == rbReg && opSize >= 4) {
+ if (opSize == 8) {
+ // Encode 'xchg rax, rax' as '90' (REX and other prefixes are optional).
+ opcode &= Opcode::kW;
+ opcode |= 0x90;
+ goto EmitX86OpReg;
+ }
+ else {
+ // Encode 'xchg eax, eax' by by using a generic path.
+ }
+ }
+ else if (!Support::test(options, InstOptions::kLongForm)) {
+ // The special encoding encodes only one register, which is non-zero.
+ opReg += rbReg;
+
+ opcode.addArithBySize(opSize);
+ opcode &= Opcode::kW | Opcode::kPP_66;
+ opcode |= 0x90;
+ goto EmitX86OpReg;
+ }
}
+
+ opcode.addArithBySize(opSize);
+ goto EmitX86R;
}
if (isign3 == ENC_OPS2(Mem, Reg)) {
@@ -2164,7 +2300,7 @@ CaseX86PushPop_Gp:
rbReg = o1.id();
// ModRM encoding:
- if (!(options & Inst::kOptionModMR))
+ if (!Support::test(options, InstOptions::kX86_ModMR))
goto EmitX86R;
// ModMR encoding:
@@ -2188,9 +2324,8 @@ CaseX86PushPop_Gp:
}
break;
- // ------------------------------------------------------------------------
- // [FPU]
- // ------------------------------------------------------------------------
+ // FPU Instructions
+ // ----------------
case InstDB::kEncodingFpuOp:
goto EmitFpuOp;
@@ -2250,18 +2385,18 @@ CaseFpuArith_Mem:
if (isign3 == ENC_OPS1(Mem)) {
rmRel = &o0;
- if (o0.size() == 4 && commonInfo->hasFlag(InstDB::kFlagFpuM32)) {
+ if (o0.size() == 4 && commonInfo->hasFlag(InstDB::InstFlags::kFpuM32)) {
goto EmitX86M;
}
- if (o0.size() == 8 && commonInfo->hasFlag(InstDB::kFlagFpuM64)) {
+ if (o0.size() == 8 && commonInfo->hasFlag(InstDB::InstFlags::kFpuM64)) {
opcode += 4;
goto EmitX86M;
}
- if (o0.size() == 10 && commonInfo->hasFlag(InstDB::kFlagFpuM80)) {
+ if (o0.size() == 10 && commonInfo->hasFlag(InstDB::InstFlags::kFpuM80)) {
opcode = x86AltOpcodeOf(instInfo);
- opReg = opcode.extractO();
+ opReg = opcode.extractModO();
goto EmitX86M;
}
}
@@ -2279,18 +2414,18 @@ CaseFpuArith_Mem:
opcode &= ~uint32_t(Opcode::kCDSHL_Mask);
rmRel = &o0;
- if (o0.size() == 2 && commonInfo->hasFlag(InstDB::kFlagFpuM16)) {
+ if (o0.size() == 2 && commonInfo->hasFlag(InstDB::InstFlags::kFpuM16)) {
opcode += 4;
goto EmitX86M;
}
- if (o0.size() == 4 && commonInfo->hasFlag(InstDB::kFlagFpuM32)) {
+ if (o0.size() == 4 && commonInfo->hasFlag(InstDB::InstFlags::kFpuM32)) {
goto EmitX86M;
}
- if (o0.size() == 8 && commonInfo->hasFlag(InstDB::kFlagFpuM64)) {
+ if (o0.size() == 8 && commonInfo->hasFlag(InstDB::InstFlags::kFpuM64)) {
opcode = x86AltOpcodeOf(instInfo) & ~uint32_t(Opcode::kCDSHL_Mask);
- opReg = opcode.extractO();
+ opReg = opcode.extractModO();
goto EmitX86M;
}
}
@@ -2328,9 +2463,8 @@ CaseFpuArith_Mem:
}
break;
- // ------------------------------------------------------------------------
- // [Ext]
- // ------------------------------------------------------------------------
+ // Ext Instructions (Legacy Extensions)
+ // ------------------------------------
case InstDB::kEncodingExtPextrw:
if (isign3 == ENC_OPS3(Reg, Reg, Imm)) {
@@ -2388,7 +2522,7 @@ CaseFpuArith_Mem:
opReg = o0.id();
rbReg = o1.id();
- if (!(options & Inst::kOptionModMR) || !instInfo->_altOpcodeIndex)
+ if (!Support::test(options, InstOptions::kX86_ModMR) || !instInfo->_altOpcodeIndex)
goto EmitX86R;
opcode = x86AltOpcodeOf(instInfo);
@@ -2484,7 +2618,7 @@ CaseExtMovd:
if (Reg::isMm(o0) && Reg::isMm(o1)) {
opcode = Opcode::k000F00 | 0x6F;
- if (!(options & Inst::kOptionModMR))
+ if (!Support::test(options, InstOptions::kX86_ModMR))
goto EmitX86R;
opcode += 0x10;
@@ -2496,7 +2630,7 @@ CaseExtMovd:
if (Reg::isXmm(o0) && Reg::isXmm(o1)) {
opcode = Opcode::kF30F00 | 0x7E;
- if (!(options & Inst::kOptionModMR))
+ if (!Support::test(options, InstOptions::kX86_ModMR))
goto EmitX86R;
opcode = Opcode::k660F00 | 0xD6;
@@ -2558,7 +2692,11 @@ CaseExtMovd:
goto CaseExtRm;
case InstDB::kEncodingExtRm_Wx:
- opcode.addWIf(Reg::isGpq(o0) || o1.size() == 8);
+ opcode.addWIf(o1.size() == 8);
+ ASMJIT_FALLTHROUGH;
+
+ case InstDB::kEncodingExtRm_Wx_GpqOnly:
+ opcode.addWIf(Reg::isGpq(o0));
ASMJIT_FALLTHROUGH;
case InstDB::kEncodingExtRm:
@@ -2609,7 +2747,7 @@ CaseExtRm:
// The following instruction uses the secondary opcode.
opcode = x86AltOpcodeOf(instInfo);
- opReg = opcode.extractO();
+ opReg = opcode.extractModO();
if (isign3 == ENC_OPS2(Reg, Imm)) {
immValue = o1.as<Imm>().value();
@@ -2639,7 +2777,7 @@ CaseExtRm:
// The following instruction uses the secondary opcode.
opcode = x86AltOpcodeOf(instInfo);
- opReg = opcode.extractO();
+ opReg = opcode.extractModO();
if (isign3 == ENC_OPS2(Reg, Imm)) {
opcode.add66hIf(Reg::isXmm(o0));
@@ -2690,9 +2828,8 @@ CaseExtRm:
}
break;
- // ------------------------------------------------------------------------
- // [Extrq / Insertq (SSE4A)]
- // ------------------------------------------------------------------------
+ // Extrq & Insertq (SSE4A)
+ // -----------------------
case InstDB::kEncodingExtExtrq:
opReg = o0.id();
@@ -2701,22 +2838,22 @@ CaseExtRm:
if (isign3 == ENC_OPS2(Reg, Reg))
goto EmitX86R;
- // The following instruction uses the secondary opcode.
- opcode = x86AltOpcodeOf(instInfo);
-
if (isign3 == ENC_OPS3(Reg, Imm, Imm)) {
+ // This variant of the instruction uses the secondary opcode.
+ opcode = x86AltOpcodeOf(instInfo);
+ rbReg = opReg;
+ opReg = opcode.extractModO();
+
immValue = (uint32_t(o1.as<Imm>().valueAs<uint8_t>()) ) +
(uint32_t(o2.as<Imm>().valueAs<uint8_t>()) << 8) ;
immSize = 2;
-
- rbReg = opcode.extractO();
goto EmitX86R;
}
break;
case InstDB::kEncodingExtInsertq: {
const Operand_& o3 = opExt[EmitterUtils::kOp3];
- const uint32_t isign4 = isign3 + (o3.opType() << 9);
+ const uint32_t isign4 = isign3 + (uint32_t(o3.opType()) << 9);
opReg = o0.id();
rbReg = o1.id();
@@ -2724,10 +2861,10 @@ CaseExtRm:
if (isign4 == ENC_OPS2(Reg, Reg))
goto EmitX86R;
- // The following instruction uses the secondary opcode.
- opcode = x86AltOpcodeOf(instInfo);
-
if (isign4 == ENC_OPS4(Reg, Reg, Imm, Imm)) {
+ // This variant of the instruction uses the secondary opcode.
+ opcode = x86AltOpcodeOf(instInfo);
+
immValue = (uint32_t(o2.as<Imm>().valueAs<uint8_t>()) ) +
(uint32_t(o3.as<Imm>().valueAs<uint8_t>()) << 8) ;
immSize = 2;
@@ -2736,9 +2873,8 @@ CaseExtRm:
break;
}
- // ------------------------------------------------------------------------
- // [3dNow]
- // ------------------------------------------------------------------------
+ // 3DNOW Instructions
+ // ------------------
case InstDB::kEncodingExt3dNow:
// Every 3dNow instruction starts with 0x0F0F and the actual opcode is
@@ -2760,12 +2896,15 @@ CaseExtRm:
}
break;
- // ------------------------------------------------------------------------
- // [VEX/EVEX]
- // ------------------------------------------------------------------------
+ // VEX/EVEX Instructions
+ // ---------------------
case InstDB::kEncodingVexOp:
- goto EmitVexEvexOp;
+ goto EmitVexOp;
+
+ case InstDB::kEncodingVexOpMod:
+ rbReg = 0;
+ goto EmitVexEvexR;
case InstDB::kEncodingVexKmov:
if (isign3 == ENC_OPS2(Reg, Reg)) {
@@ -2785,7 +2924,7 @@ CaseExtRm:
}
// Form 'k, k'.
- if (!(options & Inst::kOptionModMR))
+ if (!Support::test(options, InstOptions::kX86_ModMR))
goto EmitVexEvexR;
opcode.add(1);
@@ -2825,7 +2964,6 @@ CaseExtRm:
case InstDB::kEncodingVexM_VM:
if (isign3 == ENC_OPS1(Mem)) {
- opcode |= x86OpcodeLByVMem(o0);
rmRel = &o0;
goto EmitVexEvexM;
}
@@ -2857,11 +2995,27 @@ CaseExtRm:
}
break;
+ case InstDB::kEncodingVexMri_Vpextrw:
+ // Use 'vpextrw reg, xmm1, i8' when possible.
+ if (isign3 == ENC_OPS3(Reg, Reg, Imm)) {
+ opcode = Opcode::k660F00 | 0xC5;
+
+ opReg = o0.id();
+ rbReg = o1.id();
+
+ immValue = o2.as<Imm>().value();
+ immSize = 1;
+ goto EmitVexEvexR;
+ }
+
+ goto CaseVexMri;
+
case InstDB::kEncodingVexMri_Lx:
opcode |= x86OpcodeLBySize(o0.size() | o1.size());
ASMJIT_FALLTHROUGH;
case InstDB::kEncodingVexMri:
+CaseVexMri:
immValue = o2.as<Imm>().value();
immSize = 1;
@@ -2889,6 +3043,13 @@ CaseExtRm:
opcode.addWIf(Reg::isGpq(o0) | Reg::isGpq(o1));
goto CaseVexRm;
+ case InstDB::kEncodingVexRm_Lx_Narrow:
+ if (o1.size())
+ opcode |= x86OpcodeLBySize(o1.size());
+ else if (o0.size() == 32)
+ opcode |= Opcode::kLL_2;
+ goto CaseVexRm;
+
case InstDB::kEncodingVexRm_Lx_Bcst:
if (isign3 == ENC_OPS2(Reg, Reg) && Reg::isGp(o1.as<Reg>())) {
opcode = x86AltOpcodeOf(instInfo) | x86OpcodeLBySize(o0.size() | o1.size());
@@ -2931,7 +3092,7 @@ CaseVexRm:
const Operand_& o4 = opExt[EmitterUtils::kOp4];
const Operand_& o5 = opExt[EmitterUtils::kOp5];
- if (Reg::isZmm(o0) && Reg::isZmm(o1) && Reg::isZmm(o2) && Reg::isZmm(o3) && Reg::isZmm(o4) && o5.isMem()) {
+ if (Reg::isVec(o0) && Reg::isVec(o1) && Reg::isVec(o2) && Reg::isVec(o3) && Reg::isVec(o4) && o5.isMem()) {
// Registers [o1, o2, o3, o4] must start aligned and must be consecutive.
uint32_t i1 = o1.id();
uint32_t i2 = o2.id();
@@ -2941,7 +3102,7 @@ CaseVexRm:
if (ASMJIT_UNLIKELY((i1 & 0x3) != 0 || i2 != i1 + 1 || i3 != i1 + 2 || i4 != i1 + 3))
goto NotConsecutiveRegs;
- opReg = o0.id();
+ opReg = x86PackRegAndVvvvv(o0.id(), i1);
rmRel = &o5;
goto EmitVexEvexM;
}
@@ -3002,11 +3163,42 @@ CaseVexRvm_R:
goto CaseVexRvm;
}
+ case InstDB::kEncodingVexRvm_Lx_KEvex: {
+ opcode.forceEvexIf(Reg::isKReg(o0));
+ ASMJIT_FALLTHROUGH;
+ }
+
case InstDB::kEncodingVexRvm_Lx: {
opcode |= x86OpcodeLBySize(o0.size() | o1.size());
goto CaseVexRvm;
}
+ case InstDB::kEncodingVexRvm_Lx_2xK: {
+ if (isign3 == ENC_OPS3(Reg, Reg, Reg)) {
+ // Two registers are encoded as a single register.
+ // - First K register must be even.
+ // - Second K register must be first+1.
+ if ((o0.id() & 1) != 0 || o0.id() + 1 != o1.id())
+ goto InvalidPhysId;
+
+ const Operand_& o3 = opExt[EmitterUtils::kOp3];
+
+ opcode |= x86OpcodeLBySize(o2.size());
+ opReg = x86PackRegAndVvvvv(o0.id(), o2.id());
+
+ if (o3.isReg()) {
+ rbReg = o3.id();
+ goto EmitVexEvexR;
+ }
+
+ if (o3.isMem()) {
+ rmRel = &o3;
+ goto EmitVexEvexM;
+ }
+ }
+ break;
+ }
+
case InstDB::kEncodingVexRvmr_Lx: {
opcode |= x86OpcodeLBySize(o0.size() | o1.size());
ASMJIT_FALLTHROUGH;
@@ -3014,7 +3206,7 @@ CaseVexRvm_R:
case InstDB::kEncodingVexRvmr: {
const Operand_& o3 = opExt[EmitterUtils::kOp3];
- const uint32_t isign4 = isign3 + (o3.opType() << 9);
+ const uint32_t isign4 = isign3 + (uint32_t(o3.opType()) << 9);
immValue = o3.id() << 4;
immSize = 1;
@@ -3033,13 +3225,23 @@ CaseVexRvm_R:
break;
}
+ case InstDB::kEncodingVexRvmi_KEvex:
+ opcode.forceEvexIf(Reg::isKReg(o0));
+ goto VexRvmi;
+
+ case InstDB::kEncodingVexRvmi_Lx_KEvex:
+ opcode.forceEvexIf(Reg::isKReg(o0));
+ ASMJIT_FALLTHROUGH;
+
case InstDB::kEncodingVexRvmi_Lx:
opcode |= x86OpcodeLBySize(o0.size() | o1.size());
ASMJIT_FALLTHROUGH;
- case InstDB::kEncodingVexRvmi: {
+ case InstDB::kEncodingVexRvmi:
+VexRvmi:
+ {
const Operand_& o3 = opExt[EmitterUtils::kOp3];
- const uint32_t isign4 = isign3 + (o3.opType() << 9);
+ const uint32_t isign4 = isign3 + (uint32_t(o3.opType()) << 9);
immValue = o3.as<Imm>().value();
immSize = 1;
@@ -3085,7 +3287,6 @@ CaseVexRvm_R:
rmRel = &o1;
goto EmitVexEvexM;
}
-
ASMJIT_FALLTHROUGH;
case InstDB::kEncodingVexRmv_VM:
@@ -3101,7 +3302,7 @@ CaseVexRvm_R:
case InstDB::kEncodingVexRmvi: {
const Operand_& o3 = opExt[EmitterUtils::kOp3];
- const uint32_t isign4 = isign3 + (o3.opType() << 9);
+ const uint32_t isign4 = isign3 + (uint32_t(o3.opType()) << 9);
immValue = o3.as<Imm>().value();
immSize = 1;
@@ -3207,7 +3408,7 @@ CaseVexRvm_R:
opReg = x86PackRegAndVvvvv(o0.id(), o2.id());
rbReg = o1.id();
- if (!(options & Inst::kOptionModMR))
+ if (!Support::test(options, InstOptions::kX86_ModMR))
goto EmitVexEvexR;
opcode.addW();
@@ -3272,7 +3473,7 @@ CaseVexRvm_R:
opReg = x86PackRegAndVvvvv(o0.id(), o2.id());
rbReg = o1.id();
- if (!(options & Inst::kOptionModMR))
+ if (!Support::test(options, InstOptions::kX86_ModMR))
goto EmitVexEvexR;
opcode.addW();
@@ -3370,6 +3571,10 @@ CaseVexRvm_R:
}
break;
+ case InstDB::kEncodingVexRvmVmi_Lx_MEvex:
+ opcode.forceEvexIf(o1.isMem());
+ ASMJIT_FALLTHROUGH;
+
case InstDB::kEncodingVexRvmVmi_Lx:
opcode |= x86OpcodeLBySize(o0.size() | o1.size());
ASMJIT_FALLTHROUGH;
@@ -3388,9 +3593,9 @@ CaseVexRvm_R:
}
// The following instruction uses the secondary opcode.
- opcode &= Opcode::kLL_Mask;
+ opcode &= Opcode::kLL_Mask | Opcode::kMM_ForceEvex;
opcode |= x86AltOpcodeOf(instInfo);
- opReg = opcode.extractO();
+ opReg = opcode.extractModO();
immValue = o2.as<Imm>().value();
immSize = 1;
@@ -3426,9 +3631,9 @@ CaseVexRvm_R:
}
break;
- case InstDB::kEncodingVexEvexVmi_Lx:
+ case InstDB::kEncodingVexVmi_Lx_MEvex:
if (isign3 == ENC_OPS3(Reg, Mem, Imm))
- opcode |= Opcode::kMM_ForceEvex;
+ opcode.forceEvex();
ASMJIT_FALLTHROUGH;
case InstDB::kEncodingVexVmi_Lx:
@@ -3465,7 +3670,7 @@ CaseVexVmi_AfterImm:
case InstDB::kEncodingVexRvrmRvmr: {
const Operand_& o3 = opExt[EmitterUtils::kOp3];
- const uint32_t isign4 = isign3 + (o3.opType() << 9);
+ const uint32_t isign4 = isign3 + (uint32_t(o3.opType()) << 9);
if (isign4 == ENC_OPS4(Reg, Reg, Reg, Reg)) {
opReg = x86PackRegAndVvvvv(o0.id(), o1.id());
@@ -3504,7 +3709,7 @@ CaseVexVmi_AfterImm:
if (ASMJIT_UNLIKELY(!o4.isImm()))
goto InvalidInstruction;
- const uint32_t isign4 = isign3 + (o3.opType() << 9);
+ const uint32_t isign4 = isign3 + (uint32_t(o3.opType()) << 9);
opcode |= x86OpcodeLBySize(o0.size() | o1.size() | o2.size() | o3.size());
immValue = o4.as<Imm>().valueAs<uint8_t>() & 0x0F;
@@ -3556,9 +3761,8 @@ CaseVexVmi_AfterImm:
}
break;
- // ------------------------------------------------------------------------
- // [FMA4]
- // ------------------------------------------------------------------------
+ // FMA4 Instructions
+ // -----------------
case InstDB::kEncodingFma4_Lx:
// It's fine to just check the first operand, second is just for sanity.
@@ -3567,15 +3771,28 @@ CaseVexVmi_AfterImm:
case InstDB::kEncodingFma4: {
const Operand_& o3 = opExt[EmitterUtils::kOp3];
- const uint32_t isign4 = isign3 + (o3.opType() << 9);
+ const uint32_t isign4 = isign3 + (uint32_t(o3.opType()) << 9);
if (isign4 == ENC_OPS4(Reg, Reg, Reg, Reg)) {
opReg = x86PackRegAndVvvvv(o0.id(), o1.id());
- rbReg = o2.id();
- immValue = o3.id() << 4;
- immSize = 1;
- goto EmitVexEvexR;
+ if (!Support::test(options, InstOptions::kX86_ModMR)) {
+ // MOD/RM - Encoding preferred by LLVM.
+ opcode.addW();
+ rbReg = o3.id();
+
+ immValue = o2.id() << 4;
+ immSize = 1;
+ goto EmitVexEvexR;
+ }
+ else {
+ // MOD/MR - Alternative encoding.
+ rbReg = o2.id();
+
+ immValue = o3.id() << 4;
+ immSize = 1;
+ goto EmitVexEvexR;
+ }
}
if (isign4 == ENC_OPS4(Reg, Reg, Reg, Mem)) {
@@ -3598,13 +3815,54 @@ CaseVexVmi_AfterImm:
}
break;
}
+
+ // AMX Instructions
+ // ----------------
+
+ case InstDB::kEncodingAmxCfg:
+ if (isign3 == ENC_OPS1(Mem)) {
+ rmRel = &o0;
+ goto EmitVexEvexM;
+ }
+ break;
+
+ case InstDB::kEncodingAmxR:
+ if (isign3 == ENC_OPS1(Reg)) {
+ opReg = o0.id();
+ rbReg = 0;
+ goto EmitVexEvexR;
+ }
+ break;
+
+ case InstDB::kEncodingAmxRm:
+ if (isign3 == ENC_OPS2(Reg, Mem)) {
+ opReg = o0.id();
+ rmRel = &o1;
+ goto EmitVexEvexM;
+ }
+ break;
+
+ case InstDB::kEncodingAmxMr:
+ if (isign3 == ENC_OPS2(Mem, Reg)) {
+ opReg = o1.id();
+ rmRel = &o0;
+ goto EmitVexEvexM;
+ }
+ break;
+
+ case InstDB::kEncodingAmxRmv:
+ if (isign3 == ENC_OPS3(Reg, Reg, Reg)) {
+ opReg = x86PackRegAndVvvvv(o0.id(), o2.id());
+ rbReg = o1.id();
+ goto EmitVexEvexR;
+ }
+ break;
}
goto InvalidInstruction;
- // --------------------------------------------------------------------------
- // [Emit - X86]
- // --------------------------------------------------------------------------
+ // Emit - X86 Opcode
+ // -----------------
EmitX86OpMovAbs:
immSize = FastUInt8(registerSize());
@@ -3628,6 +3886,9 @@ EmitX86Op:
writer.emitImmediate(uint64_t(immValue), immSize);
goto EmitDone;
+ // Emit - X86 - Opcode + Reg
+ // -------------------------
+
EmitX86OpReg:
// Emit mandatory instruction prefix.
writer.emitPP(opcode.v);
@@ -3649,8 +3910,10 @@ EmitX86OpReg:
writer.emitImmediate(uint64_t(immValue), immSize);
goto EmitDone;
+ // Emit - X86 - Opcode with Implicit <mem> Operand
+ // -----------------------------------------------
+
EmitX86OpImplicitMem:
- // NOTE: Don't change the emit order here, it's compatible with KeyStone/LLVM.
rmInfo = x86MemInfo[rmRel->as<Mem>().baseAndIndexTypes()];
if (ASMJIT_UNLIKELY(rmRel->as<Mem>().hasOffset() || (rmInfo & kX86MemInfo_Index)))
goto InvalidInstruction;
@@ -3667,19 +3930,61 @@ EmitX86OpImplicitMem:
writer.emit8If(rex | kX86ByteRex, rex != 0);
}
+ // Emit override prefixes.
writer.emitSegmentOverride(rmRel->as<Mem>().segmentId());
writer.emitAddressOverride((rmInfo & _addressOverrideMask()) != 0);
// Emit instruction opcodes.
writer.emitMMAndOpcode(opcode.v);
+
+ // Emit immediate value.
writer.emitImmediate(uint64_t(immValue), immSize);
goto EmitDone;
+ // Emit - X86 - Opcode /r - Register
+ // ---------------------------------
+
EmitX86R:
// Mandatory instruction prefix.
writer.emitPP(opcode.v);
- // Rex prefix (64-bit only).
+ // Emit REX prefix (64-bit only).
+ {
+ uint32_t rex = opcode.extractRex(options) |
+ ((opReg & 0x08) >> 1) | // REX.R (0x04).
+ ((rbReg & 0x08) >> 3) ; // REX.B (0x01).
+
+ if (ASMJIT_UNLIKELY(x86IsRexInvalid(rex)))
+ goto InvalidRexPrefix;
+ rex &= ~kX86ByteInvalidRex & 0xFF;
+ writer.emit8If(rex | kX86ByteRex, rex != 0);
+
+ opReg &= 0x07;
+ rbReg &= 0x07;
+ }
+
+ // Emit instruction opcodes.
+ writer.emitMMAndOpcode(opcode.v);
+
+ // Emit ModR.
+ writer.emit8(x86EncodeMod(3, opReg, rbReg));
+
+ // Emit immediate value.
+ writer.emitImmediate(uint64_t(immValue), immSize);
+ goto EmitDone;
+
+ // Emit - X86 - Opcode /r - Memory Base
+ // ------------------------------------
+
+EmitX86RFromM:
+ rmInfo = x86MemInfo[rmRel->as<Mem>().baseAndIndexTypes()];
+ if (ASMJIT_UNLIKELY(rmRel->as<Mem>().hasOffset() || (rmInfo & kX86MemInfo_Index)))
+ goto InvalidInstruction;
+
+ // Emit mandatory instruction prefix.
+ writer.emitPP(opcode.v);
+
+ // Emit REX prefix (64-bit only).
{
uint32_t rex = opcode.extractRex(options) |
((opReg & 0x08) >> 1) | // REX.R (0x04).
@@ -3694,32 +3999,42 @@ EmitX86R:
rbReg &= 0x07;
}
- // Instruction opcodes.
+ // Emit override prefixes.
+ writer.emitSegmentOverride(rmRel->as<Mem>().segmentId());
+ writer.emitAddressOverride((rmInfo & _addressOverrideMask()) != 0);
+
+ // Emit instruction opcodes.
writer.emitMMAndOpcode(opcode.v);
- // ModR.
+
+ // Emit ModR/M.
writer.emit8(x86EncodeMod(3, opReg, rbReg));
+
+ // Emit immediate value.
writer.emitImmediate(uint64_t(immValue), immSize);
goto EmitDone;
+ // Emit - X86 - Opcode /r - memory Operand
+ // ---------------------------------------
+
EmitX86M:
// `rmRel` operand must be memory.
ASMJIT_ASSERT(rmRel != nullptr);
- ASMJIT_ASSERT(rmRel->opType() == Operand::kOpMem);
+ ASMJIT_ASSERT(rmRel->opType() == OperandType::kMem);
ASMJIT_ASSERT((opcode & Opcode::kCDSHL_Mask) == 0);
+ // Emit override prefixes.
rmInfo = x86MemInfo[rmRel->as<Mem>().baseAndIndexTypes()];
writer.emitSegmentOverride(rmRel->as<Mem>().segmentId());
memOpAOMark = writer.cursor();
writer.emitAddressOverride((rmInfo & _addressOverrideMask()) != 0);
- // Mandatory instruction prefix.
+ // Emit mandatory instruction prefix.
writer.emitPP(opcode.v);
+ // Emit REX prefix (64-bit only).
rbReg = rmRel->as<Mem>().baseId();
rxReg = rmRel->as<Mem>().indexId();
-
- // REX prefix (64-bit only).
{
uint32_t rex;
@@ -3738,13 +4053,13 @@ EmitX86M:
opReg &= 0x07;
}
- // Instruction opcodes.
+ // Emit instruction opcodes.
writer.emitMMAndOpcode(opcode.v);
+
// ... Fall through ...
- // --------------------------------------------------------------------------
- // [Emit - MOD/SIB]
- // --------------------------------------------------------------------------
+ // Emit - MOD/SIB
+ // --------------
EmitModSib:
if (!(rmInfo & (kX86MemInfo_Index | kX86MemInfo_67H_X86))) {
@@ -3754,25 +4069,28 @@ EmitModSib:
relOffset = rmRel->as<Mem>().offsetLo32();
uint32_t mod = x86EncodeMod(0, opReg, rbReg);
- if (rbReg == Gp::kIdSp) {
- // [XSP|R12].
- if (relOffset == 0) {
+ bool forceSIB = commonInfo->isTsibOp();
+
+ if (rbReg == Gp::kIdSp || forceSIB) {
+ // TSIB or [XSP|R12].
+ mod = (mod & 0xF8u) | 0x04u;
+ if (rbReg != Gp::kIdBp && relOffset == 0) {
writer.emit8(mod);
- writer.emit8(x86EncodeSib(0, 4, 4));
+ writer.emit8(x86EncodeSib(0, 4, rbReg));
}
- // [XSP|R12 + DISP8|DISP32].
+ // TSIB or [XSP|R12 + DISP8|DISP32].
else {
uint32_t cdShift = (opcode & Opcode::kCDSHL_Mask) >> Opcode::kCDSHL_Shift;
int32_t cdOffset = relOffset >> cdShift;
if (Support::isInt8(cdOffset) && relOffset == int32_t(uint32_t(cdOffset) << cdShift)) {
writer.emit8(mod + 0x40); // <- MOD(1, opReg, rbReg).
- writer.emit8(x86EncodeSib(0, 4, 4));
+ writer.emit8(x86EncodeSib(0, 4, rbReg));
writer.emit8(cdOffset & 0xFF);
}
else {
writer.emit8(mod + 0x80); // <- MOD(2, opReg, rbReg).
- writer.emit8(x86EncodeSib(0, 4, 4));
+ writer.emit8(x86EncodeSib(0, 4, rbReg));
writer.emit32uLE(uint32_t(relOffset));
}
}
@@ -3798,12 +4116,12 @@ EmitModSib:
}
// ==========|> [ABSOLUTE | DISP32].
else if (!(rmInfo & (kX86MemInfo_BaseLabel | kX86MemInfo_BaseRip))) {
- uint32_t addrType = rmRel->as<Mem>().addrType();
+ Mem::AddrType addrType = rmRel->as<Mem>().addrType();
relOffset = rmRel->as<Mem>().offsetLo32();
if (is32Bit()) {
// Explicit relative addressing doesn't work in 32-bit mode.
- if (ASMJIT_UNLIKELY(addrType == BaseMem::kAddrTypeRel))
+ if (ASMJIT_UNLIKELY(addrType == Mem::AddrType::kRel))
goto InvalidAddress;
writer.emit8(x86EncodeMod(0, opReg, 5));
@@ -3814,14 +4132,13 @@ EmitModSib:
bool isOffsetU32 = rmRel->as<Mem>().offsetHi32() == 0;
uint64_t baseAddress = code()->baseAddress();
- // If relative addressing was not explicitly set then we can try to guess.
- // By guessing we check some properties of the memory operand and try to
- // base the decision on the segment prefix and the address type.
- if (addrType == BaseMem::kAddrTypeDefault) {
+ // If relative addressing was not explicitly set then we can try to guess. By guessing we check some
+ // properties of the memory operand and try to base the decision on the segment prefix and the address type.
+ if (addrType == Mem::AddrType::kDefault) {
if (baseAddress == Globals::kNoBaseAddress) {
// Prefer absolute addressing mode if the offset is 32-bit.
- addrType = isOffsetI32 || isOffsetU32 ? BaseMem::kAddrTypeAbs
- : BaseMem::kAddrTypeRel;
+ addrType = isOffsetI32 || isOffsetU32 ? Mem::AddrType::kAbs
+ : Mem::AddrType::kRel;
}
else {
// Prefer absolute addressing mode if FS|GS segment override is present.
@@ -3829,30 +4146,30 @@ EmitModSib:
// Prefer absolute addressing mode if this is LEA with 32-bit immediate.
bool isLea32 = (instId == Inst::kIdLea) && (isOffsetI32 || isOffsetU32);
- addrType = hasFsGs || isLea32 ? BaseMem::kAddrTypeAbs
- : BaseMem::kAddrTypeRel;
+ addrType = hasFsGs || isLea32 ? Mem::AddrType::kAbs
+ : Mem::AddrType::kRel;
}
}
- if (addrType == BaseMem::kAddrTypeRel) {
+ if (addrType == Mem::AddrType::kRel) {
uint32_t kModRel32Size = 5;
uint64_t virtualOffset = uint64_t(writer.offsetFrom(_bufferData)) + immSize + kModRel32Size;
- if (baseAddress == Globals::kNoBaseAddress) {
+ if (baseAddress == Globals::kNoBaseAddress || _section->id() != 0) {
// Create a new RelocEntry as we cannot calculate the offset right now.
- err = _code->newRelocEntry(&re, RelocEntry::kTypeAbsToRel, 4);
+ err = _code->newRelocEntry(&re, RelocType::kAbsToRel);
if (ASMJIT_UNLIKELY(err))
goto Failed;
writer.emit8(x86EncodeMod(0, opReg, 5));
- writer.emit32uLE(0);
re->_sourceSectionId = _section->id();
re->_sourceOffset = offset();
- re->_leadingSize = uint8_t(writer.offsetFrom(_bufferPtr) - 4);
- re->_trailingSize = uint8_t(immSize);
+ re->_format.resetToSimpleValue(OffsetType::kSignedOffset, 4);
+ re->_format.setLeadingAndTrailingSize(writer.offsetFrom(_bufferPtr), immSize);
re->_payload = uint64_t(rmRel->as<Mem>().offset());
+ writer.emit32uLE(0);
writer.emitImmediate(uint64_t(immValue), immSize);
goto EmitDone;
}
@@ -3875,22 +4192,19 @@ EmitModSib:
}
}
- // Handle unsigned 32-bit address that doesn't work with sign extension.
- // Consider the following instructions:
+ // Handle unsigned 32-bit address that doesn't work with sign extension. Consider the following instructions:
//
// 1. lea rax, [-1] - Sign extended to 0xFFFFFFFFFFFFFFFF
// 2. lea rax, [0xFFFFFFFF] - Zero extended to 0x00000000FFFFFFFF
// 3. add rax, [-1] - Sign extended to 0xFFFFFFFFFFFFFFFF
// 4. add rax, [0xFFFFFFFF] - Zero extended to 0x00000000FFFFFFFF
//
- // Sign extension is naturally performed by the CPU so we don't have to
- // bother, however, zero extension requires address-size override prefix,
- // which we probably don't have at this moment. So to make the address
+ // Sign extension is naturally performed by the CPU so we don't have to bother, however, zero extension
+ // requires address-size override prefix, which we probably don't have at this moment. So to make the address
// valid we need to insert it at `memOpAOMark` if it's not already there.
//
- // If this is 'lea' instruction then it's possible to remove REX.W part
- // from REX prefix (if it's there), which would be one-byte shorter than
- // inserting address-size override.
+ // If this is 'lea' instruction then it's possible to remove REX.W part from REX prefix (if it's there), which
+ // would be one-byte shorter than inserting address-size override.
//
// NOTE: If we don't do this then these instructions are unencodable.
if (!isOffsetI32) {
@@ -3901,16 +4215,15 @@ EmitModSib:
// We only patch the existing code if we don't have address-size override.
if (*memOpAOMark != 0x67) {
if (instId == Inst::kIdLea) {
- // LEA: Remove REX.W, if present. This is easy as we know that 'lea'
- // doesn't use any PP prefix so if REX prefix was emitted it would be
- // at `memOpAOMark`.
+ // LEA: Remove REX.W, if present. This is easy as we know that 'lea' doesn't use any PP prefix so if REX
+ // prefix was emitted it would be at `memOpAOMark`.
uint32_t rex = *memOpAOMark;
if (rex & kX86ByteRex) {
rex &= (~kX86ByteRexW) & 0xFF;
*memOpAOMark = uint8_t(rex);
// We can remove the REX prefix completely if it was not forced.
- if (rex == kX86ByteRex && !(options & Inst::kOptionRex))
+ if (rex == kX86ByteRex && !Support::test(options, InstOptions::kX86_Rex))
writer.remove8(memOpAOMark);
}
}
@@ -3943,14 +4256,14 @@ EmitModSib_LabelRip_X86:
if (ASMJIT_UNLIKELY(!label))
goto InvalidLabel;
- err = _code->newRelocEntry(&re, RelocEntry::kTypeRelToAbs, 4);
+ err = _code->newRelocEntry(&re, RelocType::kRelToAbs);
if (ASMJIT_UNLIKELY(err))
goto Failed;
re->_sourceSectionId = _section->id();
re->_sourceOffset = offset();
- re->_leadingSize = uint8_t(writer.offsetFrom(_bufferPtr));
- re->_trailingSize = uint8_t(immSize);
+ re->_format.resetToSimpleValue(OffsetType::kUnsignedOffset, 4);
+ re->_format.setLeadingAndTrailingSize(writer.offsetFrom(_bufferPtr), immSize);
re->_payload = uint64_t(int64_t(relOffset));
if (label->isBound()) {
@@ -3968,16 +4281,16 @@ EmitModSib_LabelRip_X86:
}
else {
// [RIP->ABS].
- err = _code->newRelocEntry(&re, RelocEntry::kTypeRelToAbs, 4);
+ err = _code->newRelocEntry(&re, RelocType::kRelToAbs);
if (ASMJIT_UNLIKELY(err))
goto Failed;
re->_sourceSectionId = _section->id();
re->_targetSectionId = _section->id();
+ re->_format.resetToSimpleValue(OffsetType::kUnsignedOffset, 4);
+ re->_format.setLeadingAndTrailingSize(writer.offsetFrom(_bufferPtr), immSize);
re->_sourceOffset = offset();
- re->_leadingSize = uint8_t(writer.offsetFrom(_bufferPtr));
- re->_trailingSize = uint8_t(immSize);
- re->_payload = re->_sourceOffset + re->_leadingSize + 4 + re->_trailingSize + uint64_t(int64_t(relOffset));
+ re->_payload = re->_sourceOffset + re->_format.regionSize() + uint64_t(int64_t(relOffset));
writer.emit32uLE(0);
}
@@ -4010,8 +4323,8 @@ EmitModSib_LabelRip_X86:
}
}
else if (!(rmInfo & kX86MemInfo_67H_X86)) {
- // ESP|RSP can't be used as INDEX in pure SIB mode, however, VSIB mode
- // allows XMM4|YMM4|ZMM4 (that's why the check is before the label).
+ // ESP|RSP can't be used as INDEX in pure SIB mode, however, VSIB mode allows XMM4|YMM4|ZMM4 (that's why the
+ // check is before the label).
if (ASMJIT_UNLIKELY(rxReg == Gp::kIdSp))
goto InvalidAddressIndex;
@@ -4075,10 +4388,9 @@ EmitModVSib:
// 16-bit address mode (32-bit mode with 67 override prefix).
relOffset = (int32_t(rmRel->as<Mem>().offsetLo32()) << 16) >> 16;
- // NOTE: 16-bit addresses don't use SIB byte and their encoding differs. We
- // use a table-based approach to calculate the proper MOD byte as it's easier.
- // Also, not all BASE [+ INDEX] combinations are supported in 16-bit mode, so
- // this may fail.
+ // NOTE: 16-bit addresses don't use SIB byte and their encoding differs. We use a table-based approach to
+ // calculate the proper MOD byte as it's easier. Also, not all BASE [+ INDEX] combinations are supported
+ // in 16-bit mode, so this may fail.
const uint32_t kBaseGpIdx = (kX86MemInfo_BaseGp | kX86MemInfo_Index);
if (rmInfo & kBaseGpIdx) {
@@ -4130,9 +4442,8 @@ EmitModVSib:
writer.emitImmediate(uint64_t(immValue), immSize);
goto EmitDone;
- // --------------------------------------------------------------------------
- // [Emit - FPU]
- // --------------------------------------------------------------------------
+ // Emit - FPU
+ // ----------
EmitFpuOp:
// Mandatory instruction prefix.
@@ -4143,31 +4454,29 @@ EmitFpuOp:
writer.emit8(opcode.v);
goto EmitDone;
- // --------------------------------------------------------------------------
- // [Emit - VEX / EVEX]
- // --------------------------------------------------------------------------
+ // Emit - VEX Opcode
+ // -----------------
-EmitVexEvexOp:
+EmitVexOp:
{
// These don't use immediate.
ASMJIT_ASSERT(immSize == 0);
- // Only 'vzeroall' and 'vzeroupper' instructions use this encoding, they
- // don't define 'W' to be '1' so we can just check the 'mmmmm' field. Both
- // functions can encode by using VEX2 prefix so VEX3 is basically only used
+ // Only 'vzeroall' and 'vzeroupper' instructions use this encoding, they don't define 'W' to be '1' so we can
+ // just check the 'mmmmm' field. Both functions can encode by using VEX2 prefix so VEX3 is basically only used
// when specified as instruction option.
ASMJIT_ASSERT((opcode & Opcode::kW) == 0);
- uint32_t x = ((opcode & Opcode::kMM_Mask ) >> (Opcode::kMM_Shift )) |
- ((opcode & Opcode::kLL_Mask ) >> (Opcode::kLL_Shift - 10)) |
- ((opcode & Opcode::kPP_VEXMask ) >> (Opcode::kPP_Shift - 8)) |
- ((options & Inst::kOptionVex3 ) >> (Opcode::kMM_Shift )) ;
- if (x & 0x04u) {
- x = (x & (0x4 ^ 0xFFFF)) << 8; // [00000000|00000Lpp|0000m0mm|00000000].
- x ^= (kX86ByteVex3) | // [........|00000Lpp|0000m0mm|__VEX3__].
- (0x07u << 13) | // [........|00000Lpp|1110m0mm|__VEX3__].
- (0x0Fu << 19) | // [........|01111Lpp|1110m0mm|__VEX3__].
- (opcode << 24) ; // [_OPCODE_|01111Lpp|1110m0mm|__VEX3__].
+ uint32_t x = (uint32_t(opcode & Opcode::kMM_Mask ) >> (Opcode::kMM_Shift )) |
+ (uint32_t(opcode & Opcode::kLL_Mask ) >> (Opcode::kLL_Shift - 10)) |
+ (uint32_t(opcode & Opcode::kPP_VEXMask ) >> (Opcode::kPP_Shift - 8)) ;
+
+ if (Support::test(options, InstOptions::kX86_Vex3)) {
+ x = (x & 0xFFFF) << 8; // [00000000|00000Lpp|000mmmmm|00000000].
+ x ^= (kX86ByteVex3) | // [........|00000Lpp|000mmmmm|__VEX3__].
+ (0x07u << 13) | // [........|00000Lpp|111mmmmm|__VEX3__].
+ (0x0Fu << 19) | // [........|01111Lpp|111mmmmm|__VEX3__].
+ (opcode << 24) ; // [_OPCODE_|01111Lpp|111mmmmm|__VEX3__].
writer.emit32uLE(x);
goto EmitDone;
@@ -4181,67 +4490,83 @@ EmitVexEvexOp:
}
}
+ // Emit - VEX|EVEX - /r - Register
+ // -------------------------------
+
EmitVexEvexR:
{
// Construct `x` - a complete EVEX|VEX prefix.
- uint32_t x = ((opReg << 4) & 0xF980u) | // [........|........|Vvvvv..R|R.......].
- ((rbReg << 2) & 0x0060u) | // [........|........|........|.BB.....].
- (opcode.extractLLMM(options)) | // [........|.LL.....|Vvvvv..R|RBBmmmmm].
- (_extraReg.id() << 16); // [........|.LL..aaa|Vvvvv..R|RBBmmmmm].
+ uint32_t x = ((opReg << 4) & 0xF980u) | // [........|........|Vvvvv..R|R.......].
+ ((rbReg << 2) & 0x0060u) | // [........|........|........|.BB.....].
+ (opcode.extractLLMMMMM(options)) | // [........|.LL.....|Vvvvv..R|RBBmmmmm].
+ (_extraReg.id() << 16); // [........|.LL..aaa|Vvvvv..R|RBBmmmmm].
opReg &= 0x7;
// Handle AVX512 options by a single branch.
- const uint32_t kAvx512Options = Inst::kOptionZMask | Inst::kOptionER | Inst::kOptionSAE;
- if (options & kAvx512Options) {
- uint32_t kBcstMask = 0x1 << 20;
- uint32_t kLLMask10 = 0x2 << 21;
- uint32_t kLLMask11 = 0x3 << 21;
+ const InstOptions kAvx512Options = InstOptions::kX86_ZMask | InstOptions::kX86_ER | InstOptions::kX86_SAE;
+ if (Support::test(options, kAvx512Options)) {
+ static constexpr uint32_t kBcstMask = 0x1 << 20;
+ static constexpr uint32_t kLLMask10 = 0x2 << 21;
+ static constexpr uint32_t kLLMask11 = 0x3 << 21;
- // Designed to be easily encodable so the position must be exact.
- // The {rz-sae} is encoded as {11}, so it should match the mask.
- ASMJIT_ASSERT(Inst::kOptionRZ_SAE == kLLMask11);
+ // Designed to be easily encodable so the position must be exact. The {rz-sae} is encoded as {11},
+ // so it should match the mask.
+ static_assert(uint32_t(InstOptions::kX86_RZ_SAE) == kLLMask11,
+ "This code requires InstOptions::X86_RZ_SAE to match kLLMask11 to work properly");
- x |= options & Inst::kOptionZMask; // [........|zLLb.aaa|Vvvvv..R|RBBmmmmm].
+ x |= uint32_t(options & InstOptions::kX86_ZMask); // [........|zLLb.aaa|Vvvvv..R|RBBmmmmm].
// Support embedded-rounding {er} and suppress-all-exceptions {sae}.
- if (options & (Inst::kOptionER | Inst::kOptionSAE)) {
- // Embedded rounding is only encodable if the instruction is either
- // scalar or it's a 512-bit operation as the {er} rounding predicate
- // collides with LL part of the instruction.
+ if (Support::test(options, InstOptions::kX86_ER | InstOptions::kX86_SAE)) {
+ // Embedded rounding is only encodable if the instruction is either scalar or it's a 512-bit
+ // operation as the {er} rounding predicate collides with LL part of the instruction.
if ((x & kLLMask11) != kLLMask10) {
- // Ok, so LL is not 10, thus the instruction must be scalar.
- // Scalar instructions don't support broadcast so if this
- // instruction supports it {er} nor {sae} would be encodable.
+ // Ok, so LL is not 10, thus the instruction must be scalar. Scalar instructions don't
+ // support broadcast so if this instruction supports it {er} nor {sae} would be encodable.
if (ASMJIT_UNLIKELY(commonInfo->hasAvx512B()))
goto InvalidEROrSAE;
}
- if (options & Inst::kOptionER) {
+ if (Support::test(options, InstOptions::kX86_ER)) {
if (ASMJIT_UNLIKELY(!commonInfo->hasAvx512ER()))
goto InvalidEROrSAE;
- x &=~kLLMask11; // [........|.00..aaa|Vvvvv..R|RBBmmmmm].
- x |= kBcstMask | (options & kLLMask11); // [........|.LLb.aaa|Vvvvv..R|RBBmmmmm].
+ x &=~kLLMask11; // [........|.00..aaa|Vvvvv..R|RBBmmmmm].
+ x |= kBcstMask | (uint32_t(options) & kLLMask11); // [........|.LLb.aaa|Vvvvv..R|RBBmmmmm].
}
else {
if (ASMJIT_UNLIKELY(!commonInfo->hasAvx512SAE()))
goto InvalidEROrSAE;
- x |= kBcstMask; // [........|.LLb.aaa|Vvvvv..R|RBBmmmmm].
+ x &=~kLLMask11; // [........|.00..aaa|Vvvvv..R|RBBmmmmm].
+ x |= kBcstMask; // [........|.00b.aaa|Vvvvv..R|RBBmmmmm].
}
}
}
- // Check if EVEX is required by checking bits in `x` : [........|xx.x.xxx|x......x|.x.x....].
- if (x & 0x00D78150u) {
- uint32_t y = ((x << 4) & 0x00080000u) | // [........|...bV...|........|........].
- ((x >> 4) & 0x00000010u) ; // [........|...bV...|........|...R....].
- x = (x & 0x00FF78E3u) | y; // [........|zLLbVaaa|0vvvv000|RBBR00mm].
- x = x << 8; // [zLLbVaaa|0vvvv000|RBBR00mm|00000000].
- x |= (opcode >> kVSHR_W ) & 0x00800000u; // [zLLbVaaa|Wvvvv000|RBBR00mm|00000000].
- x |= (opcode >> kVSHR_PP_EW) & 0x00830000u; // [zLLbVaaa|Wvvvv0pp|RBBR00mm|00000000] (added PP and EVEX.W).
- // _ ____ ____
- x ^= 0x087CF000u | kX86ByteEvex; // [zLLbVaaa|Wvvvv1pp|RBBR00mm|01100010].
+ // These bits would force EVEX prefix.
+ constexpr uint32_t kEvexForce = 0x00000010u; // [........|........|........|...x....].
+ constexpr uint32_t kEvexBits = 0x00D78150u; // [........|xx.x.xxx|x......x|.x.x....].
+
+ // Force EVEX prefix even in case the instruction has VEX encoding, because EVEX encoding is preferred. At the
+ // moment this is only required by AVX_VNNI instructions, which were added after AVX512_VNNI instructions. If
+ // such instruction doesn't specify prefix, EVEX (AVX512_VNNI) is selected by default.
+ if (commonInfo->preferEvex()) {
+ if ((x & kEvexBits) == 0 && !Support::test(options, InstOptions::kX86_Vex | InstOptions::kX86_Vex3)) {
+ x |= kEvexForce;
+ }
+ }
+
+ // Check if EVEX is required by checking bits in `x` : [........|xx.x.xxx|x......x|.x.x....].
+ if (x & kEvexBits) {
+ uint32_t y = ((x << 4) & 0x00080000u) | // [........|...bV...|........|........].
+ ((x >> 4) & 0x00000010u) ; // [........|...bV...|........|...R....].
+ x = (x & 0x00FF78EFu) | y; // [........|zLLbVaaa|0vvvv000|RBBRmmmm].
+ x = x << 8; // [zLLbVaaa|0vvvv000|RBBRmmmm|00000000].
+ x |= (opcode >> kVSHR_W ) & 0x00800000u; // [zLLbVaaa|Wvvvv000|RBBRmmmm|00000000].
+ x |= (opcode >> kVSHR_PP_EW) & 0x00830000u; // [zLLbVaaa|Wvvvv0pp|RBBRmmmm|00000000] (added PP and EVEX.W).
+ // _ ____ ____
+ x ^= 0x087CF000u | kX86ByteEvex; // [zLLbVaaa|Wvvvv1pp|RBBRmmmm|01100010].
writer.emit32uLE(x);
writer.emit8(opcode.v);
@@ -4252,19 +4577,20 @@ EmitVexEvexR:
goto EmitDone;
}
- // Not EVEX, prepare `x` for VEX2 or VEX3: x = [........|00L00000|0vvvv000|R0B0mmmm].
- x |= ((opcode >> (kVSHR_W + 8)) & 0x8000u) | // [00000000|00L00000|Wvvvv000|R0B0mmmm].
- ((opcode >> (kVSHR_PP + 8)) & 0x0300u) | // [00000000|00L00000|0vvvv0pp|R0B0mmmm].
- ((x >> 11 ) & 0x0400u) ; // [00000000|00L00000|WvvvvLpp|R0B0mmmm].
+ // Not EVEX, prepare `x` for VEX2 or VEX3: x = [........|00L00000|0vvvv000|R0Bmmmmm].
+ x |= ((opcode >> (kVSHR_W + 8)) & 0x8000u) | // [00000000|00L00000|Wvvvv000|R0Bmmmmm].
+ ((opcode >> (kVSHR_PP + 8)) & 0x0300u) | // [00000000|00L00000|0vvvv0pp|R0Bmmmmm].
+ ((x >> 11 ) & 0x0400u) ; // [00000000|00L00000|WvvvvLpp|R0Bmmmmm].
+ x |= x86GetForceEvex3MaskInLastBit(options); // [x0000000|00L00000|WvvvvLpp|R0Bmmmmm].
- // Check if VEX3 is required / forced: [........|........|x.......|..x..x..].
- if (x & 0x0008024u) {
+ // Check if VEX3 is required / forced: [x.......|........|x.......|..xxxxx.].
+ if (x & 0x8000803Eu) {
uint32_t xorMsk = x86VEXPrefix[x & 0xF] | (opcode << 24);
- // Clear 'FORCE-VEX3' bit and all high bits.
- x = (x & (0x4 ^ 0xFFFF)) << 8; // [00000000|WvvvvLpp|R0B0m0mm|00000000].
- // ____ _ _
- x ^= xorMsk; // [_OPCODE_|WvvvvLpp|R1Bmmmmm|VEX3|XOP].
+ // Clear all high bits.
+ x = (x & 0xFFFF) << 8; // [00000000|WvvvvLpp|R0Bmmmmm|00000000].
+ // ____ _ _
+ x ^= xorMsk; // [_OPCODE_|WvvvvLpp|R1Bmmmmm|VEX3|XOP].
writer.emit32uLE(x);
rbReg &= 0x7;
@@ -4288,9 +4614,12 @@ EmitVexEvexR:
}
}
+ // Emit - VEX|EVEX - /r - Memory
+ // -----------------------------
+
EmitVexEvexM:
ASMJIT_ASSERT(rmRel != nullptr);
- ASMJIT_ASSERT(rmRel->opType() == Operand::kOpMem);
+ ASMJIT_ASSERT(rmRel->opType() == OperandType::kMem);
rmInfo = x86MemInfo[rmRel->as<Mem>().baseAndIndexTypes()];
writer.emitSegmentOverride(rmRel->as<Mem>().segmentId());
@@ -4305,49 +4634,83 @@ EmitVexEvexM:
uint32_t broadcastBit = uint32_t(rmRel->as<Mem>().hasBroadcast());
// Construct `x` - a complete EVEX|VEX prefix.
- uint32_t x = ((opReg << 4) & 0x0000F980u) | // [........|........|Vvvvv..R|R.......].
- ((rxReg << 3) & 0x00000040u) | // [........|........|........|.X......].
- ((rxReg << 15) & 0x00080000u) | // [........|....X...|........|........].
- ((rbReg << 2) & 0x00000020u) | // [........|........|........|..B.....].
- opcode.extractLLMM(options) | // [........|.LL.X...|Vvvvv..R|RXBmmmmm].
- (_extraReg.id() << 16) | // [........|.LL.Xaaa|Vvvvv..R|RXBmmmmm].
- (broadcastBit << 20) ; // [........|.LLbXaaa|Vvvvv..R|RXBmmmmm].
+ uint32_t x = ((opReg << 4) & 0x0000F980u) | // [........|........|Vvvvv..R|R.......].
+ ((rxReg << 3) & 0x00000040u) | // [........|........|........|.X......].
+ ((rxReg << 15) & 0x00080000u) | // [........|....X...|........|........].
+ ((rbReg << 2) & 0x00000020u) | // [........|........|........|..B.....].
+ opcode.extractLLMMMMM(options) | // [........|.LL.X...|Vvvvv..R|RXBmmmmm].
+ (_extraReg.id() << 16) | // [........|.LL.Xaaa|Vvvvv..R|RXBmmmmm].
+ (broadcastBit << 20) ; // [........|.LLbXaaa|Vvvvv..R|RXBmmmmm].
opReg &= 0x07u;
- // Mark invalid VEX (force EVEX) case: // [@.......|.LLbXaaa|Vvvvv..R|RXBmmmmm].
- x |= (~commonInfo->flags() & InstDB::kFlagVex) << (31 - Support::constCtz(InstDB::kFlagVex));
+ // Mark invalid VEX (force EVEX) case: // [@.......|.LLbXaaa|Vvvvv..R|RXBmmmmm].
+ x |= uint32_t(~commonInfo->flags() & InstDB::InstFlags::kVex) << (31 - Support::ConstCTZ<uint32_t(InstDB::InstFlags::kVex)>::value);
// Handle AVX512 options by a single branch.
- const uint32_t kAvx512Options = Inst::kOptionZMask |
- Inst::kOptionER |
- Inst::kOptionSAE ;
- if (options & kAvx512Options) {
+ const InstOptions kAvx512Options = InstOptions::kX86_ZMask |
+ InstOptions::kX86_ER |
+ InstOptions::kX86_SAE ;
+ if (Support::test(options, kAvx512Options)) {
// {er} and {sae} are both invalid if memory operand is used.
- if (ASMJIT_UNLIKELY(options & (Inst::kOptionER | Inst::kOptionSAE)))
+ if (ASMJIT_UNLIKELY(Support::test(options, InstOptions::kX86_ER | InstOptions::kX86_SAE)))
goto InvalidEROrSAE;
- x |= options & (Inst::kOptionZMask); // [@.......|zLLbXaaa|Vvvvv..R|RXBmmmmm].
+ x |= uint32_t(options & InstOptions::kX86_ZMask); // [@.......|zLLbXaaa|Vvvvv..R|RXBmmmmm].
}
- // Check if EVEX is required by checking bits in `x` : [@.......|xx.xxxxx|x......x|...x....].
- if (x & 0x80DF8110u) {
- uint32_t y = ((x << 4) & 0x00080000u) | // [@.......|....V...|........|........].
- ((x >> 4) & 0x00000010u) ; // [@.......|....V...|........|...R....].
- x = (x & 0x00FF78E3u) | y; // [........|zLLbVaaa|0vvvv000|RXBR00mm].
- x = x << 8; // [zLLbVaaa|0vvvv000|RBBR00mm|00000000].
- x |= (opcode >> kVSHR_W ) & 0x00800000u; // [zLLbVaaa|Wvvvv000|RBBR00mm|00000000].
- x |= (opcode >> kVSHR_PP_EW) & 0x00830000u; // [zLLbVaaa|Wvvvv0pp|RBBR00mm|00000000] (added PP and EVEX.W).
- // _ ____ ____
- x ^= 0x087CF000u | kX86ByteEvex; // [zLLbVaaa|Wvvvv1pp|RBBR00mm|01100010].
+ // If these bits are used then EVEX prefix is required.
+ constexpr uint32_t kEvexForce = 0x00000010u; // [........|........|........|...x....].
+ constexpr uint32_t kEvexBits = 0x80DF8110u; // [@.......|xx.xxxxx|x......x|...x....].
- writer.emit32uLE(x);
- writer.emit8(opcode.v);
+ // Force EVEX prefix even in case the instruction has VEX encoding, because EVEX encoding is preferred. At the
+ // moment this is only required for AVX_VNNI instructions, which were added after AVX512_VNNI instructions. If
+ // such instruction doesn't specify prefix, EVEX (AVX512_VNNI) would be used by default,
+ if (commonInfo->preferEvex()) {
+ if ((x & kEvexBits) == 0 && !Support::test(options, InstOptions::kX86_Vex | InstOptions::kX86_Vex3)) {
+ x |= kEvexForce;
+ }
+ }
+
+ // Check if EVEX is required by checking bits in `x` : [@.......|xx.xxxxx|x......x|...x....].
+ if (x & kEvexBits) {
+ uint32_t y = ((x << 4) & 0x00080000u) | // [@.......|....V...|........|........].
+ ((x >> 4) & 0x00000010u) ; // [@.......|....V...|........|...R....].
+ x = (x & 0x00FF78EFu) | y; // [........|zLLbVaaa|0vvvv000|RXBRmmmm].
+ x = x << 8; // [zLLbVaaa|0vvvv000|RBBRmmmm|00000000].
+ x |= (opcode >> kVSHR_W ) & 0x00800000u; // [zLLbVaaa|Wvvvv000|RBBRmmmm|00000000].
+ x |= (opcode >> kVSHR_PP_EW) & 0x00830000u; // [zLLbVaaa|Wvvvv0pp|RBBRmmmm|00000000] (added PP and EVEX.W).
+ // _ ____ ____
+ x ^= 0x087CF000u | kX86ByteEvex; // [zLLbVaaa|Wvvvv1pp|RBBRmmmm|01100010].
if (x & 0x10000000u) {
- // Broadcast, change the compressed displacement scale to either x4 (SHL 2) or x8 (SHL 3)
- // depending on instruction's W. If 'W' is 1 'SHL' must be 3, otherwise it must be 2.
+ // Broadcast support.
+ //
+ // 1. Verify our LL field is correct as broadcast changes the "size" of the source operand. For example if
+ // a broadcasted operand is qword_ptr[X] {1to8} the source size becomes 64 and not 8 as the memory operand
+ // would report.
+ //
+ // 2. Change the compressed displacement scale to either x2 (SHL1), x4 (SHL 2), or x8 (SHL 3) depending on
+ // the broadcast unit/element size.
+ uint32_t broadcastUnitSize = commonInfo->broadcastSize();
+ uint32_t broadcastVectorSize = broadcastUnitSize << uint32_t(rmRel->as<Mem>().getBroadcast());
+
+ if (ASMJIT_UNLIKELY(broadcastUnitSize == 0))
+ goto InvalidBroadcast;
+
+ // LL was already shifted 8 bits right.
+ constexpr uint32_t kLLShift = 21 + 8;
+
+ uint32_t currentLL = x & (0x3u << kLLShift);
+ uint32_t broadcastLL = (Support::max<uint32_t>(Support::ctz(broadcastVectorSize), 4) - 4) << kLLShift;
+
+ if (broadcastLL > (2u << kLLShift))
+ goto InvalidBroadcast;
+
+ uint32_t newLL = Support::max(currentLL, broadcastLL);
+ x = (x & ~(uint32_t(0x3) << kLLShift)) | newLL;
+
opcode &=~uint32_t(Opcode::kCDSHL_Mask);
- opcode |= ((x & 0x00800000u) ? 3u : 2u) << Opcode::kCDSHL_Shift;
+ opcode |= Support::ctz(broadcastUnitSize) << Opcode::kCDSHL_Shift;
}
else {
// Add the compressed displacement 'SHF' to the opcode based on 'TTWLL'.
@@ -4357,24 +4720,28 @@ EmitVexEvexM:
((x >> 29) & 0x3);
opcode += x86CDisp8SHL[TTWLL];
}
+
+ writer.emit32uLE(x);
+ writer.emit8(opcode.v);
}
else {
- // Not EVEX, prepare `x` for VEX2 or VEX3: x = [........|00L00000|0vvvv000|RXB0mmmm].
- x |= ((opcode >> (kVSHR_W + 8)) & 0x8000u) | // [00000000|00L00000|Wvvvv000|RXB0mmmm].
- ((opcode >> (kVSHR_PP + 8)) & 0x0300u) | // [00000000|00L00000|Wvvvv0pp|RXB0mmmm].
- ((x >> 11 ) & 0x0400u) ; // [00000000|00L00000|WvvvvLpp|RXB0mmmm].
+ // Not EVEX, prepare `x` for VEX2 or VEX3: x = [........|00L00000|0vvvv000|RXBmmmmm].
+ x |= ((opcode >> (kVSHR_W + 8)) & 0x8000u) | // [00000000|00L00000|Wvvvv000|RXBmmmmm].
+ ((opcode >> (kVSHR_PP + 8)) & 0x0300u) | // [00000000|00L00000|Wvvvv0pp|RXBmmmmm].
+ ((x >> 11 ) & 0x0400u) ; // [00000000|00L00000|WvvvvLpp|RXBmmmmm].
+ x |= x86GetForceEvex3MaskInLastBit(options); // [x0000000|00L00000|WvvvvLpp|RXBmmmmm].
// Clear a possible CDisp specified by EVEX.
opcode &= ~Opcode::kCDSHL_Mask;
- // Check if VEX3 is required / forced: [........|........|x.......|.xx..x..].
- if (x & 0x0008064u) {
+ // Check if VEX3 is required / forced: [x.......|........|x.......|.xxxxxx.].
+ if (x & 0x8000807Eu) {
uint32_t xorMsk = x86VEXPrefix[x & 0xF] | (opcode << 24);
- // Clear 'FORCE-VEX3' bit and all high bits.
- x = (x & (0x4 ^ 0xFFFF)) << 8; // [00000000|WvvvvLpp|RXB0m0mm|00000000].
- // ____ ___
- x ^= xorMsk; // [_OPCODE_|WvvvvLpp|RXBmmmmm|VEX3_XOP].
+ // Clear all high bits.
+ x = (x & 0xFFFF) << 8; // [00000000|WvvvvLpp|RXBmmmmm|00000000].
+ // ____ ___
+ x ^= xorMsk; // [_OPCODE_|WvvvvLpp|RXBmmmmm|VEX3_XOP].
writer.emit32uLE(x);
}
else {
@@ -4390,7 +4757,7 @@ EmitVexEvexM:
}
// MOD|SIB address.
- if (!commonInfo->hasFlag(InstDB::kFlagVsib))
+ if (!commonInfo->hasFlag(InstDB::InstFlags::kVsib))
goto EmitModSib;
// MOD|VSIB address without INDEX is invalid.
@@ -4398,9 +4765,8 @@ EmitVexEvexM:
goto EmitModVSib;
goto InvalidInstruction;
- // --------------------------------------------------------------------------
- // [Emit - Jmp/Jcc/Call]
- // --------------------------------------------------------------------------
+ // Emit - Jmp/Jcc/Call
+ // -------------------
EmitJmpCall:
{
@@ -4440,7 +4806,7 @@ EmitJmpCall:
}
else {
// Non-bound label or label bound to a different section.
- if (opCode8 && (!opcode.v || (options & Inst::kOptionShortForm))) {
+ if (opCode8 && (!opcode.v || Support::test(options, InstOptions::kShortForm))) {
writer.emit8(opCode8);
// Record DISP8 (non-bound label).
@@ -4450,7 +4816,7 @@ EmitJmpCall:
}
else {
// Refuse also 'short' prefix, if specified.
- if (ASMJIT_UNLIKELY(!opcode.v || (options & Inst::kOptionShortForm) != 0))
+ if (ASMJIT_UNLIKELY(!opcode.v || Support::test(options, InstOptions::kShortForm)))
goto InvalidDisplacement;
writer.emit8If(0x0F, (opcode & Opcode::kMM_Mask) != 0);// Emit 0F prefix.
@@ -4469,9 +4835,8 @@ EmitJmpCall:
uint64_t baseAddress = code()->baseAddress();
uint64_t jumpAddress = rmRel->as<Imm>().valueAs<uint64_t>();
- // If the base-address is known calculate a relative displacement and
- // check if it fits in 32 bits (which is always true in 32-bit mode).
- // Emit relative displacement as it was a bound label if all checks are ok.
+ // If the base-address is known calculate a relative displacement and check if it fits in 32 bits (which is
+ // always true in 32-bit mode). Emit relative displacement as it was a bound label if all checks are ok.
if (baseAddress != Globals::kNoBaseAddress) {
uint64_t rel64 = jumpAddress - (ip + baseAddress) - inst32Size;
if (Environment::is32Bit(arch()) || Support::isInt32(int64_t(rel64))) {
@@ -4479,14 +4844,14 @@ EmitJmpCall:
goto EmitJmpCallRel;
}
else {
- // Relative displacement exceeds 32-bits - relocator can only
- // insert trampoline for jmp/call, but not for jcc/jecxz.
+ // Relative displacement exceeds 32-bits - relocator can only insert trampoline for jmp/call, but not
+ // for jcc/jecxz.
if (ASMJIT_UNLIKELY(!x86IsJmpOrCall(instId)))
goto InvalidDisplacement;
}
}
- err = _code->newRelocEntry(&re, RelocEntry::kTypeAbsToRel, 0);
+ err = _code->newRelocEntry(&re, RelocType::kAbsToRel);
if (ASMJIT_UNLIKELY(err))
goto Failed;
@@ -4495,10 +4860,9 @@ EmitJmpCall:
re->_payload = jumpAddress;
if (ASMJIT_LIKELY(opcode.v)) {
- // 64-bit: Emit REX prefix so the instruction can be patched later.
- // REX prefix does nothing if not patched, but allows to patch the
- // instruction to use MOD/M and to point to a memory where the final
- // 64-bit address is stored.
+ // 64-bit: Emit REX prefix so the instruction can be patched later. REX prefix does nothing if not patched,
+ // but allows to patch the instruction to use MOD/M and to point to a memory where the final 64-bit address
+ // is stored.
if (Environment::is64Bit(arch()) && x86IsJmpOrCall(instId)) {
if (!rex)
writer.emit8(kX86ByteRex);
@@ -4507,25 +4871,21 @@ EmitJmpCall:
if (ASMJIT_UNLIKELY(err))
goto Failed;
- re->_relocType = RelocEntry::kTypeX64AddressEntry;
+ re->_relocType = RelocType::kX64AddressEntry;
}
writer.emit8If(0x0F, (opcode & Opcode::kMM_Mask) != 0); // Emit 0F prefix.
writer.emit8(opcode.v); // Emit opcode.
writer.emit8If(x86EncodeMod(3, opReg, 0), opReg != 0); // Emit MOD.
+ re->_format.resetToSimpleValue(OffsetType::kSignedOffset, 4);
+ re->_format.setLeadingAndTrailingSize(writer.offsetFrom(_bufferPtr), immSize);
writer.emit32uLE(0); // Emit DISP32.
-
- re->_valueSize = 4;
- re->_leadingSize = uint8_t(writer.offsetFrom(_bufferPtr) - 4);
- re->_trailingSize = uint8_t(immSize);
}
else {
writer.emit8(opCode8); // Emit opcode.
+ re->_format.resetToSimpleValue(OffsetType::kSignedOffset, 1);
+ re->_format.setLeadingAndTrailingSize(writer.offsetFrom(_bufferPtr), immSize);
writer.emit8(0); // Emit DISP8 (zero).
-
- re->_valueSize = 1;
- re->_leadingSize = uint8_t(writer.offsetFrom(_bufferPtr) - 1);
- re->_trailingSize = uint8_t(immSize);
}
goto EmitDone;
}
@@ -4533,21 +4893,20 @@ EmitJmpCall:
// Not Label|Imm -> Invalid.
goto InvalidInstruction;
- // Emit jmp/call with relative displacement known at assembly-time. Decide
- // between 8-bit and 32-bit displacement encoding. Some instructions only
- // allow either 8-bit or 32-bit encoding, others allow both encodings.
+ // Emit jmp/call with relative displacement known at assembly-time. Decide between 8-bit and 32-bit displacement
+ // encoding. Some instructions only allow either 8-bit or 32-bit encoding, others allow both encodings.
EmitJmpCallRel:
- if (Support::isInt8(int32_t(rel32 + inst32Size - inst8Size)) && opCode8 && !(options & Inst::kOptionLongForm)) {
- options |= Inst::kOptionShortForm;
+ if (Support::isInt8(int32_t(rel32 + inst32Size - inst8Size)) && opCode8 && !Support::test(options, InstOptions::kLongForm)) {
+ options |= InstOptions::kShortForm;
writer.emit8(opCode8); // Emit opcode
writer.emit8(rel32 + inst32Size - inst8Size); // Emit DISP8.
goto EmitDone;
}
else {
- if (ASMJIT_UNLIKELY(!opcode.v || (options & Inst::kOptionShortForm) != 0))
+ if (ASMJIT_UNLIKELY(!opcode.v || Support::test(options, InstOptions::kShortForm)))
goto InvalidDisplacement;
- options &= ~Inst::kOptionShortForm;
+ options &= ~InstOptions::kShortForm;
writer.emit8If(0x0F, (opcode & Opcode::kMM_Mask) != 0); // Emit 0x0F prefix.
writer.emit8(opcode.v); // Emit Opcode.
writer.emit8If(x86EncodeMod(3, opReg, 0), opReg != 0); // Emit MOD.
@@ -4556,9 +4915,8 @@ EmitJmpCallRel:
}
}
- // --------------------------------------------------------------------------
- // [Emit - Relative]
- // --------------------------------------------------------------------------
+ // Emit - Relative
+ // ---------------
EmitRel:
{
@@ -4566,28 +4924,26 @@ EmitRel:
// Chain with label.
size_t offset = size_t(writer.offsetFrom(_bufferData));
- LabelLink* link = _code->newLabelLink(label, _section->id(), offset, relOffset);
+ OffsetFormat of;
+ of.resetToSimpleValue(OffsetType::kSignedOffset, relSize);
+ LabelLink* link = _code->newLabelLink(label, _section->id(), offset, relOffset, of);
if (ASMJIT_UNLIKELY(!link))
goto OutOfMemory;
if (re)
link->relocId = re->id();
- // Emit label size as dummy data.
- if (relSize == 1)
- writer.emit8(0x01);
- else // if (relSize == 4)
- writer.emit32uLE(0x04040404);
+ // Emit dummy zeros, must be patched later when the reference becomes known.
+ writer.emitZeros(relSize);
}
writer.emitImmediate(uint64_t(immValue), immSize);
- // --------------------------------------------------------------------------
- // [Done]
- // --------------------------------------------------------------------------
+ // Emit - Done
+ // -----------
EmitDone:
- if (ASMJIT_UNLIKELY(options & Inst::kOptionReserved)) {
+ if (Support::test(options, InstOptions::kReserved)) {
#ifndef ASMJIT_NO_LOGGING
if (_logger)
EmitterUtils::logInstructionEmitted(this, instId, options, o0, o1, o2, opExt, relSize, immSize, writer.cursor());
@@ -4601,15 +4957,10 @@ EmitDone:
writer.done(this);
return kErrorOk;
- // --------------------------------------------------------------------------
- // [Error Cases]
- // --------------------------------------------------------------------------
-
- #define ERROR_HANDLER(ERROR) \
- ERROR: \
- err = DebugUtils::errored(kError##ERROR); \
- goto Failed;
+ // Error Handler
+ // -------------
+#define ERROR_HANDLER(ERR) ERR: err = DebugUtils::errored(kError##ERR); goto Failed;
ERROR_HANDLER(OutOfMemory)
ERROR_HANDLER(InvalidLabel)
ERROR_HANDLER(InvalidInstruction)
@@ -4623,13 +4974,14 @@ EmitDone:
ERROR_HANDLER(InvalidAddressIndex)
ERROR_HANDLER(InvalidAddress64Bit)
ERROR_HANDLER(InvalidDisplacement)
+ ERROR_HANDLER(InvalidPhysId)
ERROR_HANDLER(InvalidSegment)
ERROR_HANDLER(InvalidImmediate)
+ ERROR_HANDLER(InvalidBroadcast)
ERROR_HANDLER(OperandSizeMismatch)
ERROR_HANDLER(AmbiguousOperandSize)
ERROR_HANDLER(NotConsecutiveRegs)
-
- #undef ERROR_HANDLER
+#undef ERROR_HANDLER
Failed:
#ifndef ASMJIT_NO_LOGGING
@@ -4642,15 +4994,14 @@ Failed:
#endif
}
-// ============================================================================
-// [asmjit::x86::Assembler - Align]
-// ============================================================================
+//x86::Assembler - Align
+// =====================
-Error Assembler::align(uint32_t alignMode, uint32_t alignment) {
+Error Assembler::align(AlignMode alignMode, uint32_t alignment) {
if (ASMJIT_UNLIKELY(!_code))
return reportError(DebugUtils::errored(kErrorNotInitialized));
- if (ASMJIT_UNLIKELY(alignMode >= kAlignCount))
+ if (ASMJIT_UNLIKELY(uint32_t(alignMode) > uint32_t(AlignMode::kMaxValue)))
return reportError(DebugUtils::errored(kErrorInvalidArgument));
if (alignment <= 1)
@@ -4661,13 +5012,13 @@ Error Assembler::align(uint32_t alignMode, uint32_t alignment) {
uint32_t i = uint32_t(Support::alignUpDiff<size_t>(offset(), alignment));
if (i > 0) {
- CodeBufferWriter writer(this);
+ CodeWriter writer(this);
ASMJIT_PROPAGATE(writer.ensureSpace(this, i));
uint8_t pattern = 0x00;
switch (alignMode) {
- case kAlignCode: {
- if (hasEncodingOption(kEncodingOptionOptimizedAlign)) {
+ case AlignMode::kCode: {
+ if (hasEncodingOption(EncodingOptions::kOptimizedAlign)) {
// Intel 64 and IA-32 Architectures Software Developer's Manual - Volume 2B (NOP).
enum { kMaxNopSize = 9 };
@@ -4698,11 +5049,11 @@ Error Assembler::align(uint32_t alignMode, uint32_t alignment) {
break;
}
- case kAlignData:
+ case AlignMode::kData:
pattern = 0xCC;
break;
- case kAlignZero:
+ case AlignMode::kZero:
// Pattern already set to zero.
break;
}
@@ -4718,7 +5069,7 @@ Error Assembler::align(uint32_t alignMode, uint32_t alignment) {
#ifndef ASMJIT_NO_LOGGING
if (_logger) {
StringTmp<128> sb;
- sb.appendChars(' ', _logger->indentation(FormatOptions::kIndentationCode));
+ sb.appendChars(' ', _logger->indentation(FormatIndentationGroup::kCode));
sb.appendFormat("align %u\n", alignment);
_logger->log(sb);
}
@@ -4727,27 +5078,21 @@ Error Assembler::align(uint32_t alignMode, uint32_t alignment) {
return kErrorOk;
}
-// ============================================================================
-// [asmjit::x86::Assembler - Events]
-// ============================================================================
+// x86::Assembler - Events
+// =======================
Error Assembler::onAttach(CodeHolder* code) noexcept {
- uint32_t arch = code->arch();
- if (!Environment::isFamilyX86(arch))
- return DebugUtils::errored(kErrorInvalidArch);
-
+ Arch arch = code->arch();
ASMJIT_PROPAGATE(Base::onAttach(code));
if (Environment::is32Bit(arch)) {
// 32 bit architecture - X86.
- _gpRegInfo.setSignature(Gpd::kSignature);
- _forcedInstOptions |= Inst::_kOptionInvalidRex;
+ _forcedInstOptions |= InstOptions::kX86_InvalidRex;
_setAddressOverrideMask(kX86MemInfo_67H_X86);
}
else {
// 64 bit architecture - X64.
- _gpRegInfo.setSignature(Gpq::kSignature);
- _forcedInstOptions &= ~Inst::_kOptionInvalidRex;
+ _forcedInstOptions &= ~InstOptions::kX86_InvalidRex;
_setAddressOverrideMask(kX86MemInfo_67H_X64);
}
@@ -4755,12 +5100,11 @@ Error Assembler::onAttach(CodeHolder* code) noexcept {
}
Error Assembler::onDetach(CodeHolder* code) noexcept {
- _forcedInstOptions &= ~Inst::_kOptionInvalidRex;
+ _forcedInstOptions &= ~InstOptions::kX86_InvalidRex;
_setAddressOverrideMask(0);
-
return Base::onDetach(code);
}
ASMJIT_END_SUB_NAMESPACE
-#endif // ASMJIT_BUILD_X86
+#endif // !ASMJIT_NO_X86