summaryrefslogtreecommitdiffstatshomepage
path: root/3rdparty/asmjit/src/asmjit/x86/x86internal.cpp
diff options
context:
space:
mode:
Diffstat (limited to '3rdparty/asmjit/src/asmjit/x86/x86internal.cpp')
-rw-r--r--3rdparty/asmjit/src/asmjit/x86/x86internal.cpp1733
1 files changed, 0 insertions, 1733 deletions
diff --git a/3rdparty/asmjit/src/asmjit/x86/x86internal.cpp b/3rdparty/asmjit/src/asmjit/x86/x86internal.cpp
deleted file mode 100644
index 062525f5982..00000000000
--- a/3rdparty/asmjit/src/asmjit/x86/x86internal.cpp
+++ /dev/null
@@ -1,1733 +0,0 @@
-// AsmJit - Machine code generation for C++
-//
-// * Official AsmJit Home Page: https://asmjit.com
-// * Official Github Repository: https://github.com/asmjit/asmjit
-//
-// Copyright (c) 2008-2020 The AsmJit Authors
-//
-// This software is provided 'as-is', without any express or implied
-// warranty. In no event will the authors be held liable for any damages
-// arising from the use of this software.
-//
-// Permission is granted to anyone to use this software for any purpose,
-// including commercial applications, and to alter it and redistribute it
-// freely, subject to the following restrictions:
-//
-// 1. The origin of this software must not be misrepresented; you must not
-// claim that you wrote the original software. If you use this software
-// in a product, an acknowledgment in the product documentation would be
-// appreciated but is not required.
-// 2. Altered source versions must be plainly marked as such, and must not be
-// misrepresented as being the original software.
-// 3. This notice may not be removed or altered from any source distribution.
-
-#include "../core/api-build_p.h"
-#ifdef ASMJIT_BUILD_X86
-
-#include "../core/formatter.h"
-#include "../core/string.h"
-#include "../core/support.h"
-#include "../core/type.h"
-#include "../x86/x86internal_p.h"
-
-// Can be used for debugging...
-// #define ASMJIT_DUMP_ARGS_ASSIGNMENT
-
-ASMJIT_BEGIN_SUB_NAMESPACE(x86)
-
-// ============================================================================
-// [asmjit::X86Internal - Helpers]
-// ============================================================================
-
-static ASMJIT_INLINE uint32_t x86GetXmmMovInst(const FuncFrame& frame) {
- bool avx = frame.isAvxEnabled();
- bool aligned = frame.hasAlignedVecSR();
-
- return aligned ? (avx ? Inst::kIdVmovaps : Inst::kIdMovaps)
- : (avx ? Inst::kIdVmovups : Inst::kIdMovups);
-}
-
-static ASMJIT_INLINE uint32_t x86VecTypeIdToRegType(uint32_t typeId) noexcept {
- return typeId <= Type::_kIdVec128End ? Reg::kTypeXmm :
- typeId <= Type::_kIdVec256End ? Reg::kTypeYmm : Reg::kTypeZmm;
-}
-
-//! Converts `size` to a 'kmov?' instructio.
-static inline uint32_t x86KmovFromSize(uint32_t size) noexcept {
- switch (size) {
- case 1: return Inst::kIdKmovb;
- case 2: return Inst::kIdKmovw;
- case 4: return Inst::kIdKmovd;
- case 8: return Inst::kIdKmovq;
- default: return Inst::kIdNone;
- }
-}
-
-// ============================================================================
-// [asmjit::X86Internal - FuncDetail]
-// ============================================================================
-
-ASMJIT_FAVOR_SIZE Error X86Internal::initFuncDetail(FuncDetail& func, const FuncSignature& signature, uint32_t registerSize) noexcept {
- const CallConv& cc = func.callConv();
- uint32_t arch = cc.arch();
- uint32_t stackOffset = cc._spillZoneSize;
-
- uint32_t i;
- uint32_t argCount = func.argCount();
-
- if (func.retCount() != 0) {
- uint32_t typeId = func._rets[0].typeId();
- switch (typeId) {
- case Type::kIdI64:
- case Type::kIdU64: {
- if (Environment::is32Bit(arch)) {
- // Convert a 64-bit return value to two 32-bit return values.
- func._retCount = 2;
- typeId -= 2;
-
- // 64-bit value is returned in EDX:EAX on X86.
- func._rets[0].initReg(Reg::kTypeGpd, Gp::kIdAx, typeId);
- func._rets[1].initReg(Reg::kTypeGpd, Gp::kIdDx, typeId);
- break;
- }
- else {
- func._rets[0].initReg(Reg::kTypeGpq, Gp::kIdAx, typeId);
- }
- break;
- }
-
- case Type::kIdI8:
- case Type::kIdI16:
- case Type::kIdI32: {
- func._rets[0].initReg(Reg::kTypeGpd, Gp::kIdAx, Type::kIdI32);
- break;
- }
-
- case Type::kIdU8:
- case Type::kIdU16:
- case Type::kIdU32: {
- func._rets[0].initReg(Reg::kTypeGpd, Gp::kIdAx, Type::kIdU32);
- break;
- }
-
- case Type::kIdF32:
- case Type::kIdF64: {
- uint32_t regType = Environment::is32Bit(arch) ? Reg::kTypeSt : Reg::kTypeXmm;
- func._rets[0].initReg(regType, 0, typeId);
- break;
- }
-
- case Type::kIdF80: {
- // 80-bit floats are always returned by FP0.
- func._rets[0].initReg(Reg::kTypeSt, 0, typeId);
- break;
- }
-
- case Type::kIdMmx32:
- case Type::kIdMmx64: {
- // MM registers are returned through XMM (SystemV) or GPQ (Win64).
- uint32_t regType = Reg::kTypeMm;
- if (Environment::is64Bit(arch))
- regType = cc.strategy() == CallConv::kStrategyDefault ? Reg::kTypeXmm : Reg::kTypeGpq;
-
- func._rets[0].initReg(regType, 0, typeId);
- break;
- }
-
- default: {
- func._rets[0].initReg(x86VecTypeIdToRegType(typeId), 0, typeId);
- break;
- }
- }
- }
-
- switch (cc.strategy()) {
- case CallConv::kStrategyDefault: {
- uint32_t gpzPos = 0;
- uint32_t vecPos = 0;
-
- for (i = 0; i < argCount; i++) {
- FuncValue& arg = func._args[i];
- uint32_t typeId = arg.typeId();
-
- if (Type::isInt(typeId)) {
- uint32_t regId = BaseReg::kIdBad;
-
- if (gpzPos < CallConv::kMaxRegArgsPerGroup)
- regId = cc._passedOrder[Reg::kGroupGp].id[gpzPos];
-
- if (regId != BaseReg::kIdBad) {
- uint32_t regType = (typeId <= Type::kIdU32) ? Reg::kTypeGpd : Reg::kTypeGpq;
- arg.assignRegData(regType, regId);
- func.addUsedRegs(Reg::kGroupGp, Support::bitMask(regId));
- gpzPos++;
- }
- else {
- uint32_t size = Support::max<uint32_t>(Type::sizeOf(typeId), registerSize);
- arg.assignStackOffset(int32_t(stackOffset));
- stackOffset += size;
- }
- continue;
- }
-
- if (Type::isFloat(typeId) || Type::isVec(typeId)) {
- uint32_t regId = BaseReg::kIdBad;
-
- if (vecPos < CallConv::kMaxRegArgsPerGroup)
- regId = cc._passedOrder[Reg::kGroupVec].id[vecPos];
-
- if (Type::isFloat(typeId)) {
- // If this is a float, but `kFlagPassFloatsByVec` is false, we have
- // to use stack instead. This should be only used by 32-bit calling
- // conventions.
- if (!cc.hasFlag(CallConv::kFlagPassFloatsByVec))
- regId = BaseReg::kIdBad;
- }
- else {
- // Pass vector registers via stack if this is a variable arguments
- // function. This should be only used by 32-bit calling conventions.
- if (signature.hasVarArgs() && cc.hasFlag(CallConv::kFlagPassVecByStackIfVA))
- regId = BaseReg::kIdBad;
- }
-
- if (regId != BaseReg::kIdBad) {
- arg.initTypeId(typeId);
- arg.assignRegData(x86VecTypeIdToRegType(typeId), regId);
- func.addUsedRegs(Reg::kGroupVec, Support::bitMask(regId));
- vecPos++;
- }
- else {
- uint32_t size = Type::sizeOf(typeId);
- arg.assignStackOffset(int32_t(stackOffset));
- stackOffset += size;
- }
- continue;
- }
- }
- break;
- }
-
- case CallConv::kStrategyX64Windows:
- case CallConv::kStrategyX64VectorCall: {
- // Both X64 and VectorCall behave similarly - arguments are indexed
- // from left to right. The position of the argument determines in
- // which register the argument is allocated, so it's either GP or
- // one of XMM/YMM/ZMM registers.
- //
- // [ X64 ] [VecCall]
- // Index: #0 #1 #2 #3 #4 #5
- //
- // GP : RCX RDX R8 R9
- // VEC : XMM0 XMM1 XMM2 XMM3 XMM4 XMM5
- //
- // For example function `f(int a, double b, int c, double d)` will be:
- //
- // (a) (b) (c) (d)
- // RCX XMM1 R8 XMM3
- //
- // Unused vector registers are used by HVA.
-
- bool isVectorCall = (cc.strategy() == CallConv::kStrategyX64VectorCall);
-
- for (i = 0; i < argCount; i++) {
- FuncValue& arg = func._args[i];
-
- uint32_t typeId = arg.typeId();
- uint32_t size = Type::sizeOf(typeId);
-
- if (Type::isInt(typeId) || Type::isMmx(typeId)) {
- uint32_t regId = BaseReg::kIdBad;
-
- if (i < CallConv::kMaxRegArgsPerGroup)
- regId = cc._passedOrder[Reg::kGroupGp].id[i];
-
- if (regId != BaseReg::kIdBad) {
- uint32_t regType = (size <= 4 && !Type::isMmx(typeId)) ? Reg::kTypeGpd : Reg::kTypeGpq;
- arg.assignRegData(regType, regId);
- func.addUsedRegs(Reg::kGroupGp, Support::bitMask(regId));
- }
- else {
- arg.assignStackOffset(int32_t(stackOffset));
- stackOffset += 8;
- }
- continue;
- }
-
- if (Type::isFloat(typeId) || Type::isVec(typeId)) {
- uint32_t regId = BaseReg::kIdBad;
-
- if (i < CallConv::kMaxRegArgsPerGroup)
- regId = cc._passedOrder[Reg::kGroupVec].id[i];
-
- if (regId != BaseReg::kIdBad) {
- // X64-ABI doesn't allow vector types (XMM|YMM|ZMM) to be passed
- // via registers, however, VectorCall was designed for that purpose.
- if (Type::isFloat(typeId) || isVectorCall) {
- uint32_t regType = x86VecTypeIdToRegType(typeId);
- arg.assignRegData(regType, regId);
- func.addUsedRegs(Reg::kGroupVec, Support::bitMask(regId));
- continue;
- }
- }
-
- // Passed via stack if the argument is float/double or indirectly.
- // The trap is - if the argument is passed indirectly, the address
- // can be passed via register, if the argument's index has GP one.
- if (Type::isFloat(typeId)) {
- arg.assignStackOffset(int32_t(stackOffset));
- }
- else {
- uint32_t gpRegId = cc._passedOrder[Reg::kGroupGp].id[i];
- if (gpRegId != BaseReg::kIdBad)
- arg.assignRegData(Reg::kTypeGpq, gpRegId);
- else
- arg.assignStackOffset(int32_t(stackOffset));
- arg.addFlags(FuncValue::kFlagIsIndirect);
- }
-
- // Always 8 bytes (float/double/pointer).
- stackOffset += 8;
- continue;
- }
- }
- break;
- }
- }
-
- func._argStackSize = stackOffset;
- return kErrorOk;
-}
-
-// ============================================================================
-// [asmjit::X86FuncArgsContext]
-// ============================================================================
-
-static RegInfo x86GetRegForMemToMemMove(uint32_t arch, uint32_t dstTypeId, uint32_t srcTypeId) noexcept {
- uint32_t dstSize = Type::sizeOf(dstTypeId);
- uint32_t srcSize = Type::sizeOf(srcTypeId);
- uint32_t maxSize = Support::max<uint32_t>(dstSize, srcSize);
- uint32_t regSize = Environment::registerSizeFromArch(arch);
-
- uint32_t signature = 0;
- if (maxSize <= regSize || (Type::isInt(dstTypeId) && Type::isInt(srcTypeId)))
- signature = maxSize <= 4 ? Gpd::kSignature : Gpq::kSignature;
- else if (maxSize <= 16)
- signature = Xmm::kSignature;
- else if (maxSize <= 32)
- signature = Ymm::kSignature;
- else if (maxSize <= 64)
- signature = Zmm::kSignature;
-
- return RegInfo { signature };
-}
-
-// Used by both `argsToFuncFrame()` and `emitArgsAssignment()`.
-class X86FuncArgsContext {
-public:
- enum VarId : uint32_t {
- kVarIdNone = 0xFF
- };
-
- //! Contains information about a single argument or SA register that may need shuffling.
- struct Var {
- inline void init(const FuncValue& cur_, const FuncValue& out_) noexcept {
- cur = cur_;
- out = out_;
- }
-
- //! Reset the value to its unassigned state.
- inline void reset() noexcept {
- cur.reset();
- out.reset();
- }
-
- inline bool isDone() const noexcept { return cur.isDone(); }
- inline void markDone() noexcept { cur.addFlags(FuncValue::kFlagIsDone); }
-
- FuncValue cur;
- FuncValue out;
- };
-
- struct WorkData {
- inline void reset() noexcept {
- _archRegs = 0;
- _workRegs = 0;
- _usedRegs = 0;
- _assignedRegs = 0;
- _dstRegs = 0;
- _dstShuf = 0;
- _numSwaps = 0;
- _numStackArgs = 0;
- memset(_reserved, 0, sizeof(_reserved));
- memset(_physToVarId, kVarIdNone, 32);
- }
-
- inline bool isAssigned(uint32_t regId) const noexcept {
- ASMJIT_ASSERT(regId < 32);
- return Support::bitTest(_assignedRegs, regId);
- }
-
- inline void assign(uint32_t varId, uint32_t regId) noexcept {
- ASMJIT_ASSERT(!isAssigned(regId));
- ASMJIT_ASSERT(_physToVarId[regId] == kVarIdNone);
-
- _physToVarId[regId] = uint8_t(varId);
- _assignedRegs ^= Support::bitMask(regId);
- }
-
- inline void reassign(uint32_t varId, uint32_t newId, uint32_t oldId) noexcept {
- ASMJIT_ASSERT( isAssigned(oldId));
- ASMJIT_ASSERT(!isAssigned(newId));
- ASMJIT_ASSERT(_physToVarId[oldId] == varId);
- ASMJIT_ASSERT(_physToVarId[newId] == kVarIdNone);
-
- _physToVarId[oldId] = uint8_t(kVarIdNone);
- _physToVarId[newId] = uint8_t(varId);
- _assignedRegs ^= Support::bitMask(newId) ^ Support::bitMask(oldId);
- }
-
- inline void swap(uint32_t aVarId, uint32_t aRegId, uint32_t bVarId, uint32_t bRegId) noexcept {
- ASMJIT_ASSERT(isAssigned(aRegId));
- ASMJIT_ASSERT(isAssigned(bRegId));
- ASMJIT_ASSERT(_physToVarId[aRegId] == aVarId);
- ASMJIT_ASSERT(_physToVarId[bRegId] == bVarId);
-
- _physToVarId[aRegId] = uint8_t(bVarId);
- _physToVarId[bRegId] = uint8_t(aVarId);
- }
-
- inline void unassign(uint32_t varId, uint32_t regId) noexcept {
- ASMJIT_ASSERT(isAssigned(regId));
- ASMJIT_ASSERT(_physToVarId[regId] == varId);
-
- DebugUtils::unused(varId);
- _physToVarId[regId] = uint8_t(kVarIdNone);
- _assignedRegs ^= Support::bitMask(regId);
- }
-
- inline uint32_t archRegs() const noexcept { return _archRegs; }
- inline uint32_t workRegs() const noexcept { return _workRegs; }
- inline uint32_t usedRegs() const noexcept { return _usedRegs; }
- inline uint32_t assignedRegs() const noexcept { return _assignedRegs; }
- inline uint32_t dstRegs() const noexcept { return _dstRegs; }
- inline uint32_t availableRegs() const noexcept { return _workRegs & ~_assignedRegs; }
-
- uint32_t _archRegs; //!< All allocable registers provided by the architecture.
- uint32_t _workRegs; //!< All registers that can be used by the shuffler.
- uint32_t _usedRegs; //!< Registers used by the shuffler (all).
- uint32_t _assignedRegs; //!< Assigned registers.
- uint32_t _dstRegs; //!< Destination registers assigned to arguments or SA.
- uint32_t _dstShuf; //!< Destination registers that require shuffling.
- uint8_t _numSwaps; //!< Number of register swaps.
- uint8_t _numStackArgs; //!< Number of stack loads.
- uint8_t _reserved[6]; //!< Reserved (only used as padding).
- uint8_t _physToVarId[32]; //!< Physical ID to variable ID mapping.
- };
-
- uint8_t _arch;
- bool _hasStackSrc; //!< Has arguments passed via stack (SRC).
- bool _hasPreservedFP; //!< Has preserved frame-pointer (FP).
- uint8_t _stackDstMask; //!< Has arguments assigned to stack (DST).
- uint8_t _regSwapsMask; //!< Register swap groups (bit-mask).
- uint8_t _saVarId;
- uint32_t _varCount;
- WorkData _workData[BaseReg::kGroupVirt];
- Var _vars[kFuncArgCountLoHi + 1];
-
- X86FuncArgsContext() noexcept;
-
- inline uint32_t arch() const noexcept { return _arch; }
- inline uint32_t varCount() const noexcept { return _varCount; }
-
- inline Var& var(size_t varId) noexcept { return _vars[varId]; }
- inline const Var& var(size_t varId) const noexcept { return _vars[varId]; }
- inline size_t indexOf(const Var* var) const noexcept { return (size_t)(var - _vars); }
-
- Error initWorkData(const FuncFrame& frame, const FuncArgsAssignment& args) noexcept;
- Error markScratchRegs(FuncFrame& frame) noexcept;
- Error markDstRegsDirty(FuncFrame& frame) noexcept;
- Error markStackArgsReg(FuncFrame& frame) noexcept;
-};
-
-X86FuncArgsContext::X86FuncArgsContext() noexcept {
- _arch = Environment::kArchUnknown;
- _varCount = 0;
- _hasStackSrc = false;
- _hasPreservedFP = false;
- _stackDstMask = 0;
- _regSwapsMask = 0;
- _saVarId = kVarIdNone;
-
- for (uint32_t group = 0; group < BaseReg::kGroupVirt; group++)
- _workData[group].reset();
-}
-
-ASMJIT_FAVOR_SIZE Error X86FuncArgsContext::initWorkData(const FuncFrame& frame, const FuncArgsAssignment& args) noexcept {
- // The code has to be updated if this changes.
- ASMJIT_ASSERT(BaseReg::kGroupVirt == 4);
-
- uint32_t i;
- const FuncDetail& func = *args.funcDetail();
-
- // Initialize Architecture.
- uint32_t arch = func.callConv().arch();
- uint32_t archRegCount = Environment::is32Bit(arch) ? 8 : 16;
-
- _arch = uint8_t(arch);
-
- // Initialize `_archRegs`.
- _workData[Reg::kGroupGp ]._archRegs = Support::lsbMask<uint32_t>(archRegCount) & ~Support::bitMask(Gp::kIdSp);
- _workData[Reg::kGroupVec ]._archRegs = Support::lsbMask<uint32_t>(archRegCount);
- _workData[Reg::kGroupMm ]._archRegs = Support::lsbMask<uint32_t>(8);
- _workData[Reg::kGroupKReg]._archRegs = Support::lsbMask<uint32_t>(8);
-
- if (frame.hasPreservedFP())
- _workData[Reg::kGroupGp]._archRegs &= ~Support::bitMask(Gp::kIdBp);
-
- // Extract information from all function arguments/assignments and build Var[] array.
- uint32_t varId = 0;
- for (i = 0; i < kFuncArgCountLoHi; i++) {
- const FuncValue& dst_ = args.arg(i);
- if (!dst_.isAssigned())
- continue;
-
- const FuncValue& src_ = func.arg(i);
- if (ASMJIT_UNLIKELY(!src_.isAssigned()))
- return DebugUtils::errored(kErrorInvalidState);
-
- Var& var = _vars[varId];
- var.init(src_, dst_);
-
- FuncValue& src = var.cur;
- FuncValue& dst = var.out;
-
- uint32_t dstGroup = 0xFFFFFFFFu;
- uint32_t dstId = BaseReg::kIdBad;
- WorkData* dstWd = nullptr;
-
- // Not supported.
- if (src.isIndirect())
- return DebugUtils::errored(kErrorInvalidAssignment);
-
- if (dst.isReg()) {
- uint32_t dstType = dst.regType();
- if (ASMJIT_UNLIKELY(dstType >= Reg::kTypeCount))
- return DebugUtils::errored(kErrorInvalidRegType);
-
- // Copy TypeId from source if the destination doesn't have it. The RA
- // used by BaseCompiler would never leave TypeId undefined, but users
- // of FuncAPI can just assign phys regs without specifying the type.
- if (!dst.hasTypeId())
- dst.setTypeId(Reg::typeIdOf(dst.regType()));
-
- dstGroup = Reg::groupOf(dstType);
- if (ASMJIT_UNLIKELY(dstGroup >= BaseReg::kGroupVirt))
- return DebugUtils::errored(kErrorInvalidRegGroup);
-
- dstWd = &_workData[dstGroup];
- dstId = dst.regId();
- if (ASMJIT_UNLIKELY(dstId >= 32 || !Support::bitTest(dstWd->archRegs(), dstId)))
- return DebugUtils::errored(kErrorInvalidPhysId);
-
- if (ASMJIT_UNLIKELY(Support::bitTest(dstWd->dstRegs(), dstId)))
- return DebugUtils::errored(kErrorOverlappedRegs);
-
- dstWd->_dstRegs |= Support::bitMask(dstId);
- dstWd->_dstShuf |= Support::bitMask(dstId);
- dstWd->_usedRegs |= Support::bitMask(dstId);
- }
- else {
- if (!dst.hasTypeId())
- dst.setTypeId(src.typeId());
-
- RegInfo regInfo = x86GetRegForMemToMemMove(arch, dst.typeId(), src.typeId());
- if (ASMJIT_UNLIKELY(!regInfo.isValid()))
- return DebugUtils::errored(kErrorInvalidState);
- _stackDstMask = uint8_t(_stackDstMask | Support::bitMask(regInfo.group()));
- }
-
- if (src.isReg()) {
- uint32_t srcId = src.regId();
- uint32_t srcGroup = Reg::groupOf(src.regType());
-
- if (dstGroup == srcGroup) {
- dstWd->assign(varId, srcId);
-
- // The best case, register is allocated where it is expected to be.
- if (dstId == srcId)
- var.markDone();
- }
- else {
- if (ASMJIT_UNLIKELY(srcGroup >= BaseReg::kGroupVirt))
- return DebugUtils::errored(kErrorInvalidState);
-
- WorkData& srcData = _workData[srcGroup];
- srcData.assign(varId, srcId);
- }
- }
- else {
- if (dstWd)
- dstWd->_numStackArgs++;
- _hasStackSrc = true;
- }
-
- varId++;
- }
-
- // Initialize WorkData::workRegs.
- for (i = 0; i < BaseReg::kGroupVirt; i++)
- _workData[i]._workRegs = (_workData[i].archRegs() & (frame.dirtyRegs(i) | ~frame.preservedRegs(i))) | _workData[i].dstRegs() | _workData[i].assignedRegs();
-
- // Create a variable that represents `SARegId` if necessary.
- bool saRegRequired = _hasStackSrc && frame.hasDynamicAlignment() && !frame.hasPreservedFP();
-
- WorkData& gpRegs = _workData[BaseReg::kGroupGp];
- uint32_t saCurRegId = frame.saRegId();
- uint32_t saOutRegId = args.saRegId();
-
- if (saCurRegId != BaseReg::kIdBad) {
- // Check if the provided `SARegId` doesn't collide with input registers.
- if (ASMJIT_UNLIKELY(gpRegs.isAssigned(saCurRegId)))
- return DebugUtils::errored(kErrorOverlappedRegs);
- }
-
- if (saOutRegId != BaseReg::kIdBad) {
- // Check if the provided `SARegId` doesn't collide with argument assignments.
- if (ASMJIT_UNLIKELY(Support::bitTest(gpRegs.dstRegs(), saOutRegId)))
- return DebugUtils::errored(kErrorOverlappedRegs);
- saRegRequired = true;
- }
-
- if (saRegRequired) {
- uint32_t ptrTypeId = Environment::is32Bit(arch) ? Type::kIdU32 : Type::kIdU64;
- uint32_t ptrRegType = Environment::is32Bit(arch) ? BaseReg::kTypeGp32 : BaseReg::kTypeGp64;
-
- _saVarId = uint8_t(varId);
- _hasPreservedFP = frame.hasPreservedFP();
-
- Var& var = _vars[varId];
- var.reset();
-
- if (saCurRegId == BaseReg::kIdBad) {
- if (saOutRegId != BaseReg::kIdBad && !gpRegs.isAssigned(saOutRegId)) {
- saCurRegId = saOutRegId;
- }
- else {
- uint32_t availableRegs = gpRegs.availableRegs();
- if (!availableRegs)
- availableRegs = gpRegs.archRegs() & ~gpRegs.workRegs();
-
- if (ASMJIT_UNLIKELY(!availableRegs))
- return DebugUtils::errored(kErrorNoMorePhysRegs);
-
- saCurRegId = Support::ctz(availableRegs);
- }
- }
-
- var.cur.initReg(ptrRegType, saCurRegId, ptrTypeId);
- gpRegs.assign(varId, saCurRegId);
- gpRegs._workRegs |= Support::bitMask(saCurRegId);
-
- if (saOutRegId != BaseReg::kIdBad) {
- var.out.initReg(ptrRegType, saOutRegId, ptrTypeId);
- gpRegs._dstRegs |= Support::bitMask(saOutRegId);
- gpRegs._workRegs |= Support::bitMask(saOutRegId);
- }
- else {
- var.markDone();
- }
-
- varId++;
- }
-
- _varCount = varId;
-
- // Detect register swaps.
- for (varId = 0; varId < _varCount; varId++) {
- Var& var = _vars[varId];
- if (var.cur.isReg() && var.out.isReg()) {
- uint32_t srcId = var.cur.regId();
- uint32_t dstId = var.out.regId();
-
- uint32_t group = Reg::groupOf(var.cur.regType());
- if (group != Reg::groupOf(var.out.regType()))
- continue;
-
- WorkData& wd = _workData[group];
- if (wd.isAssigned(dstId)) {
- Var& other = _vars[wd._physToVarId[dstId]];
- if (Reg::groupOf(other.out.regType()) == group && other.out.regId() == srcId) {
- wd._numSwaps++;
- _regSwapsMask = uint8_t(_regSwapsMask | Support::bitMask(group));
- }
- }
- }
- }
-
- return kErrorOk;
-}
-
-ASMJIT_FAVOR_SIZE Error X86FuncArgsContext::markDstRegsDirty(FuncFrame& frame) noexcept {
- for (uint32_t i = 0; i < BaseReg::kGroupVirt; i++) {
- WorkData& wd = _workData[i];
- uint32_t regs = wd.usedRegs() | wd._dstShuf;
-
- wd._workRegs |= regs;
- frame.addDirtyRegs(i, regs);
- }
-
- return kErrorOk;
-}
-
-ASMJIT_FAVOR_SIZE Error X86FuncArgsContext::markScratchRegs(FuncFrame& frame) noexcept {
- uint32_t groupMask = 0;
-
- // Handle stack to stack moves.
- groupMask |= _stackDstMask;
-
- // Handle register swaps.
- groupMask |= _regSwapsMask & ~Support::bitMask(BaseReg::kGroupGp);
-
- if (!groupMask)
- return kErrorOk;
-
- // Selects one dirty register per affected group that can be used as a scratch register.
- for (uint32_t group = 0; group < BaseReg::kGroupVirt; group++) {
- if (Support::bitTest(groupMask, group)) {
- WorkData& wd = _workData[group];
-
- // Initially, pick some clobbered or dirty register.
- uint32_t workRegs = wd.workRegs();
- uint32_t regs = workRegs & ~(wd.usedRegs() | wd._dstShuf);
-
- // If that didn't work out pick some register which is not in 'used'.
- if (!regs)
- regs = workRegs & ~wd.usedRegs();
-
- // If that didn't work out pick any other register that is allocable.
- // This last resort case will, however, result in marking one more
- // register dirty.
- if (!regs)
- regs = wd.archRegs() & ~workRegs;
-
- // If that didn't work out we will have to use XORs instead of MOVs.
- if (!regs)
- continue;
-
- uint32_t regMask = Support::blsi(regs);
- wd._workRegs |= regMask;
- frame.addDirtyRegs(group, regMask);
- }
- }
-
- return kErrorOk;
-}
-
-ASMJIT_FAVOR_SIZE Error X86FuncArgsContext::markStackArgsReg(FuncFrame& frame) noexcept {
- if (_saVarId != kVarIdNone) {
- const Var& var = _vars[_saVarId];
- frame.setSARegId(var.cur.regId());
- }
- else if (frame.hasPreservedFP()) {
- // Always EBP|RBP if the frame-pointer isn't omitted.
- frame.setSARegId(Gp::kIdBp);
- }
-
- return kErrorOk;
-}
-
-// ============================================================================
-// [asmjit::X86Internal - FrameLayout]
-// ============================================================================
-
-ASMJIT_FAVOR_SIZE Error X86Internal::initFuncFrame(FuncFrame& frame, const FuncDetail& func) noexcept {
- uint32_t arch = func.callConv().arch();
-
- // Initializing FuncFrame means making a copy of some properties of `func`.
- // Properties like `_localStackSize` will be set by the user before the frame
- // is finalized.
- frame.reset();
-
- frame._arch = uint8_t(arch);
- frame._spRegId = Gp::kIdSp;
- frame._saRegId = Gp::kIdBad;
-
- uint32_t naturalStackAlignment = func.callConv().naturalStackAlignment();
- uint32_t minDynamicAlignment = Support::max<uint32_t>(naturalStackAlignment, 16);
-
- if (minDynamicAlignment == naturalStackAlignment)
- minDynamicAlignment <<= 1;
-
- frame._naturalStackAlignment = uint8_t(naturalStackAlignment);
- frame._minDynamicAlignment = uint8_t(minDynamicAlignment);
- frame._redZoneSize = uint8_t(func.redZoneSize());
- frame._spillZoneSize = uint8_t(func.spillZoneSize());
- frame._finalStackAlignment = uint8_t(frame._naturalStackAlignment);
-
- if (func.hasFlag(CallConv::kFlagCalleePopsStack)) {
- frame._calleeStackCleanup = uint16_t(func.argStackSize());
- }
-
- // Initial masks of dirty and preserved registers.
- for (uint32_t group = 0; group < BaseReg::kGroupVirt; group++) {
- frame._dirtyRegs[group] = func.usedRegs(group);
- frame._preservedRegs[group] = func.preservedRegs(group);
- }
-
- // Exclude ESP/RSP - this register is never included in saved GP regs.
- frame._preservedRegs[BaseReg::kGroupGp] &= ~Support::bitMask(Gp::kIdSp);
-
- return kErrorOk;
-}
-
-ASMJIT_FAVOR_SIZE Error X86Internal::finalizeFuncFrame(FuncFrame& frame) noexcept {
- uint32_t registerSize = Environment::registerSizeFromArch(frame.arch());
-
- // The final stack alignment must be updated accordingly to call and local stack alignments.
- uint32_t stackAlignment = frame._finalStackAlignment;
- ASMJIT_ASSERT(stackAlignment == Support::max(frame._naturalStackAlignment,
- frame._callStackAlignment,
- frame._localStackAlignment));
-
- // TODO: Must be configurable.
- uint32_t vecSize = 16;
-
- bool hasFP = frame.hasPreservedFP();
- bool hasDA = frame.hasDynamicAlignment();
-
- // Include EBP|RBP if the function preserves the frame-pointer.
- if (hasFP)
- frame._dirtyRegs[Reg::kGroupGp] |= Support::bitMask(Gp::kIdBp);
-
- // These two are identical if the function doesn't align its stack dynamically.
- uint32_t saRegId = frame.saRegId();
- if (saRegId == BaseReg::kIdBad)
- saRegId = Gp::kIdSp;
-
- // Fix stack arguments base-register from ESP|RSP to EBP|RBP in case it was
- // not picked before and the function performs dynamic stack alignment.
- if (hasDA && saRegId == Gp::kIdSp)
- saRegId = Gp::kIdBp;
-
- // Mark as dirty any register but ESP|RSP if used as SA pointer.
- if (saRegId != Gp::kIdSp)
- frame._dirtyRegs[Reg::kGroupGp] |= Support::bitMask(saRegId);
-
- frame._spRegId = uint8_t(Gp::kIdSp);
- frame._saRegId = uint8_t(saRegId);
-
- // Setup stack size used to save preserved registers.
- frame._gpSaveSize = uint16_t(Support::popcnt(frame.savedRegs(Reg::kGroupGp )) * registerSize);
- frame._nonGpSaveSize = uint16_t(Support::popcnt(frame.savedRegs(Reg::kGroupVec )) * vecSize +
- Support::popcnt(frame.savedRegs(Reg::kGroupMm )) * 8 +
- Support::popcnt(frame.savedRegs(Reg::kGroupKReg)) * 8);
-
- uint32_t v = 0; // The beginning of the stack frame relative to SP after prolog.
- v += frame.callStackSize(); // Count 'callStackSize' <- This is used to call functions.
- v = Support::alignUp(v, stackAlignment); // Align to function's stack alignment.
-
- frame._localStackOffset = v; // Store 'localStackOffset' <- Function's local stack starts here.
- v += frame.localStackSize(); // Count 'localStackSize' <- Function's local stack ends here.
-
- // If the function's stack must be aligned, calculate the alignment necessary
- // to store vector registers, and set `FuncFrame::kAttrAlignedVecSR` to inform
- // PEI that it can use instructions that perform aligned stores/loads.
- if (stackAlignment >= vecSize && frame._nonGpSaveSize) {
- frame.addAttributes(FuncFrame::kAttrAlignedVecSR);
- v = Support::alignUp(v, vecSize); // Align '_nonGpSaveOffset'.
- }
-
- frame._nonGpSaveOffset = v; // Store '_nonGpSaveOffset' <- Non-GP Save/Restore starts here.
- v += frame._nonGpSaveSize; // Count '_nonGpSaveSize' <- Non-GP Save/Restore ends here.
-
- // Calculate if dynamic alignment (DA) slot (stored as offset relative to SP) is required and its offset.
- if (hasDA && !hasFP) {
- frame._daOffset = v; // Store 'daOffset' <- DA pointer would be stored here.
- v += registerSize; // Count 'daOffset'.
- }
- else {
- frame._daOffset = FuncFrame::kTagInvalidOffset;
- }
-
- // The return address should be stored after GP save/restore regs. It has
- // the same size as `registerSize` (basically the native register/pointer
- // size). We don't adjust it now as `v` now contains the exact size that the
- // function requires to adjust (call frame + stack frame, vec stack size).
- // The stack (if we consider this size) is misaligned now, as it's always
- // aligned before the function call - when `call()` is executed it pushes
- // the current EIP|RIP onto the stack, and misaligns it by 12 or 8 bytes
- // (depending on the architecture). So count number of bytes needed to align
- // it up to the function's CallFrame (the beginning).
- if (v || frame.hasFuncCalls())
- v += Support::alignUpDiff(v + frame.gpSaveSize() + registerSize, stackAlignment);
-
- frame._gpSaveOffset = v; // Store 'gpSaveOffset' <- Function's GP Save/Restore starts here.
- frame._stackAdjustment = v; // Store 'stackAdjustment' <- SA used by 'add zsp, SA' and 'sub zsp, SA'.
-
- v += frame._gpSaveSize; // Count 'gpSaveSize' <- Function's GP Save/Restore ends here.
- v += registerSize; // Count 'ReturnAddress' <- As CALL pushes onto stack.
-
- // If the function performs dynamic stack alignment then the stack-adjustment must be aligned.
- if (hasDA)
- frame._stackAdjustment = Support::alignUp(frame._stackAdjustment, stackAlignment);
-
- uint32_t saInvOff = FuncFrame::kTagInvalidOffset;
- uint32_t saTmpOff = registerSize + frame._gpSaveSize;
-
- // Calculate where the function arguments start relative to SP.
- frame._saOffsetFromSP = hasDA ? saInvOff : v;
-
- // Calculate where the function arguments start relative to FP or user-provided register.
- frame._saOffsetFromSA = hasFP ? registerSize * 2 // Return address + frame pointer.
- : saTmpOff; // Return address + all saved GP regs.
-
- return kErrorOk;
-}
-
-// ============================================================================
-// [asmjit::X86Internal - ArgsToFrameInfo]
-// ============================================================================
-
-ASMJIT_FAVOR_SIZE Error X86Internal::argsToFuncFrame(const FuncArgsAssignment& args, FuncFrame& frame) noexcept {
- X86FuncArgsContext ctx;
- ASMJIT_PROPAGATE(ctx.initWorkData(frame, args));
- ASMJIT_PROPAGATE(ctx.markDstRegsDirty(frame));
- ASMJIT_PROPAGATE(ctx.markScratchRegs(frame));
- ASMJIT_PROPAGATE(ctx.markStackArgsReg(frame));
- return kErrorOk;
-}
-
-// ============================================================================
-// [asmjit::X86Internal - Emit Helpers]
-// ============================================================================
-
-ASMJIT_FAVOR_SIZE Error X86Internal::emitRegMove(Emitter* emitter,
- const Operand_& dst_,
- const Operand_& src_, uint32_t typeId, bool avxEnabled, const char* comment) {
-
- // Invalid or abstract TypeIds are not allowed.
- ASMJIT_ASSERT(Type::isValid(typeId) && !Type::isAbstract(typeId));
-
- Operand dst(dst_);
- Operand src(src_);
-
- uint32_t instId = Inst::kIdNone;
- uint32_t memFlags = 0;
- uint32_t overrideMemSize = 0;
-
- enum MemFlags : uint32_t {
- kDstMem = 0x1,
- kSrcMem = 0x2
- };
-
- // Detect memory operands and patch them to have the same size as the register.
- // BaseCompiler always sets memory size of allocs and spills, so it shouldn't
- // be really necessary, however, after this function was separated from Compiler
- // it's better to make sure that the size is always specified, as we can use
- // 'movzx' and 'movsx' that rely on it.
- if (dst.isMem()) { memFlags |= kDstMem; dst.as<Mem>().setSize(src.size()); }
- if (src.isMem()) { memFlags |= kSrcMem; src.as<Mem>().setSize(dst.size()); }
-
- switch (typeId) {
- case Type::kIdI8:
- case Type::kIdU8:
- case Type::kIdI16:
- case Type::kIdU16:
- // Special case - 'movzx' load.
- if (memFlags & kSrcMem) {
- instId = Inst::kIdMovzx;
- dst.setSignature(Reg::signatureOfT<Reg::kTypeGpd>());
- }
- else if (!memFlags) {
- // Change both destination and source registers to GPD (safer, no dependencies).
- dst.setSignature(Reg::signatureOfT<Reg::kTypeGpd>());
- src.setSignature(Reg::signatureOfT<Reg::kTypeGpd>());
- }
- ASMJIT_FALLTHROUGH;
-
- case Type::kIdI32:
- case Type::kIdU32:
- case Type::kIdI64:
- case Type::kIdU64:
- instId = Inst::kIdMov;
- break;
-
- case Type::kIdMmx32:
- instId = Inst::kIdMovd;
- if (memFlags) break;
- ASMJIT_FALLTHROUGH;
-
- case Type::kIdMmx64 : instId = Inst::kIdMovq ; break;
- case Type::kIdMask8 : instId = Inst::kIdKmovb; break;
- case Type::kIdMask16: instId = Inst::kIdKmovw; break;
- case Type::kIdMask32: instId = Inst::kIdKmovd; break;
- case Type::kIdMask64: instId = Inst::kIdKmovq; break;
-
- default: {
- uint32_t elementTypeId = Type::baseOf(typeId);
- if (Type::isVec32(typeId) && memFlags) {
- overrideMemSize = 4;
- if (elementTypeId == Type::kIdF32)
- instId = avxEnabled ? Inst::kIdVmovss : Inst::kIdMovss;
- else
- instId = avxEnabled ? Inst::kIdVmovd : Inst::kIdMovd;
- break;
- }
-
- if (Type::isVec64(typeId) && memFlags) {
- overrideMemSize = 8;
- if (elementTypeId == Type::kIdF64)
- instId = avxEnabled ? Inst::kIdVmovsd : Inst::kIdMovsd;
- else
- instId = avxEnabled ? Inst::kIdVmovq : Inst::kIdMovq;
- break;
- }
-
- if (elementTypeId == Type::kIdF32)
- instId = avxEnabled ? Inst::kIdVmovaps : Inst::kIdMovaps;
- else if (elementTypeId == Type::kIdF64)
- instId = avxEnabled ? Inst::kIdVmovapd : Inst::kIdMovapd;
- else if (typeId <= Type::_kIdVec256End)
- instId = avxEnabled ? Inst::kIdVmovdqa : Inst::kIdMovdqa;
- else if (elementTypeId <= Type::kIdU32)
- instId = Inst::kIdVmovdqa32;
- else
- instId = Inst::kIdVmovdqa64;
- break;
- }
- }
-
- if (!instId)
- return DebugUtils::errored(kErrorInvalidState);
-
- if (overrideMemSize) {
- if (dst.isMem()) dst.as<Mem>().setSize(overrideMemSize);
- if (src.isMem()) src.as<Mem>().setSize(overrideMemSize);
- }
-
- emitter->setInlineComment(comment);
- return emitter->emit(instId, dst, src);
-}
-
-ASMJIT_FAVOR_SIZE Error X86Internal::emitArgMove(Emitter* emitter,
- const Reg& dst_, uint32_t dstTypeId,
- const Operand_& src_, uint32_t srcTypeId, bool avxEnabled, const char* comment) {
-
- // Deduce optional `dstTypeId`, which may be `Type::kIdVoid` in some cases.
- if (!dstTypeId)
- dstTypeId = opData.archRegs.regTypeToTypeId[dst_.type()];
-
- // Invalid or abstract TypeIds are not allowed.
- ASMJIT_ASSERT(Type::isValid(dstTypeId) && !Type::isAbstract(dstTypeId));
- ASMJIT_ASSERT(Type::isValid(srcTypeId) && !Type::isAbstract(srcTypeId));
-
- Reg dst(dst_);
- Operand src(src_);
-
- uint32_t dstSize = Type::sizeOf(dstTypeId);
- uint32_t srcSize = Type::sizeOf(srcTypeId);
-
- uint32_t instId = Inst::kIdNone;
-
- // Not a real loop, just 'break' is nicer than 'goto'.
- for (;;) {
- if (Type::isInt(dstTypeId)) {
- if (Type::isInt(srcTypeId)) {
- instId = Inst::kIdMovsx;
- uint32_t typeOp = (dstTypeId << 8) | srcTypeId;
-
- // Sign extend by using 'movsx'.
- if (typeOp == ((Type::kIdI16 << 8) | Type::kIdI8 ) ||
- typeOp == ((Type::kIdI32 << 8) | Type::kIdI8 ) ||
- typeOp == ((Type::kIdI32 << 8) | Type::kIdI16) ||
- typeOp == ((Type::kIdI64 << 8) | Type::kIdI8 ) ||
- typeOp == ((Type::kIdI64 << 8) | Type::kIdI16))
- break;
-
- // Sign extend by using 'movsxd'.
- instId = Inst::kIdMovsxd;
- if (typeOp == ((Type::kIdI64 << 8) | Type::kIdI32))
- break;
- }
-
- if (Type::isInt(srcTypeId) || src_.isMem()) {
- // Zero extend by using 'movzx' or 'mov'.
- if (dstSize <= 4 && srcSize < 4) {
- instId = Inst::kIdMovzx;
- dst.setSignature(Reg::signatureOfT<Reg::kTypeGpd>());
- }
- else {
- // We should have caught all possibilities where `srcSize` is less
- // than 4, so we don't have to worry about 'movzx' anymore. Minimum
- // size is enough to determine if we want 32-bit or 64-bit move.
- instId = Inst::kIdMov;
- srcSize = Support::min(srcSize, dstSize);
-
- dst.setSignature(srcSize == 4 ? Reg::signatureOfT<Reg::kTypeGpd>()
- : Reg::signatureOfT<Reg::kTypeGpq>());
- if (src.isReg())
- src.setSignature(dst.signature());
- }
- break;
- }
-
- // NOTE: The previous branch caught all memory sources, from here it's
- // always register to register conversion, so catch the remaining cases.
- srcSize = Support::min(srcSize, dstSize);
-
- if (Type::isMmx(srcTypeId)) {
- // 64-bit move.
- instId = Inst::kIdMovq;
- if (srcSize == 8)
- break;
-
- // 32-bit move.
- instId = Inst::kIdMovd;
- dst.setSignature(Reg::signatureOfT<Reg::kTypeGpd>());
- break;
- }
-
- if (Type::isMask(srcTypeId)) {
- instId = x86KmovFromSize(srcSize);
- dst.setSignature(srcSize <= 4 ? Reg::signatureOfT<Reg::kTypeGpd>()
- : Reg::signatureOfT<Reg::kTypeGpq>());
- break;
- }
-
- if (Type::isVec(srcTypeId)) {
- // 64-bit move.
- instId = avxEnabled ? Inst::kIdVmovq : Inst::kIdMovq;
- if (srcSize == 8)
- break;
-
- // 32-bit move.
- instId = avxEnabled ? Inst::kIdVmovd : Inst::kIdMovd;
- dst.setSignature(Reg::signatureOfT<Reg::kTypeGpd>());
- break;
- }
- }
-
- if (Type::isMmx(dstTypeId)) {
- instId = Inst::kIdMovq;
- srcSize = Support::min(srcSize, dstSize);
-
- if (Type::isInt(srcTypeId) || src.isMem()) {
- // 64-bit move.
- if (srcSize == 8)
- break;
-
- // 32-bit move.
- instId = Inst::kIdMovd;
- if (src.isReg())
- src.setSignature(Reg::signatureOfT<Reg::kTypeGpd>());
- break;
- }
-
- if (Type::isMmx(srcTypeId))
- break;
-
- // This will hurt if `avxEnabled`.
- instId = Inst::kIdMovdq2q;
- if (Type::isVec(srcTypeId))
-break;
- }
-
- if (Type::isMask(dstTypeId)) {
- srcSize = Support::min(srcSize, dstSize);
-
- if (Type::isInt(srcTypeId) || Type::isMask(srcTypeId) || src.isMem()) {
- instId = x86KmovFromSize(srcSize);
- if (Reg::isGp(src) && srcSize <= 4)
- src.setSignature(Reg::signatureOfT<Reg::kTypeGpd>());
- break;
- }
- }
-
- if (Type::isVec(dstTypeId)) {
- // By default set destination to XMM, will be set to YMM|ZMM if needed.
- dst.setSignature(Reg::signatureOfT<Reg::kTypeXmm>());
-
- // This will hurt if `avxEnabled`.
- if (Reg::isMm(src)) {
- // 64-bit move.
- instId = Inst::kIdMovq2dq;
- break;
- }
-
- // Argument conversion.
- uint32_t dstElement = Type::baseOf(dstTypeId);
- uint32_t srcElement = Type::baseOf(srcTypeId);
-
- if (dstElement == Type::kIdF32 && srcElement == Type::kIdF64) {
- srcSize = Support::min(dstSize * 2, srcSize);
- dstSize = srcSize / 2;
-
- if (srcSize <= 8)
- instId = avxEnabled ? Inst::kIdVcvtss2sd : Inst::kIdCvtss2sd;
- else
- instId = avxEnabled ? Inst::kIdVcvtps2pd : Inst::kIdCvtps2pd;
-
- if (dstSize == 32)
- dst.setSignature(Reg::signatureOfT<Reg::kTypeYmm>());
- if (src.isReg())
- src.setSignature(Reg::signatureOfVecBySize(srcSize));
- break;
- }
-
- if (dstElement == Type::kIdF64 && srcElement == Type::kIdF32) {
- srcSize = Support::min(dstSize, srcSize * 2) / 2;
- dstSize = srcSize * 2;
-
- if (srcSize <= 4)
- instId = avxEnabled ? Inst::kIdVcvtsd2ss : Inst::kIdCvtsd2ss;
- else
- instId = avxEnabled ? Inst::kIdVcvtpd2ps : Inst::kIdCvtpd2ps;
-
- dst.setSignature(Reg::signatureOfVecBySize(dstSize));
- if (src.isReg() && srcSize >= 32)
- src.setSignature(Reg::signatureOfT<Reg::kTypeYmm>());
- break;
- }
-
- srcSize = Support::min(srcSize, dstSize);
- if (Reg::isGp(src) || src.isMem()) {
- // 32-bit move.
- if (srcSize <= 4) {
- instId = avxEnabled ? Inst::kIdVmovd : Inst::kIdMovd;
- if (src.isReg())
- src.setSignature(Reg::signatureOfT<Reg::kTypeGpd>());
- break;
- }
-
- // 64-bit move.
- if (srcSize == 8) {
- instId = avxEnabled ? Inst::kIdVmovq : Inst::kIdMovq;
- break;
- }
- }
-
- if (Reg::isVec(src) || src.isMem()) {
- instId = avxEnabled ? Inst::kIdVmovaps : Inst::kIdMovaps;
-
- if (src.isMem() && srcSize < emitter->environment().stackAlignment())
- instId = avxEnabled ? Inst::kIdVmovups : Inst::kIdMovups;
-
- uint32_t signature = Reg::signatureOfVecBySize(srcSize);
- dst.setSignature(signature);
- if (src.isReg())
- src.setSignature(signature);
- break;
- }
- }
-
- return DebugUtils::errored(kErrorInvalidState);
- }
-
- if (src.isMem())
- src.as<Mem>().setSize(srcSize);
-
- emitter->setInlineComment(comment);
- return emitter->emit(instId, dst, src);
-}
-
-// ============================================================================
-// [asmjit::X86Internal - Emit Prolog & Epilog]
-// ============================================================================
-
-static ASMJIT_INLINE void X86Internal_setupSaveRestoreInfo(uint32_t group, const FuncFrame& frame, Reg& xReg, uint32_t& xInst, uint32_t& xSize) noexcept {
- switch (group) {
- case Reg::kGroupVec:
- xReg = xmm(0);
- xInst = x86GetXmmMovInst(frame);
- xSize = xReg.size();
- break;
- case Reg::kGroupMm:
- xReg = mm(0);
- xInst = Inst::kIdMovq;
- xSize = xReg.size();
- break;
- case Reg::kGroupKReg:
- xReg = k(0);
- xInst = Inst::kIdKmovq;
- xSize = xReg.size();
- break;
- }
-}
-
-ASMJIT_FAVOR_SIZE Error X86Internal::emitProlog(Emitter* emitter, const FuncFrame& frame) {
- uint32_t gpSaved = frame.savedRegs(Reg::kGroupGp);
-
- Gp zsp = emitter->zsp(); // ESP|RSP register.
- Gp zbp = emitter->zbp(); // EBP|RBP register.
- Gp gpReg = zsp; // General purpose register (temporary).
- Gp saReg = zsp; // Stack-arguments base pointer.
-
- // Emit: 'push zbp'
- // 'mov zbp, zsp'.
- if (frame.hasPreservedFP()) {
- gpSaved &= ~Support::bitMask(Gp::kIdBp);
- ASMJIT_PROPAGATE(emitter->push(zbp));
- ASMJIT_PROPAGATE(emitter->mov(zbp, zsp));
- }
-
- // Emit: 'push gp' sequence.
- {
- Support::BitWordIterator<uint32_t> it(gpSaved);
- while (it.hasNext()) {
- gpReg.setId(it.next());
- ASMJIT_PROPAGATE(emitter->push(gpReg));
- }
- }
-
- // Emit: 'mov saReg, zsp'.
- uint32_t saRegId = frame.saRegId();
- if (saRegId != BaseReg::kIdBad && saRegId != Gp::kIdSp) {
- saReg.setId(saRegId);
- if (frame.hasPreservedFP()) {
- if (saRegId != Gp::kIdBp)
- ASMJIT_PROPAGATE(emitter->mov(saReg, zbp));
- }
- else {
- ASMJIT_PROPAGATE(emitter->mov(saReg, zsp));
- }
- }
-
- // Emit: 'and zsp, StackAlignment'.
- if (frame.hasDynamicAlignment()) {
- ASMJIT_PROPAGATE(emitter->and_(zsp, -int32_t(frame.finalStackAlignment())));
- }
-
- // Emit: 'sub zsp, StackAdjustment'.
- if (frame.hasStackAdjustment()) {
- ASMJIT_PROPAGATE(emitter->sub(zsp, frame.stackAdjustment()));
- }
-
- // Emit: 'mov [zsp + DAOffset], saReg'.
- if (frame.hasDynamicAlignment() && frame.hasDAOffset()) {
- Mem saMem = ptr(zsp, int32_t(frame.daOffset()));
- ASMJIT_PROPAGATE(emitter->mov(saMem, saReg));
- }
-
- // Emit 'movxxx [zsp + X], {[x|y|z]mm, k}'.
- {
- Reg xReg;
- Mem xBase = ptr(zsp, int32_t(frame.nonGpSaveOffset()));
-
- uint32_t xInst;
- uint32_t xSize;
-
- for (uint32_t group = 1; group < BaseReg::kGroupVirt; group++) {
- Support::BitWordIterator<uint32_t> it(frame.savedRegs(group));
- if (it.hasNext()) {
- X86Internal_setupSaveRestoreInfo(group, frame, xReg, xInst, xSize);
- do {
- xReg.setId(it.next());
- ASMJIT_PROPAGATE(emitter->emit(xInst, xBase, xReg));
- xBase.addOffsetLo32(int32_t(xSize));
- } while (it.hasNext());
- }
- }
- }
-
- return kErrorOk;
-}
-
-ASMJIT_FAVOR_SIZE Error X86Internal::emitEpilog(Emitter* emitter, const FuncFrame& frame) {
- uint32_t i;
- uint32_t regId;
-
- uint32_t registerSize = emitter->registerSize();
- uint32_t gpSaved = frame.savedRegs(Reg::kGroupGp);
-
- Gp zsp = emitter->zsp(); // ESP|RSP register.
- Gp zbp = emitter->zbp(); // EBP|RBP register.
- Gp gpReg = emitter->zsp(); // General purpose register (temporary).
-
- // Don't emit 'pop zbp' in the pop sequence, this case is handled separately.
- if (frame.hasPreservedFP())
- gpSaved &= ~Support::bitMask(Gp::kIdBp);
-
- // Emit 'movxxx {[x|y|z]mm, k}, [zsp + X]'.
- {
- Reg xReg;
- Mem xBase = ptr(zsp, int32_t(frame.nonGpSaveOffset()));
-
- uint32_t xInst;
- uint32_t xSize;
-
- for (uint32_t group = 1; group < BaseReg::kGroupVirt; group++) {
- Support::BitWordIterator<uint32_t> it(frame.savedRegs(group));
- if (it.hasNext()) {
- X86Internal_setupSaveRestoreInfo(group, frame, xReg, xInst, xSize);
- do {
- xReg.setId(it.next());
- ASMJIT_PROPAGATE(emitter->emit(xInst, xReg, xBase));
- xBase.addOffsetLo32(int32_t(xSize));
- } while (it.hasNext());
- }
- }
- }
-
- // Emit 'emms' and/or 'vzeroupper'.
- if (frame.hasMmxCleanup()) ASMJIT_PROPAGATE(emitter->emms());
- if (frame.hasAvxCleanup()) ASMJIT_PROPAGATE(emitter->vzeroupper());
-
- if (frame.hasPreservedFP()) {
- // Emit 'mov zsp, zbp' or 'lea zsp, [zbp - x]'
- int32_t count = int32_t(frame.gpSaveSize() - registerSize);
- if (!count)
- ASMJIT_PROPAGATE(emitter->mov(zsp, zbp));
- else
- ASMJIT_PROPAGATE(emitter->lea(zsp, ptr(zbp, -count)));
- }
- else {
- if (frame.hasDynamicAlignment() && frame.hasDAOffset()) {
- // Emit 'mov zsp, [zsp + DsaSlot]'.
- Mem saMem = ptr(zsp, int32_t(frame.daOffset()));
- ASMJIT_PROPAGATE(emitter->mov(zsp, saMem));
- }
- else if (frame.hasStackAdjustment()) {
- // Emit 'add zsp, StackAdjustment'.
- ASMJIT_PROPAGATE(emitter->add(zsp, int32_t(frame.stackAdjustment())));
- }
- }
-
- // Emit 'pop gp' sequence.
- if (gpSaved) {
- i = gpSaved;
- regId = 16;
-
- do {
- regId--;
- if (i & 0x8000) {
- gpReg.setId(regId);
- ASMJIT_PROPAGATE(emitter->pop(gpReg));
- }
- i <<= 1;
- } while (regId != 0);
- }
-
- // Emit 'pop zbp'.
- if (frame.hasPreservedFP())
- ASMJIT_PROPAGATE(emitter->pop(zbp));
-
- // Emit 'ret' or 'ret x'.
- if (frame.hasCalleeStackCleanup())
- ASMJIT_PROPAGATE(emitter->emit(Inst::kIdRet, int(frame.calleeStackCleanup())));
- else
- ASMJIT_PROPAGATE(emitter->emit(Inst::kIdRet));
-
- return kErrorOk;
-}
-
-// ============================================================================
-// [asmjit::X86Internal - Emit Arguments Assignment]
-// ============================================================================
-
-#ifdef ASMJIT_DUMP_ARGS_ASSIGNMENT
-static void dumpFuncValue(String& sb, uint32_t arch, const FuncValue& value) noexcept {
- Formatter::formatTypeId(sb, value.typeId());
- sb.append('@');
-
- if (value.isIndirect())
- sb.append('[');
-
- if (value.isReg())
- Formatter::formatRegister(sb, 0, nullptr, arch, value.regType(), value.regId());
- else if (value.isStack())
- sb.appendFormat("[%d]", value.stackOffset());
- else
- sb.append("<none>");
-
- if (value.isIndirect())
- sb.append(']');
-}
-
-static void dumpAssignment(String& sb, const X86FuncArgsContext& ctx) noexcept {
- typedef X86FuncArgsContext::Var Var;
-
- uint32_t arch = ctx.arch();
- uint32_t varCount = ctx.varCount();
-
- for (uint32_t i = 0; i < varCount; i++) {
- const Var& var = ctx.var(i);
- const FuncValue& dst = var.out;
- const FuncValue& cur = var.cur;
-
- sb.appendFormat("Var%u: ", i);
- dumpFuncValue(sb, arch, dst);
- sb.append(" <- ");
- dumpFuncValue(sb, arch, cur);
-
- if (var.isDone())
- sb.append(" {Done}");
-
- sb.append('\n');
- }
-}
-#endif
-
-ASMJIT_FAVOR_SIZE Error X86Internal::emitArgsAssignment(Emitter* emitter, const FuncFrame& frame, const FuncArgsAssignment& args) {
- typedef X86FuncArgsContext::Var Var;
- typedef X86FuncArgsContext::WorkData WorkData;
-
- enum WorkFlags : uint32_t {
- kWorkNone = 0x00,
- kWorkDidSome = 0x01,
- kWorkPending = 0x02,
- kWorkPostponed = 0x04
- };
-
- X86FuncArgsContext ctx;
- ASMJIT_PROPAGATE(ctx.initWorkData(frame, args));
-
-#ifdef ASMJIT_DUMP_ARGS_ASSIGNMENT
- {
- String sb;
- dumpAssignment(sb, ctx);
- printf("%s\n", sb.data());
- }
-#endif
-
- uint32_t arch = ctx.arch();
- uint32_t varCount = ctx._varCount;
- WorkData* workData = ctx._workData;
-
- // Use AVX if it's enabled.
- bool avxEnabled = frame.isAvxEnabled();
-
- uint32_t saVarId = ctx._saVarId;
- uint32_t saRegId = Gp::kIdSp;
-
- if (frame.hasDynamicAlignment()) {
- if (frame.hasPreservedFP())
- saRegId = Gp::kIdBp;
- else
- saRegId = saVarId < varCount ? ctx._vars[saVarId].cur.regId() : frame.saRegId();
- }
-
- RegInfo gpRegInfo = emitter->_gpRegInfo;
-
- // --------------------------------------------------------------------------
- // Register to stack and stack to stack moves must be first as now we have
- // the biggest chance of having as many as possible unassigned registers.
- // --------------------------------------------------------------------------
-
- if (ctx._stackDstMask) {
- // Base address of all arguments passed by stack.
- Mem baseArgPtr = ptr(emitter->gpz(saRegId), int32_t(frame.saOffset(saRegId)));
- Mem baseStackPtr = ptr(emitter->gpz(Gp::kIdSp), int32_t(0));
-
- for (uint32_t varId = 0; varId < varCount; varId++) {
- Var& var = ctx._vars[varId];
-
- if (!var.out.isStack())
- continue;
-
- FuncValue& cur = var.cur;
- FuncValue& out = var.out;
-
- ASMJIT_ASSERT(cur.isReg() || cur.isStack());
- Reg reg;
-
- Mem dstStackPtr = baseStackPtr.cloneAdjusted(out.stackOffset());
- Mem srcStackPtr = baseArgPtr.cloneAdjusted(cur.stackOffset());
-
- if (cur.isIndirect()) {
- if (cur.isStack()) {
- // TODO: Indirect stack.
- return DebugUtils::errored(kErrorInvalidAssignment);
- }
- else {
- srcStackPtr = ptr(Gp(gpRegInfo.signature(), cur.regId()));
- }
- }
-
- if (cur.isReg() && !cur.isIndirect()) {
- WorkData& wd = workData[Reg::groupOf(cur.regType())];
- uint32_t rId = cur.regId();
-
- reg.setSignatureAndId(Reg::signatureOf(cur.regType()), rId);
- wd.unassign(varId, rId);
- }
- else {
- // Stack to reg move - tricky since we move stack to stack we can decide which
- // register to use. In general we follow the rule that IntToInt moves will use
- // GP regs with possibility to signature or zero extend, and all other moves will
- // either use GP or VEC regs depending on the size of the move.
- RegInfo rInfo = x86GetRegForMemToMemMove(arch, out.typeId(), cur.typeId());
- if (ASMJIT_UNLIKELY(!rInfo.isValid()))
- return DebugUtils::errored(kErrorInvalidState);
-
- WorkData& wd = workData[rInfo.group()];
- uint32_t availableRegs = wd.availableRegs();
- if (ASMJIT_UNLIKELY(!availableRegs))
- return DebugUtils::errored(kErrorInvalidState);
-
- uint32_t rId = Support::ctz(availableRegs);
- reg.setSignatureAndId(rInfo.signature(), rId);
-
- ASMJIT_PROPAGATE(emitArgMove(emitter, reg, out.typeId(), srcStackPtr, cur.typeId(), avxEnabled));
- }
-
- if (cur.isIndirect() && cur.isReg())
- workData[BaseReg::kGroupGp].unassign(varId, cur.regId());
-
- // Register to stack move.
- ASMJIT_PROPAGATE(emitRegMove(emitter, dstStackPtr, reg, cur.typeId(), avxEnabled));
- var.markDone();
- }
- }
-
- // --------------------------------------------------------------------------
- // Shuffle all registers that are currently assigned accordingly to target
- // assignment.
- // --------------------------------------------------------------------------
-
- uint32_t workFlags = kWorkNone;
- for (;;) {
- for (uint32_t varId = 0; varId < varCount; varId++) {
- Var& var = ctx._vars[varId];
- if (var.isDone() || !var.cur.isReg())
- continue;
-
- FuncValue& cur = var.cur;
- FuncValue& out = var.out;
-
- uint32_t curGroup = Reg::groupOf(cur.regType());
- uint32_t outGroup = Reg::groupOf(out.regType());
-
- uint32_t curId = cur.regId();
- uint32_t outId = out.regId();
-
- if (curGroup != outGroup) {
- // TODO: Conversion is not supported.
- return DebugUtils::errored(kErrorInvalidAssignment);
- }
- else {
- WorkData& wd = workData[outGroup];
- if (!wd.isAssigned(outId)) {
-EmitMove:
- ASMJIT_PROPAGATE(
- emitArgMove(emitter,
- Reg::fromTypeAndId(out.regType(), outId), out.typeId(),
- Reg::fromTypeAndId(cur.regType(), curId), cur.typeId(), avxEnabled));
-
- wd.reassign(varId, outId, curId);
- cur.initReg(out.regType(), outId, out.typeId());
-
- if (outId == out.regId())
- var.markDone();
- workFlags |= kWorkDidSome | kWorkPending;
- }
- else {
- uint32_t altId = wd._physToVarId[outId];
- Var& altVar = ctx._vars[altId];
-
- if (!altVar.out.isInitialized() || (altVar.out.isReg() && altVar.out.regId() == curId)) {
- // Swap operation is possible only between two GP registers.
- if (curGroup == Reg::kGroupGp) {
- uint32_t highestType = Support::max(cur.regType(), altVar.cur.regType());
- uint32_t signature = highestType == Reg::kTypeGpq ? Reg::signatureOfT<Reg::kTypeGpq>()
- : Reg::signatureOfT<Reg::kTypeGpd>();
-
- ASMJIT_PROPAGATE(emitter->emit(Inst::kIdXchg, Reg(signature, outId), Reg(signature, curId)));
- wd.swap(varId, curId, altId, outId);
- cur.setRegId(outId);
- var.markDone();
- altVar.cur.setRegId(curId);
-
- if (altVar.out.isInitialized())
- altVar.markDone();
- workFlags |= kWorkDidSome;
- }
- else {
- // If there is a scratch register it can be used to perform the swap.
- uint32_t availableRegs = wd.availableRegs();
- if (availableRegs) {
- uint32_t inOutRegs = wd.dstRegs();
- if (availableRegs & ~inOutRegs)
- availableRegs &= ~inOutRegs;
- outId = Support::ctz(availableRegs);
- goto EmitMove;
- }
- else {
- workFlags |= kWorkPending;
- }
- }
- }
- else {
- workFlags |= kWorkPending;
- }
- }
- }
- }
-
- if (!(workFlags & kWorkPending))
- break;
-
- // If we did nothing twice it means that something is really broken.
- if ((workFlags & (kWorkDidSome | kWorkPostponed)) == kWorkPostponed)
- return DebugUtils::errored(kErrorInvalidState);
-
- workFlags = (workFlags & kWorkDidSome) ? kWorkNone : kWorkPostponed;
- }
-
- // --------------------------------------------------------------------------
- // Load arguments passed by stack into registers. This is pretty simple and
- // it never requires multiple iterations like the previous phase.
- // --------------------------------------------------------------------------
-
- if (ctx._hasStackSrc) {
- uint32_t iterCount = 1;
- if (frame.hasDynamicAlignment() && !frame.hasPreservedFP())
- saRegId = saVarId < varCount ? ctx._vars[saVarId].cur.regId() : frame.saRegId();
-
- // Base address of all arguments passed by stack.
- Mem baseArgPtr = ptr(emitter->gpz(saRegId), int32_t(frame.saOffset(saRegId)));
-
- for (uint32_t iter = 0; iter < iterCount; iter++) {
- for (uint32_t varId = 0; varId < varCount; varId++) {
- Var& var = ctx._vars[varId];
- if (var.isDone())
- continue;
-
- if (var.cur.isStack()) {
- ASMJIT_ASSERT(var.out.isReg());
-
- uint32_t outId = var.out.regId();
- uint32_t outType = var.out.regType();
-
- uint32_t group = Reg::groupOf(outType);
- WorkData& wd = ctx._workData[group];
-
- if (outId == saRegId && group == BaseReg::kGroupGp) {
- // This register will be processed last as we still need `saRegId`.
- if (iterCount == 1) {
- iterCount++;
- continue;
- }
- wd.unassign(wd._physToVarId[outId], outId);
- }
-
- Reg dstReg = Reg::fromTypeAndId(outType, outId);
- Mem srcMem = baseArgPtr.cloneAdjusted(var.cur.stackOffset());
-
- ASMJIT_PROPAGATE(
- emitArgMove(emitter,
- dstReg, var.out.typeId(),
- srcMem, var.cur.typeId(), avxEnabled));
-
- wd.assign(varId, outId);
- var.cur.initReg(outType, outId, var.cur.typeId(), FuncValue::kFlagIsDone);
- }
- }
- }
- }
-
- return kErrorOk;
-}
-
-ASMJIT_END_SUB_NAMESPACE
-
-#endif // ASMJIT_BUILD_X86