diff options
Diffstat (limited to '3rdparty/asmjit/src/asmjit/x86/x86internal.cpp')
-rw-r--r-- | 3rdparty/asmjit/src/asmjit/x86/x86internal.cpp | 1733 |
1 files changed, 0 insertions, 1733 deletions
diff --git a/3rdparty/asmjit/src/asmjit/x86/x86internal.cpp b/3rdparty/asmjit/src/asmjit/x86/x86internal.cpp deleted file mode 100644 index 062525f5982..00000000000 --- a/3rdparty/asmjit/src/asmjit/x86/x86internal.cpp +++ /dev/null @@ -1,1733 +0,0 @@ -// AsmJit - Machine code generation for C++ -// -// * Official AsmJit Home Page: https://asmjit.com -// * Official Github Repository: https://github.com/asmjit/asmjit -// -// Copyright (c) 2008-2020 The AsmJit Authors -// -// This software is provided 'as-is', without any express or implied -// warranty. In no event will the authors be held liable for any damages -// arising from the use of this software. -// -// Permission is granted to anyone to use this software for any purpose, -// including commercial applications, and to alter it and redistribute it -// freely, subject to the following restrictions: -// -// 1. The origin of this software must not be misrepresented; you must not -// claim that you wrote the original software. If you use this software -// in a product, an acknowledgment in the product documentation would be -// appreciated but is not required. -// 2. Altered source versions must be plainly marked as such, and must not be -// misrepresented as being the original software. -// 3. This notice may not be removed or altered from any source distribution. - -#include "../core/api-build_p.h" -#ifdef ASMJIT_BUILD_X86 - -#include "../core/formatter.h" -#include "../core/string.h" -#include "../core/support.h" -#include "../core/type.h" -#include "../x86/x86internal_p.h" - -// Can be used for debugging... -// #define ASMJIT_DUMP_ARGS_ASSIGNMENT - -ASMJIT_BEGIN_SUB_NAMESPACE(x86) - -// ============================================================================ -// [asmjit::X86Internal - Helpers] -// ============================================================================ - -static ASMJIT_INLINE uint32_t x86GetXmmMovInst(const FuncFrame& frame) { - bool avx = frame.isAvxEnabled(); - bool aligned = frame.hasAlignedVecSR(); - - return aligned ? (avx ? Inst::kIdVmovaps : Inst::kIdMovaps) - : (avx ? Inst::kIdVmovups : Inst::kIdMovups); -} - -static ASMJIT_INLINE uint32_t x86VecTypeIdToRegType(uint32_t typeId) noexcept { - return typeId <= Type::_kIdVec128End ? Reg::kTypeXmm : - typeId <= Type::_kIdVec256End ? Reg::kTypeYmm : Reg::kTypeZmm; -} - -//! Converts `size` to a 'kmov?' instructio. -static inline uint32_t x86KmovFromSize(uint32_t size) noexcept { - switch (size) { - case 1: return Inst::kIdKmovb; - case 2: return Inst::kIdKmovw; - case 4: return Inst::kIdKmovd; - case 8: return Inst::kIdKmovq; - default: return Inst::kIdNone; - } -} - -// ============================================================================ -// [asmjit::X86Internal - FuncDetail] -// ============================================================================ - -ASMJIT_FAVOR_SIZE Error X86Internal::initFuncDetail(FuncDetail& func, const FuncSignature& signature, uint32_t registerSize) noexcept { - const CallConv& cc = func.callConv(); - uint32_t arch = cc.arch(); - uint32_t stackOffset = cc._spillZoneSize; - - uint32_t i; - uint32_t argCount = func.argCount(); - - if (func.retCount() != 0) { - uint32_t typeId = func._rets[0].typeId(); - switch (typeId) { - case Type::kIdI64: - case Type::kIdU64: { - if (Environment::is32Bit(arch)) { - // Convert a 64-bit return value to two 32-bit return values. - func._retCount = 2; - typeId -= 2; - - // 64-bit value is returned in EDX:EAX on X86. - func._rets[0].initReg(Reg::kTypeGpd, Gp::kIdAx, typeId); - func._rets[1].initReg(Reg::kTypeGpd, Gp::kIdDx, typeId); - break; - } - else { - func._rets[0].initReg(Reg::kTypeGpq, Gp::kIdAx, typeId); - } - break; - } - - case Type::kIdI8: - case Type::kIdI16: - case Type::kIdI32: { - func._rets[0].initReg(Reg::kTypeGpd, Gp::kIdAx, Type::kIdI32); - break; - } - - case Type::kIdU8: - case Type::kIdU16: - case Type::kIdU32: { - func._rets[0].initReg(Reg::kTypeGpd, Gp::kIdAx, Type::kIdU32); - break; - } - - case Type::kIdF32: - case Type::kIdF64: { - uint32_t regType = Environment::is32Bit(arch) ? Reg::kTypeSt : Reg::kTypeXmm; - func._rets[0].initReg(regType, 0, typeId); - break; - } - - case Type::kIdF80: { - // 80-bit floats are always returned by FP0. - func._rets[0].initReg(Reg::kTypeSt, 0, typeId); - break; - } - - case Type::kIdMmx32: - case Type::kIdMmx64: { - // MM registers are returned through XMM (SystemV) or GPQ (Win64). - uint32_t regType = Reg::kTypeMm; - if (Environment::is64Bit(arch)) - regType = cc.strategy() == CallConv::kStrategyDefault ? Reg::kTypeXmm : Reg::kTypeGpq; - - func._rets[0].initReg(regType, 0, typeId); - break; - } - - default: { - func._rets[0].initReg(x86VecTypeIdToRegType(typeId), 0, typeId); - break; - } - } - } - - switch (cc.strategy()) { - case CallConv::kStrategyDefault: { - uint32_t gpzPos = 0; - uint32_t vecPos = 0; - - for (i = 0; i < argCount; i++) { - FuncValue& arg = func._args[i]; - uint32_t typeId = arg.typeId(); - - if (Type::isInt(typeId)) { - uint32_t regId = BaseReg::kIdBad; - - if (gpzPos < CallConv::kMaxRegArgsPerGroup) - regId = cc._passedOrder[Reg::kGroupGp].id[gpzPos]; - - if (regId != BaseReg::kIdBad) { - uint32_t regType = (typeId <= Type::kIdU32) ? Reg::kTypeGpd : Reg::kTypeGpq; - arg.assignRegData(regType, regId); - func.addUsedRegs(Reg::kGroupGp, Support::bitMask(regId)); - gpzPos++; - } - else { - uint32_t size = Support::max<uint32_t>(Type::sizeOf(typeId), registerSize); - arg.assignStackOffset(int32_t(stackOffset)); - stackOffset += size; - } - continue; - } - - if (Type::isFloat(typeId) || Type::isVec(typeId)) { - uint32_t regId = BaseReg::kIdBad; - - if (vecPos < CallConv::kMaxRegArgsPerGroup) - regId = cc._passedOrder[Reg::kGroupVec].id[vecPos]; - - if (Type::isFloat(typeId)) { - // If this is a float, but `kFlagPassFloatsByVec` is false, we have - // to use stack instead. This should be only used by 32-bit calling - // conventions. - if (!cc.hasFlag(CallConv::kFlagPassFloatsByVec)) - regId = BaseReg::kIdBad; - } - else { - // Pass vector registers via stack if this is a variable arguments - // function. This should be only used by 32-bit calling conventions. - if (signature.hasVarArgs() && cc.hasFlag(CallConv::kFlagPassVecByStackIfVA)) - regId = BaseReg::kIdBad; - } - - if (regId != BaseReg::kIdBad) { - arg.initTypeId(typeId); - arg.assignRegData(x86VecTypeIdToRegType(typeId), regId); - func.addUsedRegs(Reg::kGroupVec, Support::bitMask(regId)); - vecPos++; - } - else { - uint32_t size = Type::sizeOf(typeId); - arg.assignStackOffset(int32_t(stackOffset)); - stackOffset += size; - } - continue; - } - } - break; - } - - case CallConv::kStrategyX64Windows: - case CallConv::kStrategyX64VectorCall: { - // Both X64 and VectorCall behave similarly - arguments are indexed - // from left to right. The position of the argument determines in - // which register the argument is allocated, so it's either GP or - // one of XMM/YMM/ZMM registers. - // - // [ X64 ] [VecCall] - // Index: #0 #1 #2 #3 #4 #5 - // - // GP : RCX RDX R8 R9 - // VEC : XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 - // - // For example function `f(int a, double b, int c, double d)` will be: - // - // (a) (b) (c) (d) - // RCX XMM1 R8 XMM3 - // - // Unused vector registers are used by HVA. - - bool isVectorCall = (cc.strategy() == CallConv::kStrategyX64VectorCall); - - for (i = 0; i < argCount; i++) { - FuncValue& arg = func._args[i]; - - uint32_t typeId = arg.typeId(); - uint32_t size = Type::sizeOf(typeId); - - if (Type::isInt(typeId) || Type::isMmx(typeId)) { - uint32_t regId = BaseReg::kIdBad; - - if (i < CallConv::kMaxRegArgsPerGroup) - regId = cc._passedOrder[Reg::kGroupGp].id[i]; - - if (regId != BaseReg::kIdBad) { - uint32_t regType = (size <= 4 && !Type::isMmx(typeId)) ? Reg::kTypeGpd : Reg::kTypeGpq; - arg.assignRegData(regType, regId); - func.addUsedRegs(Reg::kGroupGp, Support::bitMask(regId)); - } - else { - arg.assignStackOffset(int32_t(stackOffset)); - stackOffset += 8; - } - continue; - } - - if (Type::isFloat(typeId) || Type::isVec(typeId)) { - uint32_t regId = BaseReg::kIdBad; - - if (i < CallConv::kMaxRegArgsPerGroup) - regId = cc._passedOrder[Reg::kGroupVec].id[i]; - - if (regId != BaseReg::kIdBad) { - // X64-ABI doesn't allow vector types (XMM|YMM|ZMM) to be passed - // via registers, however, VectorCall was designed for that purpose. - if (Type::isFloat(typeId) || isVectorCall) { - uint32_t regType = x86VecTypeIdToRegType(typeId); - arg.assignRegData(regType, regId); - func.addUsedRegs(Reg::kGroupVec, Support::bitMask(regId)); - continue; - } - } - - // Passed via stack if the argument is float/double or indirectly. - // The trap is - if the argument is passed indirectly, the address - // can be passed via register, if the argument's index has GP one. - if (Type::isFloat(typeId)) { - arg.assignStackOffset(int32_t(stackOffset)); - } - else { - uint32_t gpRegId = cc._passedOrder[Reg::kGroupGp].id[i]; - if (gpRegId != BaseReg::kIdBad) - arg.assignRegData(Reg::kTypeGpq, gpRegId); - else - arg.assignStackOffset(int32_t(stackOffset)); - arg.addFlags(FuncValue::kFlagIsIndirect); - } - - // Always 8 bytes (float/double/pointer). - stackOffset += 8; - continue; - } - } - break; - } - } - - func._argStackSize = stackOffset; - return kErrorOk; -} - -// ============================================================================ -// [asmjit::X86FuncArgsContext] -// ============================================================================ - -static RegInfo x86GetRegForMemToMemMove(uint32_t arch, uint32_t dstTypeId, uint32_t srcTypeId) noexcept { - uint32_t dstSize = Type::sizeOf(dstTypeId); - uint32_t srcSize = Type::sizeOf(srcTypeId); - uint32_t maxSize = Support::max<uint32_t>(dstSize, srcSize); - uint32_t regSize = Environment::registerSizeFromArch(arch); - - uint32_t signature = 0; - if (maxSize <= regSize || (Type::isInt(dstTypeId) && Type::isInt(srcTypeId))) - signature = maxSize <= 4 ? Gpd::kSignature : Gpq::kSignature; - else if (maxSize <= 16) - signature = Xmm::kSignature; - else if (maxSize <= 32) - signature = Ymm::kSignature; - else if (maxSize <= 64) - signature = Zmm::kSignature; - - return RegInfo { signature }; -} - -// Used by both `argsToFuncFrame()` and `emitArgsAssignment()`. -class X86FuncArgsContext { -public: - enum VarId : uint32_t { - kVarIdNone = 0xFF - }; - - //! Contains information about a single argument or SA register that may need shuffling. - struct Var { - inline void init(const FuncValue& cur_, const FuncValue& out_) noexcept { - cur = cur_; - out = out_; - } - - //! Reset the value to its unassigned state. - inline void reset() noexcept { - cur.reset(); - out.reset(); - } - - inline bool isDone() const noexcept { return cur.isDone(); } - inline void markDone() noexcept { cur.addFlags(FuncValue::kFlagIsDone); } - - FuncValue cur; - FuncValue out; - }; - - struct WorkData { - inline void reset() noexcept { - _archRegs = 0; - _workRegs = 0; - _usedRegs = 0; - _assignedRegs = 0; - _dstRegs = 0; - _dstShuf = 0; - _numSwaps = 0; - _numStackArgs = 0; - memset(_reserved, 0, sizeof(_reserved)); - memset(_physToVarId, kVarIdNone, 32); - } - - inline bool isAssigned(uint32_t regId) const noexcept { - ASMJIT_ASSERT(regId < 32); - return Support::bitTest(_assignedRegs, regId); - } - - inline void assign(uint32_t varId, uint32_t regId) noexcept { - ASMJIT_ASSERT(!isAssigned(regId)); - ASMJIT_ASSERT(_physToVarId[regId] == kVarIdNone); - - _physToVarId[regId] = uint8_t(varId); - _assignedRegs ^= Support::bitMask(regId); - } - - inline void reassign(uint32_t varId, uint32_t newId, uint32_t oldId) noexcept { - ASMJIT_ASSERT( isAssigned(oldId)); - ASMJIT_ASSERT(!isAssigned(newId)); - ASMJIT_ASSERT(_physToVarId[oldId] == varId); - ASMJIT_ASSERT(_physToVarId[newId] == kVarIdNone); - - _physToVarId[oldId] = uint8_t(kVarIdNone); - _physToVarId[newId] = uint8_t(varId); - _assignedRegs ^= Support::bitMask(newId) ^ Support::bitMask(oldId); - } - - inline void swap(uint32_t aVarId, uint32_t aRegId, uint32_t bVarId, uint32_t bRegId) noexcept { - ASMJIT_ASSERT(isAssigned(aRegId)); - ASMJIT_ASSERT(isAssigned(bRegId)); - ASMJIT_ASSERT(_physToVarId[aRegId] == aVarId); - ASMJIT_ASSERT(_physToVarId[bRegId] == bVarId); - - _physToVarId[aRegId] = uint8_t(bVarId); - _physToVarId[bRegId] = uint8_t(aVarId); - } - - inline void unassign(uint32_t varId, uint32_t regId) noexcept { - ASMJIT_ASSERT(isAssigned(regId)); - ASMJIT_ASSERT(_physToVarId[regId] == varId); - - DebugUtils::unused(varId); - _physToVarId[regId] = uint8_t(kVarIdNone); - _assignedRegs ^= Support::bitMask(regId); - } - - inline uint32_t archRegs() const noexcept { return _archRegs; } - inline uint32_t workRegs() const noexcept { return _workRegs; } - inline uint32_t usedRegs() const noexcept { return _usedRegs; } - inline uint32_t assignedRegs() const noexcept { return _assignedRegs; } - inline uint32_t dstRegs() const noexcept { return _dstRegs; } - inline uint32_t availableRegs() const noexcept { return _workRegs & ~_assignedRegs; } - - uint32_t _archRegs; //!< All allocable registers provided by the architecture. - uint32_t _workRegs; //!< All registers that can be used by the shuffler. - uint32_t _usedRegs; //!< Registers used by the shuffler (all). - uint32_t _assignedRegs; //!< Assigned registers. - uint32_t _dstRegs; //!< Destination registers assigned to arguments or SA. - uint32_t _dstShuf; //!< Destination registers that require shuffling. - uint8_t _numSwaps; //!< Number of register swaps. - uint8_t _numStackArgs; //!< Number of stack loads. - uint8_t _reserved[6]; //!< Reserved (only used as padding). - uint8_t _physToVarId[32]; //!< Physical ID to variable ID mapping. - }; - - uint8_t _arch; - bool _hasStackSrc; //!< Has arguments passed via stack (SRC). - bool _hasPreservedFP; //!< Has preserved frame-pointer (FP). - uint8_t _stackDstMask; //!< Has arguments assigned to stack (DST). - uint8_t _regSwapsMask; //!< Register swap groups (bit-mask). - uint8_t _saVarId; - uint32_t _varCount; - WorkData _workData[BaseReg::kGroupVirt]; - Var _vars[kFuncArgCountLoHi + 1]; - - X86FuncArgsContext() noexcept; - - inline uint32_t arch() const noexcept { return _arch; } - inline uint32_t varCount() const noexcept { return _varCount; } - - inline Var& var(size_t varId) noexcept { return _vars[varId]; } - inline const Var& var(size_t varId) const noexcept { return _vars[varId]; } - inline size_t indexOf(const Var* var) const noexcept { return (size_t)(var - _vars); } - - Error initWorkData(const FuncFrame& frame, const FuncArgsAssignment& args) noexcept; - Error markScratchRegs(FuncFrame& frame) noexcept; - Error markDstRegsDirty(FuncFrame& frame) noexcept; - Error markStackArgsReg(FuncFrame& frame) noexcept; -}; - -X86FuncArgsContext::X86FuncArgsContext() noexcept { - _arch = Environment::kArchUnknown; - _varCount = 0; - _hasStackSrc = false; - _hasPreservedFP = false; - _stackDstMask = 0; - _regSwapsMask = 0; - _saVarId = kVarIdNone; - - for (uint32_t group = 0; group < BaseReg::kGroupVirt; group++) - _workData[group].reset(); -} - -ASMJIT_FAVOR_SIZE Error X86FuncArgsContext::initWorkData(const FuncFrame& frame, const FuncArgsAssignment& args) noexcept { - // The code has to be updated if this changes. - ASMJIT_ASSERT(BaseReg::kGroupVirt == 4); - - uint32_t i; - const FuncDetail& func = *args.funcDetail(); - - // Initialize Architecture. - uint32_t arch = func.callConv().arch(); - uint32_t archRegCount = Environment::is32Bit(arch) ? 8 : 16; - - _arch = uint8_t(arch); - - // Initialize `_archRegs`. - _workData[Reg::kGroupGp ]._archRegs = Support::lsbMask<uint32_t>(archRegCount) & ~Support::bitMask(Gp::kIdSp); - _workData[Reg::kGroupVec ]._archRegs = Support::lsbMask<uint32_t>(archRegCount); - _workData[Reg::kGroupMm ]._archRegs = Support::lsbMask<uint32_t>(8); - _workData[Reg::kGroupKReg]._archRegs = Support::lsbMask<uint32_t>(8); - - if (frame.hasPreservedFP()) - _workData[Reg::kGroupGp]._archRegs &= ~Support::bitMask(Gp::kIdBp); - - // Extract information from all function arguments/assignments and build Var[] array. - uint32_t varId = 0; - for (i = 0; i < kFuncArgCountLoHi; i++) { - const FuncValue& dst_ = args.arg(i); - if (!dst_.isAssigned()) - continue; - - const FuncValue& src_ = func.arg(i); - if (ASMJIT_UNLIKELY(!src_.isAssigned())) - return DebugUtils::errored(kErrorInvalidState); - - Var& var = _vars[varId]; - var.init(src_, dst_); - - FuncValue& src = var.cur; - FuncValue& dst = var.out; - - uint32_t dstGroup = 0xFFFFFFFFu; - uint32_t dstId = BaseReg::kIdBad; - WorkData* dstWd = nullptr; - - // Not supported. - if (src.isIndirect()) - return DebugUtils::errored(kErrorInvalidAssignment); - - if (dst.isReg()) { - uint32_t dstType = dst.regType(); - if (ASMJIT_UNLIKELY(dstType >= Reg::kTypeCount)) - return DebugUtils::errored(kErrorInvalidRegType); - - // Copy TypeId from source if the destination doesn't have it. The RA - // used by BaseCompiler would never leave TypeId undefined, but users - // of FuncAPI can just assign phys regs without specifying the type. - if (!dst.hasTypeId()) - dst.setTypeId(Reg::typeIdOf(dst.regType())); - - dstGroup = Reg::groupOf(dstType); - if (ASMJIT_UNLIKELY(dstGroup >= BaseReg::kGroupVirt)) - return DebugUtils::errored(kErrorInvalidRegGroup); - - dstWd = &_workData[dstGroup]; - dstId = dst.regId(); - if (ASMJIT_UNLIKELY(dstId >= 32 || !Support::bitTest(dstWd->archRegs(), dstId))) - return DebugUtils::errored(kErrorInvalidPhysId); - - if (ASMJIT_UNLIKELY(Support::bitTest(dstWd->dstRegs(), dstId))) - return DebugUtils::errored(kErrorOverlappedRegs); - - dstWd->_dstRegs |= Support::bitMask(dstId); - dstWd->_dstShuf |= Support::bitMask(dstId); - dstWd->_usedRegs |= Support::bitMask(dstId); - } - else { - if (!dst.hasTypeId()) - dst.setTypeId(src.typeId()); - - RegInfo regInfo = x86GetRegForMemToMemMove(arch, dst.typeId(), src.typeId()); - if (ASMJIT_UNLIKELY(!regInfo.isValid())) - return DebugUtils::errored(kErrorInvalidState); - _stackDstMask = uint8_t(_stackDstMask | Support::bitMask(regInfo.group())); - } - - if (src.isReg()) { - uint32_t srcId = src.regId(); - uint32_t srcGroup = Reg::groupOf(src.regType()); - - if (dstGroup == srcGroup) { - dstWd->assign(varId, srcId); - - // The best case, register is allocated where it is expected to be. - if (dstId == srcId) - var.markDone(); - } - else { - if (ASMJIT_UNLIKELY(srcGroup >= BaseReg::kGroupVirt)) - return DebugUtils::errored(kErrorInvalidState); - - WorkData& srcData = _workData[srcGroup]; - srcData.assign(varId, srcId); - } - } - else { - if (dstWd) - dstWd->_numStackArgs++; - _hasStackSrc = true; - } - - varId++; - } - - // Initialize WorkData::workRegs. - for (i = 0; i < BaseReg::kGroupVirt; i++) - _workData[i]._workRegs = (_workData[i].archRegs() & (frame.dirtyRegs(i) | ~frame.preservedRegs(i))) | _workData[i].dstRegs() | _workData[i].assignedRegs(); - - // Create a variable that represents `SARegId` if necessary. - bool saRegRequired = _hasStackSrc && frame.hasDynamicAlignment() && !frame.hasPreservedFP(); - - WorkData& gpRegs = _workData[BaseReg::kGroupGp]; - uint32_t saCurRegId = frame.saRegId(); - uint32_t saOutRegId = args.saRegId(); - - if (saCurRegId != BaseReg::kIdBad) { - // Check if the provided `SARegId` doesn't collide with input registers. - if (ASMJIT_UNLIKELY(gpRegs.isAssigned(saCurRegId))) - return DebugUtils::errored(kErrorOverlappedRegs); - } - - if (saOutRegId != BaseReg::kIdBad) { - // Check if the provided `SARegId` doesn't collide with argument assignments. - if (ASMJIT_UNLIKELY(Support::bitTest(gpRegs.dstRegs(), saOutRegId))) - return DebugUtils::errored(kErrorOverlappedRegs); - saRegRequired = true; - } - - if (saRegRequired) { - uint32_t ptrTypeId = Environment::is32Bit(arch) ? Type::kIdU32 : Type::kIdU64; - uint32_t ptrRegType = Environment::is32Bit(arch) ? BaseReg::kTypeGp32 : BaseReg::kTypeGp64; - - _saVarId = uint8_t(varId); - _hasPreservedFP = frame.hasPreservedFP(); - - Var& var = _vars[varId]; - var.reset(); - - if (saCurRegId == BaseReg::kIdBad) { - if (saOutRegId != BaseReg::kIdBad && !gpRegs.isAssigned(saOutRegId)) { - saCurRegId = saOutRegId; - } - else { - uint32_t availableRegs = gpRegs.availableRegs(); - if (!availableRegs) - availableRegs = gpRegs.archRegs() & ~gpRegs.workRegs(); - - if (ASMJIT_UNLIKELY(!availableRegs)) - return DebugUtils::errored(kErrorNoMorePhysRegs); - - saCurRegId = Support::ctz(availableRegs); - } - } - - var.cur.initReg(ptrRegType, saCurRegId, ptrTypeId); - gpRegs.assign(varId, saCurRegId); - gpRegs._workRegs |= Support::bitMask(saCurRegId); - - if (saOutRegId != BaseReg::kIdBad) { - var.out.initReg(ptrRegType, saOutRegId, ptrTypeId); - gpRegs._dstRegs |= Support::bitMask(saOutRegId); - gpRegs._workRegs |= Support::bitMask(saOutRegId); - } - else { - var.markDone(); - } - - varId++; - } - - _varCount = varId; - - // Detect register swaps. - for (varId = 0; varId < _varCount; varId++) { - Var& var = _vars[varId]; - if (var.cur.isReg() && var.out.isReg()) { - uint32_t srcId = var.cur.regId(); - uint32_t dstId = var.out.regId(); - - uint32_t group = Reg::groupOf(var.cur.regType()); - if (group != Reg::groupOf(var.out.regType())) - continue; - - WorkData& wd = _workData[group]; - if (wd.isAssigned(dstId)) { - Var& other = _vars[wd._physToVarId[dstId]]; - if (Reg::groupOf(other.out.regType()) == group && other.out.regId() == srcId) { - wd._numSwaps++; - _regSwapsMask = uint8_t(_regSwapsMask | Support::bitMask(group)); - } - } - } - } - - return kErrorOk; -} - -ASMJIT_FAVOR_SIZE Error X86FuncArgsContext::markDstRegsDirty(FuncFrame& frame) noexcept { - for (uint32_t i = 0; i < BaseReg::kGroupVirt; i++) { - WorkData& wd = _workData[i]; - uint32_t regs = wd.usedRegs() | wd._dstShuf; - - wd._workRegs |= regs; - frame.addDirtyRegs(i, regs); - } - - return kErrorOk; -} - -ASMJIT_FAVOR_SIZE Error X86FuncArgsContext::markScratchRegs(FuncFrame& frame) noexcept { - uint32_t groupMask = 0; - - // Handle stack to stack moves. - groupMask |= _stackDstMask; - - // Handle register swaps. - groupMask |= _regSwapsMask & ~Support::bitMask(BaseReg::kGroupGp); - - if (!groupMask) - return kErrorOk; - - // Selects one dirty register per affected group that can be used as a scratch register. - for (uint32_t group = 0; group < BaseReg::kGroupVirt; group++) { - if (Support::bitTest(groupMask, group)) { - WorkData& wd = _workData[group]; - - // Initially, pick some clobbered or dirty register. - uint32_t workRegs = wd.workRegs(); - uint32_t regs = workRegs & ~(wd.usedRegs() | wd._dstShuf); - - // If that didn't work out pick some register which is not in 'used'. - if (!regs) - regs = workRegs & ~wd.usedRegs(); - - // If that didn't work out pick any other register that is allocable. - // This last resort case will, however, result in marking one more - // register dirty. - if (!regs) - regs = wd.archRegs() & ~workRegs; - - // If that didn't work out we will have to use XORs instead of MOVs. - if (!regs) - continue; - - uint32_t regMask = Support::blsi(regs); - wd._workRegs |= regMask; - frame.addDirtyRegs(group, regMask); - } - } - - return kErrorOk; -} - -ASMJIT_FAVOR_SIZE Error X86FuncArgsContext::markStackArgsReg(FuncFrame& frame) noexcept { - if (_saVarId != kVarIdNone) { - const Var& var = _vars[_saVarId]; - frame.setSARegId(var.cur.regId()); - } - else if (frame.hasPreservedFP()) { - // Always EBP|RBP if the frame-pointer isn't omitted. - frame.setSARegId(Gp::kIdBp); - } - - return kErrorOk; -} - -// ============================================================================ -// [asmjit::X86Internal - FrameLayout] -// ============================================================================ - -ASMJIT_FAVOR_SIZE Error X86Internal::initFuncFrame(FuncFrame& frame, const FuncDetail& func) noexcept { - uint32_t arch = func.callConv().arch(); - - // Initializing FuncFrame means making a copy of some properties of `func`. - // Properties like `_localStackSize` will be set by the user before the frame - // is finalized. - frame.reset(); - - frame._arch = uint8_t(arch); - frame._spRegId = Gp::kIdSp; - frame._saRegId = Gp::kIdBad; - - uint32_t naturalStackAlignment = func.callConv().naturalStackAlignment(); - uint32_t minDynamicAlignment = Support::max<uint32_t>(naturalStackAlignment, 16); - - if (minDynamicAlignment == naturalStackAlignment) - minDynamicAlignment <<= 1; - - frame._naturalStackAlignment = uint8_t(naturalStackAlignment); - frame._minDynamicAlignment = uint8_t(minDynamicAlignment); - frame._redZoneSize = uint8_t(func.redZoneSize()); - frame._spillZoneSize = uint8_t(func.spillZoneSize()); - frame._finalStackAlignment = uint8_t(frame._naturalStackAlignment); - - if (func.hasFlag(CallConv::kFlagCalleePopsStack)) { - frame._calleeStackCleanup = uint16_t(func.argStackSize()); - } - - // Initial masks of dirty and preserved registers. - for (uint32_t group = 0; group < BaseReg::kGroupVirt; group++) { - frame._dirtyRegs[group] = func.usedRegs(group); - frame._preservedRegs[group] = func.preservedRegs(group); - } - - // Exclude ESP/RSP - this register is never included in saved GP regs. - frame._preservedRegs[BaseReg::kGroupGp] &= ~Support::bitMask(Gp::kIdSp); - - return kErrorOk; -} - -ASMJIT_FAVOR_SIZE Error X86Internal::finalizeFuncFrame(FuncFrame& frame) noexcept { - uint32_t registerSize = Environment::registerSizeFromArch(frame.arch()); - - // The final stack alignment must be updated accordingly to call and local stack alignments. - uint32_t stackAlignment = frame._finalStackAlignment; - ASMJIT_ASSERT(stackAlignment == Support::max(frame._naturalStackAlignment, - frame._callStackAlignment, - frame._localStackAlignment)); - - // TODO: Must be configurable. - uint32_t vecSize = 16; - - bool hasFP = frame.hasPreservedFP(); - bool hasDA = frame.hasDynamicAlignment(); - - // Include EBP|RBP if the function preserves the frame-pointer. - if (hasFP) - frame._dirtyRegs[Reg::kGroupGp] |= Support::bitMask(Gp::kIdBp); - - // These two are identical if the function doesn't align its stack dynamically. - uint32_t saRegId = frame.saRegId(); - if (saRegId == BaseReg::kIdBad) - saRegId = Gp::kIdSp; - - // Fix stack arguments base-register from ESP|RSP to EBP|RBP in case it was - // not picked before and the function performs dynamic stack alignment. - if (hasDA && saRegId == Gp::kIdSp) - saRegId = Gp::kIdBp; - - // Mark as dirty any register but ESP|RSP if used as SA pointer. - if (saRegId != Gp::kIdSp) - frame._dirtyRegs[Reg::kGroupGp] |= Support::bitMask(saRegId); - - frame._spRegId = uint8_t(Gp::kIdSp); - frame._saRegId = uint8_t(saRegId); - - // Setup stack size used to save preserved registers. - frame._gpSaveSize = uint16_t(Support::popcnt(frame.savedRegs(Reg::kGroupGp )) * registerSize); - frame._nonGpSaveSize = uint16_t(Support::popcnt(frame.savedRegs(Reg::kGroupVec )) * vecSize + - Support::popcnt(frame.savedRegs(Reg::kGroupMm )) * 8 + - Support::popcnt(frame.savedRegs(Reg::kGroupKReg)) * 8); - - uint32_t v = 0; // The beginning of the stack frame relative to SP after prolog. - v += frame.callStackSize(); // Count 'callStackSize' <- This is used to call functions. - v = Support::alignUp(v, stackAlignment); // Align to function's stack alignment. - - frame._localStackOffset = v; // Store 'localStackOffset' <- Function's local stack starts here. - v += frame.localStackSize(); // Count 'localStackSize' <- Function's local stack ends here. - - // If the function's stack must be aligned, calculate the alignment necessary - // to store vector registers, and set `FuncFrame::kAttrAlignedVecSR` to inform - // PEI that it can use instructions that perform aligned stores/loads. - if (stackAlignment >= vecSize && frame._nonGpSaveSize) { - frame.addAttributes(FuncFrame::kAttrAlignedVecSR); - v = Support::alignUp(v, vecSize); // Align '_nonGpSaveOffset'. - } - - frame._nonGpSaveOffset = v; // Store '_nonGpSaveOffset' <- Non-GP Save/Restore starts here. - v += frame._nonGpSaveSize; // Count '_nonGpSaveSize' <- Non-GP Save/Restore ends here. - - // Calculate if dynamic alignment (DA) slot (stored as offset relative to SP) is required and its offset. - if (hasDA && !hasFP) { - frame._daOffset = v; // Store 'daOffset' <- DA pointer would be stored here. - v += registerSize; // Count 'daOffset'. - } - else { - frame._daOffset = FuncFrame::kTagInvalidOffset; - } - - // The return address should be stored after GP save/restore regs. It has - // the same size as `registerSize` (basically the native register/pointer - // size). We don't adjust it now as `v` now contains the exact size that the - // function requires to adjust (call frame + stack frame, vec stack size). - // The stack (if we consider this size) is misaligned now, as it's always - // aligned before the function call - when `call()` is executed it pushes - // the current EIP|RIP onto the stack, and misaligns it by 12 or 8 bytes - // (depending on the architecture). So count number of bytes needed to align - // it up to the function's CallFrame (the beginning). - if (v || frame.hasFuncCalls()) - v += Support::alignUpDiff(v + frame.gpSaveSize() + registerSize, stackAlignment); - - frame._gpSaveOffset = v; // Store 'gpSaveOffset' <- Function's GP Save/Restore starts here. - frame._stackAdjustment = v; // Store 'stackAdjustment' <- SA used by 'add zsp, SA' and 'sub zsp, SA'. - - v += frame._gpSaveSize; // Count 'gpSaveSize' <- Function's GP Save/Restore ends here. - v += registerSize; // Count 'ReturnAddress' <- As CALL pushes onto stack. - - // If the function performs dynamic stack alignment then the stack-adjustment must be aligned. - if (hasDA) - frame._stackAdjustment = Support::alignUp(frame._stackAdjustment, stackAlignment); - - uint32_t saInvOff = FuncFrame::kTagInvalidOffset; - uint32_t saTmpOff = registerSize + frame._gpSaveSize; - - // Calculate where the function arguments start relative to SP. - frame._saOffsetFromSP = hasDA ? saInvOff : v; - - // Calculate where the function arguments start relative to FP or user-provided register. - frame._saOffsetFromSA = hasFP ? registerSize * 2 // Return address + frame pointer. - : saTmpOff; // Return address + all saved GP regs. - - return kErrorOk; -} - -// ============================================================================ -// [asmjit::X86Internal - ArgsToFrameInfo] -// ============================================================================ - -ASMJIT_FAVOR_SIZE Error X86Internal::argsToFuncFrame(const FuncArgsAssignment& args, FuncFrame& frame) noexcept { - X86FuncArgsContext ctx; - ASMJIT_PROPAGATE(ctx.initWorkData(frame, args)); - ASMJIT_PROPAGATE(ctx.markDstRegsDirty(frame)); - ASMJIT_PROPAGATE(ctx.markScratchRegs(frame)); - ASMJIT_PROPAGATE(ctx.markStackArgsReg(frame)); - return kErrorOk; -} - -// ============================================================================ -// [asmjit::X86Internal - Emit Helpers] -// ============================================================================ - -ASMJIT_FAVOR_SIZE Error X86Internal::emitRegMove(Emitter* emitter, - const Operand_& dst_, - const Operand_& src_, uint32_t typeId, bool avxEnabled, const char* comment) { - - // Invalid or abstract TypeIds are not allowed. - ASMJIT_ASSERT(Type::isValid(typeId) && !Type::isAbstract(typeId)); - - Operand dst(dst_); - Operand src(src_); - - uint32_t instId = Inst::kIdNone; - uint32_t memFlags = 0; - uint32_t overrideMemSize = 0; - - enum MemFlags : uint32_t { - kDstMem = 0x1, - kSrcMem = 0x2 - }; - - // Detect memory operands and patch them to have the same size as the register. - // BaseCompiler always sets memory size of allocs and spills, so it shouldn't - // be really necessary, however, after this function was separated from Compiler - // it's better to make sure that the size is always specified, as we can use - // 'movzx' and 'movsx' that rely on it. - if (dst.isMem()) { memFlags |= kDstMem; dst.as<Mem>().setSize(src.size()); } - if (src.isMem()) { memFlags |= kSrcMem; src.as<Mem>().setSize(dst.size()); } - - switch (typeId) { - case Type::kIdI8: - case Type::kIdU8: - case Type::kIdI16: - case Type::kIdU16: - // Special case - 'movzx' load. - if (memFlags & kSrcMem) { - instId = Inst::kIdMovzx; - dst.setSignature(Reg::signatureOfT<Reg::kTypeGpd>()); - } - else if (!memFlags) { - // Change both destination and source registers to GPD (safer, no dependencies). - dst.setSignature(Reg::signatureOfT<Reg::kTypeGpd>()); - src.setSignature(Reg::signatureOfT<Reg::kTypeGpd>()); - } - ASMJIT_FALLTHROUGH; - - case Type::kIdI32: - case Type::kIdU32: - case Type::kIdI64: - case Type::kIdU64: - instId = Inst::kIdMov; - break; - - case Type::kIdMmx32: - instId = Inst::kIdMovd; - if (memFlags) break; - ASMJIT_FALLTHROUGH; - - case Type::kIdMmx64 : instId = Inst::kIdMovq ; break; - case Type::kIdMask8 : instId = Inst::kIdKmovb; break; - case Type::kIdMask16: instId = Inst::kIdKmovw; break; - case Type::kIdMask32: instId = Inst::kIdKmovd; break; - case Type::kIdMask64: instId = Inst::kIdKmovq; break; - - default: { - uint32_t elementTypeId = Type::baseOf(typeId); - if (Type::isVec32(typeId) && memFlags) { - overrideMemSize = 4; - if (elementTypeId == Type::kIdF32) - instId = avxEnabled ? Inst::kIdVmovss : Inst::kIdMovss; - else - instId = avxEnabled ? Inst::kIdVmovd : Inst::kIdMovd; - break; - } - - if (Type::isVec64(typeId) && memFlags) { - overrideMemSize = 8; - if (elementTypeId == Type::kIdF64) - instId = avxEnabled ? Inst::kIdVmovsd : Inst::kIdMovsd; - else - instId = avxEnabled ? Inst::kIdVmovq : Inst::kIdMovq; - break; - } - - if (elementTypeId == Type::kIdF32) - instId = avxEnabled ? Inst::kIdVmovaps : Inst::kIdMovaps; - else if (elementTypeId == Type::kIdF64) - instId = avxEnabled ? Inst::kIdVmovapd : Inst::kIdMovapd; - else if (typeId <= Type::_kIdVec256End) - instId = avxEnabled ? Inst::kIdVmovdqa : Inst::kIdMovdqa; - else if (elementTypeId <= Type::kIdU32) - instId = Inst::kIdVmovdqa32; - else - instId = Inst::kIdVmovdqa64; - break; - } - } - - if (!instId) - return DebugUtils::errored(kErrorInvalidState); - - if (overrideMemSize) { - if (dst.isMem()) dst.as<Mem>().setSize(overrideMemSize); - if (src.isMem()) src.as<Mem>().setSize(overrideMemSize); - } - - emitter->setInlineComment(comment); - return emitter->emit(instId, dst, src); -} - -ASMJIT_FAVOR_SIZE Error X86Internal::emitArgMove(Emitter* emitter, - const Reg& dst_, uint32_t dstTypeId, - const Operand_& src_, uint32_t srcTypeId, bool avxEnabled, const char* comment) { - - // Deduce optional `dstTypeId`, which may be `Type::kIdVoid` in some cases. - if (!dstTypeId) - dstTypeId = opData.archRegs.regTypeToTypeId[dst_.type()]; - - // Invalid or abstract TypeIds are not allowed. - ASMJIT_ASSERT(Type::isValid(dstTypeId) && !Type::isAbstract(dstTypeId)); - ASMJIT_ASSERT(Type::isValid(srcTypeId) && !Type::isAbstract(srcTypeId)); - - Reg dst(dst_); - Operand src(src_); - - uint32_t dstSize = Type::sizeOf(dstTypeId); - uint32_t srcSize = Type::sizeOf(srcTypeId); - - uint32_t instId = Inst::kIdNone; - - // Not a real loop, just 'break' is nicer than 'goto'. - for (;;) { - if (Type::isInt(dstTypeId)) { - if (Type::isInt(srcTypeId)) { - instId = Inst::kIdMovsx; - uint32_t typeOp = (dstTypeId << 8) | srcTypeId; - - // Sign extend by using 'movsx'. - if (typeOp == ((Type::kIdI16 << 8) | Type::kIdI8 ) || - typeOp == ((Type::kIdI32 << 8) | Type::kIdI8 ) || - typeOp == ((Type::kIdI32 << 8) | Type::kIdI16) || - typeOp == ((Type::kIdI64 << 8) | Type::kIdI8 ) || - typeOp == ((Type::kIdI64 << 8) | Type::kIdI16)) - break; - - // Sign extend by using 'movsxd'. - instId = Inst::kIdMovsxd; - if (typeOp == ((Type::kIdI64 << 8) | Type::kIdI32)) - break; - } - - if (Type::isInt(srcTypeId) || src_.isMem()) { - // Zero extend by using 'movzx' or 'mov'. - if (dstSize <= 4 && srcSize < 4) { - instId = Inst::kIdMovzx; - dst.setSignature(Reg::signatureOfT<Reg::kTypeGpd>()); - } - else { - // We should have caught all possibilities where `srcSize` is less - // than 4, so we don't have to worry about 'movzx' anymore. Minimum - // size is enough to determine if we want 32-bit or 64-bit move. - instId = Inst::kIdMov; - srcSize = Support::min(srcSize, dstSize); - - dst.setSignature(srcSize == 4 ? Reg::signatureOfT<Reg::kTypeGpd>() - : Reg::signatureOfT<Reg::kTypeGpq>()); - if (src.isReg()) - src.setSignature(dst.signature()); - } - break; - } - - // NOTE: The previous branch caught all memory sources, from here it's - // always register to register conversion, so catch the remaining cases. - srcSize = Support::min(srcSize, dstSize); - - if (Type::isMmx(srcTypeId)) { - // 64-bit move. - instId = Inst::kIdMovq; - if (srcSize == 8) - break; - - // 32-bit move. - instId = Inst::kIdMovd; - dst.setSignature(Reg::signatureOfT<Reg::kTypeGpd>()); - break; - } - - if (Type::isMask(srcTypeId)) { - instId = x86KmovFromSize(srcSize); - dst.setSignature(srcSize <= 4 ? Reg::signatureOfT<Reg::kTypeGpd>() - : Reg::signatureOfT<Reg::kTypeGpq>()); - break; - } - - if (Type::isVec(srcTypeId)) { - // 64-bit move. - instId = avxEnabled ? Inst::kIdVmovq : Inst::kIdMovq; - if (srcSize == 8) - break; - - // 32-bit move. - instId = avxEnabled ? Inst::kIdVmovd : Inst::kIdMovd; - dst.setSignature(Reg::signatureOfT<Reg::kTypeGpd>()); - break; - } - } - - if (Type::isMmx(dstTypeId)) { - instId = Inst::kIdMovq; - srcSize = Support::min(srcSize, dstSize); - - if (Type::isInt(srcTypeId) || src.isMem()) { - // 64-bit move. - if (srcSize == 8) - break; - - // 32-bit move. - instId = Inst::kIdMovd; - if (src.isReg()) - src.setSignature(Reg::signatureOfT<Reg::kTypeGpd>()); - break; - } - - if (Type::isMmx(srcTypeId)) - break; - - // This will hurt if `avxEnabled`. - instId = Inst::kIdMovdq2q; - if (Type::isVec(srcTypeId)) -break; - } - - if (Type::isMask(dstTypeId)) { - srcSize = Support::min(srcSize, dstSize); - - if (Type::isInt(srcTypeId) || Type::isMask(srcTypeId) || src.isMem()) { - instId = x86KmovFromSize(srcSize); - if (Reg::isGp(src) && srcSize <= 4) - src.setSignature(Reg::signatureOfT<Reg::kTypeGpd>()); - break; - } - } - - if (Type::isVec(dstTypeId)) { - // By default set destination to XMM, will be set to YMM|ZMM if needed. - dst.setSignature(Reg::signatureOfT<Reg::kTypeXmm>()); - - // This will hurt if `avxEnabled`. - if (Reg::isMm(src)) { - // 64-bit move. - instId = Inst::kIdMovq2dq; - break; - } - - // Argument conversion. - uint32_t dstElement = Type::baseOf(dstTypeId); - uint32_t srcElement = Type::baseOf(srcTypeId); - - if (dstElement == Type::kIdF32 && srcElement == Type::kIdF64) { - srcSize = Support::min(dstSize * 2, srcSize); - dstSize = srcSize / 2; - - if (srcSize <= 8) - instId = avxEnabled ? Inst::kIdVcvtss2sd : Inst::kIdCvtss2sd; - else - instId = avxEnabled ? Inst::kIdVcvtps2pd : Inst::kIdCvtps2pd; - - if (dstSize == 32) - dst.setSignature(Reg::signatureOfT<Reg::kTypeYmm>()); - if (src.isReg()) - src.setSignature(Reg::signatureOfVecBySize(srcSize)); - break; - } - - if (dstElement == Type::kIdF64 && srcElement == Type::kIdF32) { - srcSize = Support::min(dstSize, srcSize * 2) / 2; - dstSize = srcSize * 2; - - if (srcSize <= 4) - instId = avxEnabled ? Inst::kIdVcvtsd2ss : Inst::kIdCvtsd2ss; - else - instId = avxEnabled ? Inst::kIdVcvtpd2ps : Inst::kIdCvtpd2ps; - - dst.setSignature(Reg::signatureOfVecBySize(dstSize)); - if (src.isReg() && srcSize >= 32) - src.setSignature(Reg::signatureOfT<Reg::kTypeYmm>()); - break; - } - - srcSize = Support::min(srcSize, dstSize); - if (Reg::isGp(src) || src.isMem()) { - // 32-bit move. - if (srcSize <= 4) { - instId = avxEnabled ? Inst::kIdVmovd : Inst::kIdMovd; - if (src.isReg()) - src.setSignature(Reg::signatureOfT<Reg::kTypeGpd>()); - break; - } - - // 64-bit move. - if (srcSize == 8) { - instId = avxEnabled ? Inst::kIdVmovq : Inst::kIdMovq; - break; - } - } - - if (Reg::isVec(src) || src.isMem()) { - instId = avxEnabled ? Inst::kIdVmovaps : Inst::kIdMovaps; - - if (src.isMem() && srcSize < emitter->environment().stackAlignment()) - instId = avxEnabled ? Inst::kIdVmovups : Inst::kIdMovups; - - uint32_t signature = Reg::signatureOfVecBySize(srcSize); - dst.setSignature(signature); - if (src.isReg()) - src.setSignature(signature); - break; - } - } - - return DebugUtils::errored(kErrorInvalidState); - } - - if (src.isMem()) - src.as<Mem>().setSize(srcSize); - - emitter->setInlineComment(comment); - return emitter->emit(instId, dst, src); -} - -// ============================================================================ -// [asmjit::X86Internal - Emit Prolog & Epilog] -// ============================================================================ - -static ASMJIT_INLINE void X86Internal_setupSaveRestoreInfo(uint32_t group, const FuncFrame& frame, Reg& xReg, uint32_t& xInst, uint32_t& xSize) noexcept { - switch (group) { - case Reg::kGroupVec: - xReg = xmm(0); - xInst = x86GetXmmMovInst(frame); - xSize = xReg.size(); - break; - case Reg::kGroupMm: - xReg = mm(0); - xInst = Inst::kIdMovq; - xSize = xReg.size(); - break; - case Reg::kGroupKReg: - xReg = k(0); - xInst = Inst::kIdKmovq; - xSize = xReg.size(); - break; - } -} - -ASMJIT_FAVOR_SIZE Error X86Internal::emitProlog(Emitter* emitter, const FuncFrame& frame) { - uint32_t gpSaved = frame.savedRegs(Reg::kGroupGp); - - Gp zsp = emitter->zsp(); // ESP|RSP register. - Gp zbp = emitter->zbp(); // EBP|RBP register. - Gp gpReg = zsp; // General purpose register (temporary). - Gp saReg = zsp; // Stack-arguments base pointer. - - // Emit: 'push zbp' - // 'mov zbp, zsp'. - if (frame.hasPreservedFP()) { - gpSaved &= ~Support::bitMask(Gp::kIdBp); - ASMJIT_PROPAGATE(emitter->push(zbp)); - ASMJIT_PROPAGATE(emitter->mov(zbp, zsp)); - } - - // Emit: 'push gp' sequence. - { - Support::BitWordIterator<uint32_t> it(gpSaved); - while (it.hasNext()) { - gpReg.setId(it.next()); - ASMJIT_PROPAGATE(emitter->push(gpReg)); - } - } - - // Emit: 'mov saReg, zsp'. - uint32_t saRegId = frame.saRegId(); - if (saRegId != BaseReg::kIdBad && saRegId != Gp::kIdSp) { - saReg.setId(saRegId); - if (frame.hasPreservedFP()) { - if (saRegId != Gp::kIdBp) - ASMJIT_PROPAGATE(emitter->mov(saReg, zbp)); - } - else { - ASMJIT_PROPAGATE(emitter->mov(saReg, zsp)); - } - } - - // Emit: 'and zsp, StackAlignment'. - if (frame.hasDynamicAlignment()) { - ASMJIT_PROPAGATE(emitter->and_(zsp, -int32_t(frame.finalStackAlignment()))); - } - - // Emit: 'sub zsp, StackAdjustment'. - if (frame.hasStackAdjustment()) { - ASMJIT_PROPAGATE(emitter->sub(zsp, frame.stackAdjustment())); - } - - // Emit: 'mov [zsp + DAOffset], saReg'. - if (frame.hasDynamicAlignment() && frame.hasDAOffset()) { - Mem saMem = ptr(zsp, int32_t(frame.daOffset())); - ASMJIT_PROPAGATE(emitter->mov(saMem, saReg)); - } - - // Emit 'movxxx [zsp + X], {[x|y|z]mm, k}'. - { - Reg xReg; - Mem xBase = ptr(zsp, int32_t(frame.nonGpSaveOffset())); - - uint32_t xInst; - uint32_t xSize; - - for (uint32_t group = 1; group < BaseReg::kGroupVirt; group++) { - Support::BitWordIterator<uint32_t> it(frame.savedRegs(group)); - if (it.hasNext()) { - X86Internal_setupSaveRestoreInfo(group, frame, xReg, xInst, xSize); - do { - xReg.setId(it.next()); - ASMJIT_PROPAGATE(emitter->emit(xInst, xBase, xReg)); - xBase.addOffsetLo32(int32_t(xSize)); - } while (it.hasNext()); - } - } - } - - return kErrorOk; -} - -ASMJIT_FAVOR_SIZE Error X86Internal::emitEpilog(Emitter* emitter, const FuncFrame& frame) { - uint32_t i; - uint32_t regId; - - uint32_t registerSize = emitter->registerSize(); - uint32_t gpSaved = frame.savedRegs(Reg::kGroupGp); - - Gp zsp = emitter->zsp(); // ESP|RSP register. - Gp zbp = emitter->zbp(); // EBP|RBP register. - Gp gpReg = emitter->zsp(); // General purpose register (temporary). - - // Don't emit 'pop zbp' in the pop sequence, this case is handled separately. - if (frame.hasPreservedFP()) - gpSaved &= ~Support::bitMask(Gp::kIdBp); - - // Emit 'movxxx {[x|y|z]mm, k}, [zsp + X]'. - { - Reg xReg; - Mem xBase = ptr(zsp, int32_t(frame.nonGpSaveOffset())); - - uint32_t xInst; - uint32_t xSize; - - for (uint32_t group = 1; group < BaseReg::kGroupVirt; group++) { - Support::BitWordIterator<uint32_t> it(frame.savedRegs(group)); - if (it.hasNext()) { - X86Internal_setupSaveRestoreInfo(group, frame, xReg, xInst, xSize); - do { - xReg.setId(it.next()); - ASMJIT_PROPAGATE(emitter->emit(xInst, xReg, xBase)); - xBase.addOffsetLo32(int32_t(xSize)); - } while (it.hasNext()); - } - } - } - - // Emit 'emms' and/or 'vzeroupper'. - if (frame.hasMmxCleanup()) ASMJIT_PROPAGATE(emitter->emms()); - if (frame.hasAvxCleanup()) ASMJIT_PROPAGATE(emitter->vzeroupper()); - - if (frame.hasPreservedFP()) { - // Emit 'mov zsp, zbp' or 'lea zsp, [zbp - x]' - int32_t count = int32_t(frame.gpSaveSize() - registerSize); - if (!count) - ASMJIT_PROPAGATE(emitter->mov(zsp, zbp)); - else - ASMJIT_PROPAGATE(emitter->lea(zsp, ptr(zbp, -count))); - } - else { - if (frame.hasDynamicAlignment() && frame.hasDAOffset()) { - // Emit 'mov zsp, [zsp + DsaSlot]'. - Mem saMem = ptr(zsp, int32_t(frame.daOffset())); - ASMJIT_PROPAGATE(emitter->mov(zsp, saMem)); - } - else if (frame.hasStackAdjustment()) { - // Emit 'add zsp, StackAdjustment'. - ASMJIT_PROPAGATE(emitter->add(zsp, int32_t(frame.stackAdjustment()))); - } - } - - // Emit 'pop gp' sequence. - if (gpSaved) { - i = gpSaved; - regId = 16; - - do { - regId--; - if (i & 0x8000) { - gpReg.setId(regId); - ASMJIT_PROPAGATE(emitter->pop(gpReg)); - } - i <<= 1; - } while (regId != 0); - } - - // Emit 'pop zbp'. - if (frame.hasPreservedFP()) - ASMJIT_PROPAGATE(emitter->pop(zbp)); - - // Emit 'ret' or 'ret x'. - if (frame.hasCalleeStackCleanup()) - ASMJIT_PROPAGATE(emitter->emit(Inst::kIdRet, int(frame.calleeStackCleanup()))); - else - ASMJIT_PROPAGATE(emitter->emit(Inst::kIdRet)); - - return kErrorOk; -} - -// ============================================================================ -// [asmjit::X86Internal - Emit Arguments Assignment] -// ============================================================================ - -#ifdef ASMJIT_DUMP_ARGS_ASSIGNMENT -static void dumpFuncValue(String& sb, uint32_t arch, const FuncValue& value) noexcept { - Formatter::formatTypeId(sb, value.typeId()); - sb.append('@'); - - if (value.isIndirect()) - sb.append('['); - - if (value.isReg()) - Formatter::formatRegister(sb, 0, nullptr, arch, value.regType(), value.regId()); - else if (value.isStack()) - sb.appendFormat("[%d]", value.stackOffset()); - else - sb.append("<none>"); - - if (value.isIndirect()) - sb.append(']'); -} - -static void dumpAssignment(String& sb, const X86FuncArgsContext& ctx) noexcept { - typedef X86FuncArgsContext::Var Var; - - uint32_t arch = ctx.arch(); - uint32_t varCount = ctx.varCount(); - - for (uint32_t i = 0; i < varCount; i++) { - const Var& var = ctx.var(i); - const FuncValue& dst = var.out; - const FuncValue& cur = var.cur; - - sb.appendFormat("Var%u: ", i); - dumpFuncValue(sb, arch, dst); - sb.append(" <- "); - dumpFuncValue(sb, arch, cur); - - if (var.isDone()) - sb.append(" {Done}"); - - sb.append('\n'); - } -} -#endif - -ASMJIT_FAVOR_SIZE Error X86Internal::emitArgsAssignment(Emitter* emitter, const FuncFrame& frame, const FuncArgsAssignment& args) { - typedef X86FuncArgsContext::Var Var; - typedef X86FuncArgsContext::WorkData WorkData; - - enum WorkFlags : uint32_t { - kWorkNone = 0x00, - kWorkDidSome = 0x01, - kWorkPending = 0x02, - kWorkPostponed = 0x04 - }; - - X86FuncArgsContext ctx; - ASMJIT_PROPAGATE(ctx.initWorkData(frame, args)); - -#ifdef ASMJIT_DUMP_ARGS_ASSIGNMENT - { - String sb; - dumpAssignment(sb, ctx); - printf("%s\n", sb.data()); - } -#endif - - uint32_t arch = ctx.arch(); - uint32_t varCount = ctx._varCount; - WorkData* workData = ctx._workData; - - // Use AVX if it's enabled. - bool avxEnabled = frame.isAvxEnabled(); - - uint32_t saVarId = ctx._saVarId; - uint32_t saRegId = Gp::kIdSp; - - if (frame.hasDynamicAlignment()) { - if (frame.hasPreservedFP()) - saRegId = Gp::kIdBp; - else - saRegId = saVarId < varCount ? ctx._vars[saVarId].cur.regId() : frame.saRegId(); - } - - RegInfo gpRegInfo = emitter->_gpRegInfo; - - // -------------------------------------------------------------------------- - // Register to stack and stack to stack moves must be first as now we have - // the biggest chance of having as many as possible unassigned registers. - // -------------------------------------------------------------------------- - - if (ctx._stackDstMask) { - // Base address of all arguments passed by stack. - Mem baseArgPtr = ptr(emitter->gpz(saRegId), int32_t(frame.saOffset(saRegId))); - Mem baseStackPtr = ptr(emitter->gpz(Gp::kIdSp), int32_t(0)); - - for (uint32_t varId = 0; varId < varCount; varId++) { - Var& var = ctx._vars[varId]; - - if (!var.out.isStack()) - continue; - - FuncValue& cur = var.cur; - FuncValue& out = var.out; - - ASMJIT_ASSERT(cur.isReg() || cur.isStack()); - Reg reg; - - Mem dstStackPtr = baseStackPtr.cloneAdjusted(out.stackOffset()); - Mem srcStackPtr = baseArgPtr.cloneAdjusted(cur.stackOffset()); - - if (cur.isIndirect()) { - if (cur.isStack()) { - // TODO: Indirect stack. - return DebugUtils::errored(kErrorInvalidAssignment); - } - else { - srcStackPtr = ptr(Gp(gpRegInfo.signature(), cur.regId())); - } - } - - if (cur.isReg() && !cur.isIndirect()) { - WorkData& wd = workData[Reg::groupOf(cur.regType())]; - uint32_t rId = cur.regId(); - - reg.setSignatureAndId(Reg::signatureOf(cur.regType()), rId); - wd.unassign(varId, rId); - } - else { - // Stack to reg move - tricky since we move stack to stack we can decide which - // register to use. In general we follow the rule that IntToInt moves will use - // GP regs with possibility to signature or zero extend, and all other moves will - // either use GP or VEC regs depending on the size of the move. - RegInfo rInfo = x86GetRegForMemToMemMove(arch, out.typeId(), cur.typeId()); - if (ASMJIT_UNLIKELY(!rInfo.isValid())) - return DebugUtils::errored(kErrorInvalidState); - - WorkData& wd = workData[rInfo.group()]; - uint32_t availableRegs = wd.availableRegs(); - if (ASMJIT_UNLIKELY(!availableRegs)) - return DebugUtils::errored(kErrorInvalidState); - - uint32_t rId = Support::ctz(availableRegs); - reg.setSignatureAndId(rInfo.signature(), rId); - - ASMJIT_PROPAGATE(emitArgMove(emitter, reg, out.typeId(), srcStackPtr, cur.typeId(), avxEnabled)); - } - - if (cur.isIndirect() && cur.isReg()) - workData[BaseReg::kGroupGp].unassign(varId, cur.regId()); - - // Register to stack move. - ASMJIT_PROPAGATE(emitRegMove(emitter, dstStackPtr, reg, cur.typeId(), avxEnabled)); - var.markDone(); - } - } - - // -------------------------------------------------------------------------- - // Shuffle all registers that are currently assigned accordingly to target - // assignment. - // -------------------------------------------------------------------------- - - uint32_t workFlags = kWorkNone; - for (;;) { - for (uint32_t varId = 0; varId < varCount; varId++) { - Var& var = ctx._vars[varId]; - if (var.isDone() || !var.cur.isReg()) - continue; - - FuncValue& cur = var.cur; - FuncValue& out = var.out; - - uint32_t curGroup = Reg::groupOf(cur.regType()); - uint32_t outGroup = Reg::groupOf(out.regType()); - - uint32_t curId = cur.regId(); - uint32_t outId = out.regId(); - - if (curGroup != outGroup) { - // TODO: Conversion is not supported. - return DebugUtils::errored(kErrorInvalidAssignment); - } - else { - WorkData& wd = workData[outGroup]; - if (!wd.isAssigned(outId)) { -EmitMove: - ASMJIT_PROPAGATE( - emitArgMove(emitter, - Reg::fromTypeAndId(out.regType(), outId), out.typeId(), - Reg::fromTypeAndId(cur.regType(), curId), cur.typeId(), avxEnabled)); - - wd.reassign(varId, outId, curId); - cur.initReg(out.regType(), outId, out.typeId()); - - if (outId == out.regId()) - var.markDone(); - workFlags |= kWorkDidSome | kWorkPending; - } - else { - uint32_t altId = wd._physToVarId[outId]; - Var& altVar = ctx._vars[altId]; - - if (!altVar.out.isInitialized() || (altVar.out.isReg() && altVar.out.regId() == curId)) { - // Swap operation is possible only between two GP registers. - if (curGroup == Reg::kGroupGp) { - uint32_t highestType = Support::max(cur.regType(), altVar.cur.regType()); - uint32_t signature = highestType == Reg::kTypeGpq ? Reg::signatureOfT<Reg::kTypeGpq>() - : Reg::signatureOfT<Reg::kTypeGpd>(); - - ASMJIT_PROPAGATE(emitter->emit(Inst::kIdXchg, Reg(signature, outId), Reg(signature, curId))); - wd.swap(varId, curId, altId, outId); - cur.setRegId(outId); - var.markDone(); - altVar.cur.setRegId(curId); - - if (altVar.out.isInitialized()) - altVar.markDone(); - workFlags |= kWorkDidSome; - } - else { - // If there is a scratch register it can be used to perform the swap. - uint32_t availableRegs = wd.availableRegs(); - if (availableRegs) { - uint32_t inOutRegs = wd.dstRegs(); - if (availableRegs & ~inOutRegs) - availableRegs &= ~inOutRegs; - outId = Support::ctz(availableRegs); - goto EmitMove; - } - else { - workFlags |= kWorkPending; - } - } - } - else { - workFlags |= kWorkPending; - } - } - } - } - - if (!(workFlags & kWorkPending)) - break; - - // If we did nothing twice it means that something is really broken. - if ((workFlags & (kWorkDidSome | kWorkPostponed)) == kWorkPostponed) - return DebugUtils::errored(kErrorInvalidState); - - workFlags = (workFlags & kWorkDidSome) ? kWorkNone : kWorkPostponed; - } - - // -------------------------------------------------------------------------- - // Load arguments passed by stack into registers. This is pretty simple and - // it never requires multiple iterations like the previous phase. - // -------------------------------------------------------------------------- - - if (ctx._hasStackSrc) { - uint32_t iterCount = 1; - if (frame.hasDynamicAlignment() && !frame.hasPreservedFP()) - saRegId = saVarId < varCount ? ctx._vars[saVarId].cur.regId() : frame.saRegId(); - - // Base address of all arguments passed by stack. - Mem baseArgPtr = ptr(emitter->gpz(saRegId), int32_t(frame.saOffset(saRegId))); - - for (uint32_t iter = 0; iter < iterCount; iter++) { - for (uint32_t varId = 0; varId < varCount; varId++) { - Var& var = ctx._vars[varId]; - if (var.isDone()) - continue; - - if (var.cur.isStack()) { - ASMJIT_ASSERT(var.out.isReg()); - - uint32_t outId = var.out.regId(); - uint32_t outType = var.out.regType(); - - uint32_t group = Reg::groupOf(outType); - WorkData& wd = ctx._workData[group]; - - if (outId == saRegId && group == BaseReg::kGroupGp) { - // This register will be processed last as we still need `saRegId`. - if (iterCount == 1) { - iterCount++; - continue; - } - wd.unassign(wd._physToVarId[outId], outId); - } - - Reg dstReg = Reg::fromTypeAndId(outType, outId); - Mem srcMem = baseArgPtr.cloneAdjusted(var.cur.stackOffset()); - - ASMJIT_PROPAGATE( - emitArgMove(emitter, - dstReg, var.out.typeId(), - srcMem, var.cur.typeId(), avxEnabled)); - - wd.assign(varId, outId); - var.cur.initReg(outType, outId, var.cur.typeId(), FuncValue::kFlagIsDone); - } - } - } - } - - return kErrorOk; -} - -ASMJIT_END_SUB_NAMESPACE - -#endif // ASMJIT_BUILD_X86 |