/*

* Copyright (C) 2012-2021 Apple Inc. All rights reserved.

*

* Redistribution and use in source and binary forms, with or without

* modification, are permitted provided that the following conditions

* are met:

* 1. Redistributions of source code must retain the above copyright

* notice, this list of conditions and the following disclaimer.

* 2. Redistributions in binary form must reproduce the above copyright

* notice, this list of conditions and the following disclaimer in the

* documentation and/or other materials provided with the distribution.

*

* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY

* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR

* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,

* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,

* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR

* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY

* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

*/

#pragma once

#if ENABLE(ASSEMBLER) && CPU(ARM64)

#include "ARM64Registers.h"

#include "AssemblerBuffer.h"

#include "AssemblerCommon.h"

#include "CPU.h"

#include "JSCPtrTag.h"

#include <limits.h>

#include <wtf/Assertions.h>

#include <wtf/Vector.h>

#include <stdint.h>

#if OS(FUCHSIA)

#include <zircon/syscalls.h>

#endif

#define CHECK_DATASIZE_OF(datasize) static_assert(datasize == 32 || datasize == 64)

#define CHECK_MEMOPSIZE_OF(size) static_assert(size == 8 || size == 16 || size == 32 || size == 64 || size == 128);

#define DATASIZE_OF(datasize) ((datasize == 64) ? Datasize_64 : Datasize_32)

#define MEMOPSIZE_OF(datasize) ((datasize == 8 || datasize == 128) ? MemOpSize_8_or_128 : (datasize == 16) ? MemOpSize_16 : (datasize == 32) ? MemOpSize_32 : MemOpSize_64)

#define CHECK_DATASIZE() CHECK_DATASIZE_OF(datasize)

#define CHECK_MEMOPSIZE() CHECK_MEMOPSIZE_OF(datasize)

#define CHECK_VECTOR_DATASIZE() ASSERT(datasize == 64 || datasize == 128)

#define DATASIZE DATASIZE_OF(datasize)

#define MEMOPSIZE MEMOPSIZE_OF(datasize)

#define CHECK_FP_MEMOP_DATASIZE() ASSERT(datasize == 8 || datasize == 16 || datasize == 32 || datasize == 64 || datasize == 128)

#define MEMPAIROPSIZE_INT(datasize) ((datasize == 64) ? MemPairOp_64 : MemPairOp_32)

#define MEMPAIROPSIZE_FP(datasize) ((datasize == 128) ? MemPairOp_V128 : (datasize == 64) ? MemPairOp_V64 : MemPairOp_32)

namespace JSC {

template<size_t bits, typename Type>

ALWAYS_INLINE constexpr bool isInt(Type t)

{

constexpr size_t shift = sizeof(Type) * CHAR_BIT - bits;

static_assert(sizeof(Type) * CHAR_BIT > shift, "shift is larger than the size of the value");

return ((t << shift) >> shift) == t;

}

static ALWAYS_INLINE bool is4ByteAligned(const void* ptr)

{

return !(reinterpret_cast<intptr_t>(ptr) & 0x3);

}

ALWAYS_INLINE bool isUInt5(int32_t value)

{

return !(value & ~0x1f);

}

class UInt5 {

public:

explicit UInt5(int value)

: m_value(value)

{

ASSERT(isUInt5(value));

}

operator int() { return m_value; }

private:

int m_value;

};

class UInt12 {

public:

explicit UInt12(int value)

: m_value(value)

{

ASSERT(isUInt12(value));

}

operator int() { return m_value; }

private:

int m_value;

};

class PostIndex {

public:

explicit PostIndex(int value)

: m_value(value)

{

ASSERT(isInt9(value));

}

operator int() { return m_value; }

private:

int m_value;

};

class PreIndex {

public:

explicit PreIndex(int value)

: m_value(value)

{

ASSERT(isInt9(value));

}

operator int() { return m_value; }

private:

int m_value;

};

class PairPostIndex {

public:

explicit PairPostIndex(int value)

: m_value(value)

{

ASSERT(isInt<11>(value));

}

operator int() { return m_value; }

private:

int m_value;

};

class PairPreIndex {

public:

explicit PairPreIndex(int value)

: m_value(value)

{

ASSERT(isInt<11>(value));

}

operator int() { return m_value; }

private:

int m_value;

};

typedef ARM64LogicalImmediate LogicalImmediate;

inline uint16_t getHalfword(uint64_t value, int which)

{

return value >> (which << 4);

}

namespace RegisterNames {

typedef enum : int8_t {

#define REGISTER_ID(id, name, r, cs) id,

FOR_EACH_GP_REGISTER(REGISTER_ID)

#undef REGISTER_ID

#define REGISTER_ALIAS(id, name, alias) id = alias,

FOR_EACH_REGISTER_ALIAS(REGISTER_ALIAS)

#undef REGISTER_ALIAS

InvalidGPRReg = -1,

} RegisterID;

typedef enum : int8_t {

#define REGISTER_ID(id, name) id,

FOR_EACH_SP_REGISTER(REGISTER_ID)

#undef REGISTER_ID

} SPRegisterID;

// ARM64 always has 32 FPU registers 128-bits each. See http://llvm.org/devmtg/2012-11/Northover-AArch64.pdf

// and Section 5.1.2 in http://infocenter.arm.com/help/topic/com.arm.doc.ihi0055b/IHI0055B_aapcs64.pdf.

// However, we only use them for 64-bit doubles.

typedef enum : int8_t {

#define REGISTER_ID(id, name, r, cs) id,

FOR_EACH_FP_REGISTER(REGISTER_ID)

#undef REGISTER_ID

InvalidFPRReg = -1,

} FPRegisterID;

static constexpr bool isSp(RegisterID reg) { return reg == sp; }

static constexpr bool isZr(RegisterID reg) { return reg == zr; }

} // namespace ARM64Registers

class ARM64Assembler {

public:

static constexpr size_t instructionSize = sizeof(unsigned);

typedef ARM64Registers::RegisterID RegisterID;

typedef ARM64Registers::SPRegisterID SPRegisterID;

typedef ARM64Registers::FPRegisterID FPRegisterID;

static constexpr RegisterID firstRegister() { return ARM64Registers::x0; }

static constexpr RegisterID lastRegister() { return ARM64Registers::sp; }

static constexpr unsigned numberOfRegisters() { return lastRegister() - firstRegister() + 1; }

static constexpr SPRegisterID firstSPRegister() { return ARM64Registers::pc; }

static constexpr SPRegisterID lastSPRegister() { return ARM64Registers::fpsr; }

static constexpr unsigned numberOfSPRegisters() { return lastSPRegister() - firstSPRegister() + 1; }

static constexpr FPRegisterID firstFPRegister() { return ARM64Registers::q0; }

static constexpr FPRegisterID lastFPRegister() { return ARM64Registers::q31; }

static constexpr unsigned numberOfFPRegisters() { return lastFPRegister() - firstFPRegister() + 1; }

static const char* gprName(RegisterID id)

{

ASSERT(id >= firstRegister() && id <= lastRegister());

static const char* const nameForRegister[numberOfRegisters()] = {

#define REGISTER_NAME(id, name, r, cs) name,

FOR_EACH_GP_REGISTER(REGISTER_NAME)

#undef REGISTER_NAME

};

return nameForRegister[id];

}

static const char* sprName(SPRegisterID id)

{

ASSERT(id >= firstSPRegister() && id <= lastSPRegister());

static const char* const nameForRegister[numberOfSPRegisters()] = {

#define REGISTER_NAME(id, name) name,

FOR_EACH_SP_REGISTER(REGISTER_NAME)

#undef REGISTER_NAME

};

return nameForRegister[id];

}

static const char* fprName(FPRegisterID id)

{

ASSERT(id >= firstFPRegister() && id <= lastFPRegister());

static const char* const nameForRegister[numberOfFPRegisters()] = {

#define REGISTER_NAME(id, name, r, cs) name,

FOR_EACH_FP_REGISTER(REGISTER_NAME)

#undef REGISTER_NAME

};

return nameForRegister[id];

}

protected:

static constexpr bool isSp(RegisterID reg) { return ARM64Registers::isSp(reg); }

static constexpr bool isZr(RegisterID reg) { return ARM64Registers::isZr(reg); }

public:

ARM64Assembler()

: m_indexOfLastWatchpoint(INT_MIN)

, m_indexOfTailOfLastWatchpoint(INT_MIN)

{

}

AssemblerBuffer& buffer() { return m_buffer; }

// (HS, LO, HI, LS) -> (AE, B, A, BE)

// (VS, VC) -> (O, NO)

typedef enum {

ConditionEQ,

ConditionNE,

ConditionHS, ConditionCS = ConditionHS,

ConditionLO, ConditionCC = ConditionLO,

ConditionMI,

ConditionPL,

ConditionVS,

ConditionVC,

ConditionHI,

ConditionLS,

ConditionGE,

ConditionLT,

ConditionGT,

ConditionLE,

ConditionAL,

ConditionInvalid

} Condition;

static Condition invert(Condition cond)

{

return static_cast<Condition>(cond ^ 1);

}

typedef enum {

LSL,

LSR,

ASR,

ROR

} ShiftType;

typedef enum {

UXTB,

UXTH,

UXTW,

UXTX,

SXTB,

SXTH,

SXTW,

SXTX

} ExtendType;

enum SetFlags {

DontSetFlags,

S

};

#define JUMP_ENUM_WITH_SIZE(index, value) (((value) << 4) | (index))

#define JUMP_ENUM_SIZE(jump) ((jump) >> 4)

enum JumpType { JumpFixed = JUMP_ENUM_WITH_SIZE(0, 0),

JumpNoCondition = JUMP_ENUM_WITH_SIZE(1, 1 * sizeof(uint32_t)),

JumpCondition = JUMP_ENUM_WITH_SIZE(2, 2 * sizeof(uint32_t)),

JumpCompareAndBranch = JUMP_ENUM_WITH_SIZE(3, 2 * sizeof(uint32_t)),

JumpTestBit = JUMP_ENUM_WITH_SIZE(4, 2 * sizeof(uint32_t)),

JumpNoConditionFixedSize = JUMP_ENUM_WITH_SIZE(5, 1 * sizeof(uint32_t)),

JumpConditionFixedSize = JUMP_ENUM_WITH_SIZE(6, 2 * sizeof(uint32_t)),

JumpCompareAndBranchFixedSize = JUMP_ENUM_WITH_SIZE(7, 2 * sizeof(uint32_t)),

JumpTestBitFixedSize = JUMP_ENUM_WITH_SIZE(8, 2 * sizeof(uint32_t)),

};

enum JumpLinkType {

LinkInvalid = JUMP_ENUM_WITH_SIZE(0, 0),

LinkJumpNoCondition = JUMP_ENUM_WITH_SIZE(1, 1 * sizeof(uint32_t)),

LinkJumpConditionDirect = JUMP_ENUM_WITH_SIZE(2, 1 * sizeof(uint32_t)),

LinkJumpCondition = JUMP_ENUM_WITH_SIZE(3, 2 * sizeof(uint32_t)),

LinkJumpCompareAndBranch = JUMP_ENUM_WITH_SIZE(4, 2 * sizeof(uint32_t)),

LinkJumpCompareAndBranchDirect = JUMP_ENUM_WITH_SIZE(5, 1 * sizeof(uint32_t)),

LinkJumpTestBit = JUMP_ENUM_WITH_SIZE(6, 2 * sizeof(uint32_t)),

LinkJumpTestBitDirect = JUMP_ENUM_WITH_SIZE(7, 1 * sizeof(uint32_t)),

};

class LinkRecord {

public:

LinkRecord(const ARM64Assembler* assembler, intptr_t from, intptr_t to, JumpType type, Condition condition)

{

data.realTypes.m_from = from;

#if CPU(ARM64E)

data.realTypes.m_to = tagInt(to, static_cast<PtrTag>(from ^ bitwise_cast<intptr_t>(assembler)));

#else

UNUSED_PARAM(assembler);

data.realTypes.m_to = to;

#endif

data.realTypes.m_type = type;

data.realTypes.m_linkType = LinkInvalid;

data.realTypes.m_condition = condition;

}

LinkRecord(const ARM64Assembler* assembler, intptr_t from, intptr_t to, JumpType type, Condition condition, bool is64Bit, RegisterID compareRegister)

{

data.realTypes.m_from = from;

#if CPU(ARM64E)

data.realTypes.m_to = tagInt(to, static_cast<PtrTag>(from ^ bitwise_cast<intptr_t>(assembler)));

#else

UNUSED_PARAM(assembler);

data.realTypes.m_to = to;

#endif

data.realTypes.m_type = type;

data.realTypes.m_linkType = LinkInvalid;

data.realTypes.m_condition = condition;

data.realTypes.m_is64Bit = is64Bit;

data.realTypes.m_compareRegister = compareRegister;

}

LinkRecord(const ARM64Assembler* assembler, intptr_t from, intptr_t to, JumpType type, Condition condition, unsigned bitNumber, RegisterID compareRegister)

{

data.realTypes.m_from = from;

#if CPU(ARM64E)

data.realTypes.m_to = tagInt(to, static_cast<PtrTag>(from ^ bitwise_cast<intptr_t>(assembler)));

#else

UNUSED_PARAM(assembler);

data.realTypes.m_to = to;

#endif

data.realTypes.m_type = type;

data.realTypes.m_linkType = LinkInvalid;

data.realTypes.m_condition = condition;

data.realTypes.m_bitNumber = bitNumber;

data.realTypes.m_compareRegister = compareRegister;

}

// We are defining a copy constructor and assignment operator

// because the ones provided by the compiler are not

// optimal. See https://bugs.webkit.org/show_bug.cgi?id=90930

LinkRecord(const LinkRecord& other)

{

data.copyTypes = other.data.copyTypes;

}

LinkRecord& operator=(const LinkRecord& other)

{

data.copyTypes = other.data.copyTypes;

return *this;

}

intptr_t from() const { return data.realTypes.m_from; }

void setFrom(const ARM64Assembler* assembler, intptr_t from)

{

#if CPU(ARM64E)

data.realTypes.m_to = tagInt(to(assembler), static_cast<PtrTag>(from ^ bitwise_cast<intptr_t>(assembler)));

#else

UNUSED_PARAM(assembler);

#endif

data.realTypes.m_from = from;

}

intptr_t to(const ARM64Assembler* assembler) const

{

#if CPU(ARM64E)

return untagInt(data.realTypes.m_to, static_cast<PtrTag>(data.realTypes.m_from ^ bitwise_cast<intptr_t>(assembler)));

#else

UNUSED_PARAM(assembler);

return data.realTypes.m_to;

#endif

}

JumpType type() const { return data.realTypes.m_type; }

JumpLinkType linkType() const { return data.realTypes.m_linkType; }

void setLinkType(JumpLinkType linkType) { ASSERT(data.realTypes.m_linkType == LinkInvalid); data.realTypes.m_linkType = linkType; }

Condition condition() const { return data.realTypes.m_condition; }

bool is64Bit() const { return data.realTypes.m_is64Bit; }

unsigned bitNumber() const { return data.realTypes.m_bitNumber; }

RegisterID compareRegister() const { return data.realTypes.m_compareRegister; }

private:

union {

struct RealTypes {

int64_t m_from;

int64_t m_to;

RegisterID m_compareRegister;

JumpType m_type : 8;

JumpLinkType m_linkType : 8;

Condition m_condition : 4;

unsigned m_bitNumber : 6;

bool m_is64Bit : 1;

} realTypes;

struct CopyTypes {

uint64_t content[3];

} copyTypes;

COMPILE_ASSERT(sizeof(RealTypes) == sizeof(CopyTypes), LinkRecordCopyStructSizeEqualsRealStruct);

} data;

};

// bits(N) VFPExpandImm(bits(8) imm8);

//

// Encoding of floating point immediates is a litte complicated. Here's a

// high level description:

// +/-m*2-n where m and n are integers, 16 <= m <= 31, 0 <= n <= 7

// and the algirithm for expanding to a single precision float:

// return imm8<7>:NOT(imm8<6>):Replicate(imm8<6>,5):imm8<5:0>:Zeros(19);

//

// The trickiest bit is how the exponent is handled. The following table

// may help clarify things a little:

// 654

// 100 01111100 124 -3 1020 01111111100

// 101 01111101 125 -2 1021 01111111101

// 110 01111110 126 -1 1022 01111111110

// 111 01111111 127 0 1023 01111111111

// 000 10000000 128 1 1024 10000000000

// 001 10000001 129 2 1025 10000000001

// 010 10000010 130 3 1026 10000000010

// 011 10000011 131 4 1027 10000000011

// The first column shows the bit pattern stored in bits 6-4 of the arm

// encoded immediate. The second column shows the 8-bit IEEE 754 single

// -precision exponent in binary, the third column shows the raw decimal

// value. IEEE 754 single-precision numbers are stored with a bias of 127

// to the exponent, so the fourth column shows the resulting exponent.

// From this was can see that the exponent can be in the range -3..4,

// which agrees with the high level description given above. The fifth

// and sixth columns shows the value stored in a IEEE 754 double-precision

// number to represent these exponents in decimal and binary, given the

// bias of 1023.

//

// Ultimately, detecting doubles that can be encoded as immediates on arm

// and encoding doubles is actually not too bad. A floating point value can

// be encoded by retaining the sign bit, the low three bits of the exponent

// and the high 4 bits of the mantissa. To validly be able to encode an

// immediate the remainder of the mantissa must be zero, and the high part

// of the exponent must match the top bit retained, bar the highest bit

// which must be its inverse.

static bool canEncodeFPImm(double d)

{

// Discard the sign bit, the low two bits of the exponent & the highest

// four bits of the mantissa.

uint64_t masked = bitwise_cast<uint64_t>(d) & 0x7fc0ffffffffffffull;

return (masked == 0x3fc0000000000000ull) || (masked == 0x4000000000000000ull);

}

template<int datasize>

static bool canEncodePImmOffset(int32_t offset)

{

return isValidScaledUImm12<datasize>(offset);

}

static bool canEncodeSImmOffset(int32_t offset)

{

return isValidSignedImm9(offset);

}

protected:

int encodeFPImm(double d)

{

ASSERT(canEncodeFPImm(d));

uint64_t u64 = bitwise_cast<uint64_t>(d);

return (static_cast<int>(u64 >> 56) & 0x80) | (static_cast<int>(u64 >> 48) & 0x7f);

}

template<int datasize>

int encodeShiftAmount(int amount)

{

ASSERT(!amount || datasize == (8 << amount));

return amount;

}

template<int datasize>

static int encodePositiveImmediate(unsigned pimm)

{

ASSERT(!(pimm & ((datasize / 8) - 1)));

return pimm / (datasize / 8);

}

enum Datasize {

Datasize_32,

Datasize_64,

Datasize_64_top,

Datasize_16

};

enum MemOpSize {

MemOpSize_8_or_128,

MemOpSize_16,

MemOpSize_32,

MemOpSize_64,

};

enum BranchType {

BranchType_JMP,

BranchType_CALL,

BranchType_RET

};

enum AddOp {

AddOp_ADD,

AddOp_SUB

};

enum BitfieldOp {

BitfieldOp_SBFM,

BitfieldOp_BFM,

BitfieldOp_UBFM

};

enum DataOp1Source {

DataOp_RBIT,

DataOp_REV16,

DataOp_REV32,

DataOp_REV64,

DataOp_CLZ,

DataOp_CLS

};

enum DataOp2Source {

DataOp_UDIV = 2,

DataOp_SDIV = 3,

DataOp_LSLV = 8,

DataOp_LSRV = 9,

DataOp_ASRV = 10,

DataOp_RORV = 11

};

enum DataOp3Source {

DataOp_MADD = 0,

DataOp_MSUB = 1,

DataOp_SMADDL = 2,

DataOp_SMSUBL = 3,

DataOp_SMULH = 4,

DataOp_UMADDL = 10,

DataOp_UMSUBL = 11,

DataOp_UMULH = 12

};

enum ExcepnOp {

ExcepnOp_EXCEPTION = 0,

ExcepnOp_BREAKPOINT = 1,

ExcepnOp_HALT = 2,

ExcepnOp_DCPS = 5

};

enum FPCmpOp {

FPCmpOp_FCMP = 0x00,

FPCmpOp_FCMP0 = 0x08,

FPCmpOp_FCMPE = 0x10,

FPCmpOp_FCMPE0 = 0x18

};

enum FPCondCmpOp {

FPCondCmpOp_FCMP,

FPCondCmpOp_FCMPE

};

enum FPDataOp1Source {

FPDataOp_FMOV = 0,

FPDataOp_FABS = 1,

FPDataOp_FNEG = 2,

FPDataOp_FSQRT = 3,

FPDataOp_FCVT_toSingle = 4,

FPDataOp_FCVT_toDouble = 5,

FPDataOp_FCVT_toHalf = 7,

FPDataOp_FRINTN = 8,

FPDataOp_FRINTP = 9,

FPDataOp_FRINTM = 10,

FPDataOp_FRINTZ = 11,

FPDataOp_FRINTA = 12,

FPDataOp_FRINTX = 14,

FPDataOp_FRINTI = 15

};

enum FPDataOp2Source {

FPDataOp_FMUL,

FPDataOp_FDIV,

FPDataOp_FADD,

FPDataOp_FSUB,

FPDataOp_FMAX,

FPDataOp_FMIN,

FPDataOp_FMAXNM,

FPDataOp_FMINNM,

FPDataOp_FNMUL

};

enum SIMD3Same {

SIMD_LogicalOp = 0x03

};

enum SIMD3SameLogical {

// This includes both the U bit and the "size" / opc for convience.

SIMD_LogicalOp_AND = 0x00,

SIMD_LogicalOp_BIC = 0x01,

SIMD_LogicalOp_ORR = 0x02,

SIMD_LogicalOp_ORN = 0x03,

SIMD_LogacalOp_EOR = 0x80,

SIMD_LogicalOp_BSL = 0x81,

SIMD_LogicalOp_BIT = 0x82,

SIMD_LogicalOp_BIF = 0x83,

};

enum FPIntConvOp {

FPIntConvOp_FCVTNS = 0x00,

FPIntConvOp_FCVTNU = 0x01,

FPIntConvOp_SCVTF = 0x02,

FPIntConvOp_UCVTF = 0x03,

FPIntConvOp_FCVTAS = 0x04,

FPIntConvOp_FCVTAU = 0x05,

FPIntConvOp_FMOV_QtoX = 0x06,

FPIntConvOp_FMOV_XtoQ = 0x07,

FPIntConvOp_FCVTPS = 0x08,

FPIntConvOp_FCVTPU = 0x09,

FPIntConvOp_FMOV_QtoX_top = 0x0e,

FPIntConvOp_FMOV_XtoQ_top = 0x0f,

FPIntConvOp_FCVTMS = 0x10,

FPIntConvOp_FCVTMU = 0x11,

FPIntConvOp_FCVTZS = 0x18,

FPIntConvOp_FCVTZU = 0x19,

};

enum LogicalOp {

LogicalOp_AND,

LogicalOp_ORR,

LogicalOp_EOR,

LogicalOp_ANDS

};

enum MemOp {

MemOp_STORE,

MemOp_LOAD,

MemOp_STORE_V128,

MemOp_LOAD_V128,

MemOp_PREFETCH = 2, // size must be 3

MemOp_LOAD_signed64 = 2, // size may be 0, 1 or 2

MemOp_LOAD_signed32 = 3 // size may be 0 or 1

};

enum MemPairOpSize {

MemPairOp_32 = 0,

MemPairOp_LoadSigned_32 = 1,

MemPairOp_64 = 2,

MemPairOp_V32 = MemPairOp_32,

MemPairOp_V64 = 1,

MemPairOp_V128 = 2

};

enum MoveWideOp {

MoveWideOp_N = 0,

MoveWideOp_Z = 2,

MoveWideOp_K = 3

};

enum LdrLiteralOp {

LdrLiteralOp_32BIT = 0,

LdrLiteralOp_64BIT = 1,

LdrLiteralOp_LDRSW = 2,

LdrLiteralOp_128BIT = 2

};

enum ExoticLoadFence {

ExoticLoadFence_None,

ExoticLoadFence_Acquire

};

enum ExoticLoadAtomic {

ExoticLoadAtomic_Link,

ExoticLoadAtomic_None

};

enum ExoticStoreFence {

ExoticStoreFence_None,

ExoticStoreFence_Release,

};

static unsigned memPairOffsetShift(bool V, MemPairOpSize size)

{

// return the log2 of the size in bytes, e.g. 64 bit size returns 3

if (V)

return size + 2;

return (size >> 1) + 2;

}

public:

// Integer Instructions:

template<int datasize, SetFlags setFlags = DontSetFlags>

ALWAYS_INLINE void adc(RegisterID rd, RegisterID rn, RegisterID rm)

{

CHECK_DATASIZE();

insn(addSubtractWithCarry(DATASIZE, AddOp_ADD, setFlags, rm, rn, rd));

}

template<int datasize, SetFlags setFlags = DontSetFlags>

ALWAYS_INLINE void add(RegisterID rd, RegisterID rn, UInt12 imm12, int shift = 0)

{

CHECK_DATASIZE();

ASSERT(!shift || shift == 12);

insn(addSubtractImmediate(DATASIZE, AddOp_ADD, setFlags, shift == 12, imm12, rn, rd));

}

template<int datasize, SetFlags setFlags = DontSetFlags>

ALWAYS_INLINE void add(RegisterID rd, RegisterID rn, RegisterID rm)

{

add<datasize, setFlags>(rd, rn, rm, LSL, 0);

}

template<int datasize, SetFlags setFlags = DontSetFlags>

ALWAYS_INLINE void add(RegisterID rd, RegisterID rn, RegisterID rm, ExtendType extend, int amount)

{

CHECK_DATASIZE();

insn(addSubtractExtendedRegister(DATASIZE, AddOp_ADD, setFlags, rm, extend, amount, rn, rd));

}

template<int datasize, SetFlags setFlags = DontSetFlags>

ALWAYS_INLINE void add(RegisterID rd, RegisterID rn, RegisterID rm, ShiftType shift, int amount)

{

CHECK_DATASIZE();

if (isSp(rd) || isSp(rn)) {

ASSERT(shift == LSL);

ASSERT(!isSp(rm));

add<datasize, setFlags>(rd, rn, rm, UXTX, amount);

} else

insn(addSubtractShiftedRegister(DATASIZE, AddOp_ADD, setFlags, shift, rm, amount, rn, rd));

}

ALWAYS_INLINE void adr(RegisterID rd, int offset)

{

insn(pcRelative(false, offset, rd));

}

ALWAYS_INLINE void adrp(RegisterID rd, int offset)

{

ASSERT(!(offset & 0xfff));

insn(pcRelative(true, offset >> 12, rd));

}

template<int datasize, SetFlags setFlags = DontSetFlags>

ALWAYS_INLINE void and_(RegisterID rd, RegisterID rn, RegisterID rm)

{

and_<datasize, setFlags>(rd, rn, rm, LSL, 0);

}

template<int datasize, SetFlags setFlags = DontSetFlags>

ALWAYS_INLINE void and_(RegisterID rd, RegisterID rn, RegisterID rm, ShiftType shift, int amount)

{

CHECK_DATASIZE();

insn(logicalShiftedRegister(DATASIZE, setFlags ? LogicalOp_ANDS : LogicalOp_AND, shift, false, rm, amount, rn, rd));

}

template<int datasize, SetFlags setFlags = DontSetFlags>

ALWAYS_INLINE void and_(RegisterID rd, RegisterID rn, LogicalImmediate imm)

{

CHECK_DATASIZE();

insn(logicalImmediate(DATASIZE, setFlags ? LogicalOp_ANDS : LogicalOp_AND, imm.value(), rn, rd));

}

template<int datasize>

ALWAYS_INLINE void asr(RegisterID rd, RegisterID rn, int shift)

{

ASSERT(shift < datasize);

sbfm<datasize>(rd, rn, shift, datasize - 1);

}

template<int datasize>

ALWAYS_INLINE void asr(RegisterID rd, RegisterID rn, RegisterID rm)

{

asrv<datasize>(rd, rn, rm);

}

template<int datasize>

ALWAYS_INLINE void asrv(RegisterID rd, RegisterID rn, RegisterID rm)

{

CHECK_DATASIZE();

insn(dataProcessing2Source(DATASIZE, rm, DataOp_ASRV, rn, rd));

}

ALWAYS_INLINE void b()

{

insn(unconditionalBranchImmediate(false, 0));

}

ALWAYS_INLINE void b_cond(Condition cond, int32_t offset = 0)

{

ASSERT(!(offset & 3));

offset >>= 2;

ASSERT(offset == (offset << 13) >> 13);

insn(conditionalBranchImmediate(offset, cond));

}

template<int datasize>

ALWAYS_INLINE void bfc(RegisterID rd, int lsb, int width)

{

bfi<datasize>(rd, ARM64Registers::zr, lsb, width);

}

template<int datasize>

ALWAYS_INLINE void bfi(RegisterID rd, RegisterID rn, int lsb, int width)

{

bfm<datasize>(rd, rn, (datasize - lsb) & (datasize - 1), width - 1);

}

template<int datasize>

ALWAYS_INLINE void bfm(RegisterID rd, RegisterID rn, int immr, int imms)

{

CHECK_DATASIZE();

insn(bitfield(DATASIZE, BitfieldOp_BFM, immr, imms, rn, rd));

}

template<int datasize>

ALWAYS_INLINE void bfxil(RegisterID rd, RegisterID rn, int lsb, int width)

{

bfm<datasize>(rd, rn, lsb, lsb + width - 1);

}

template<int datasize, SetFlags setFlags = DontSetFlags>

ALWAYS_INLINE void bic(RegisterID rd, RegisterID rn, RegisterID rm)

{

bic<datasize, setFlags>(rd, rn, rm, LSL, 0);

}

template<int datasize, SetFlags setFlags = DontSetFlags>

ALWAYS_INLINE void bic(RegisterID rd, RegisterID rn, RegisterID rm, ShiftType shift, int amount)

{

CHECK_DATASIZE();

insn(logicalShiftedRegister(DATASIZE, setFlags ? LogicalOp_ANDS : LogicalOp_AND, shift, true, rm, amount, rn, rd));

}

ALWAYS_INLINE void bl()

{

insn(unconditionalBranchImmediate(true, 0));

}

ALWAYS_INLINE void blr(RegisterID rn)

{

insn(unconditionalBranchRegister(BranchType_CALL, rn));

}

ALWAYS_INLINE void br(RegisterID rn)

{

insn(unconditionalBranchRegister(BranchType_JMP, rn));

}

ALWAYS_INLINE void brk(uint16_t imm)

{

insn(excepnGeneration(ExcepnOp_BREAKPOINT, imm, 0));

}

ALWAYS_INLINE static bool isBrk(void* address)

{

int expected = excepnGeneration(ExcepnOp_BREAKPOINT, 0, 0);

int immediateMask = excepnGenerationImmMask();

int candidateInstruction = *reinterpret_cast<int*>(address);

return (candidateInstruction & ~immediateMask) == expected;

}

template<int datasize>

ALWAYS_INLINE void cbnz(RegisterID rt, int32_t offset = 0)

{

CHECK_DATASIZE();

ASSERT(!(offset & 3));

offset >>= 2;

insn(compareAndBranchImmediate(DATASIZE, true, offset, rt));

}

template<int datasize>

ALWAYS_INLINE void cbz(RegisterID rt, int32_t offset = 0)

{

CHECK_DATASIZE();

ASSERT(!(offset & 3));

offset >>= 2;

insn(compareAndBranchImmediate(DATASIZE, false, offset, rt));

}

template<int datasize>

ALWAYS_INLINE void ccmn(RegisterID rn, RegisterID rm, int nzcv, Condition cond)

{

CHECK_DATASIZE();

insn(conditionalCompareRegister(DATASIZE, AddOp_ADD, rm, cond, rn, nzcv));

}

template<int datasize>

ALWAYS_INLINE void ccmn(RegisterID rn, UInt5 imm, int nzcv, Condition cond)

{

CHECK_DATASIZE();

insn(conditionalCompareImmediate(DATASIZE, AddOp_ADD, imm, cond, rn, nzcv));

}

template<int datasize>

ALWAYS_INLINE void ccmp(RegisterID rn, RegisterID rm, int nzcv, Condition cond)

{

CHECK_DATASIZE();

insn(conditionalCompareRegister(DATASIZE, AddOp_SUB, rm, cond, rn, nzcv));

}

template<int datasize>

ALWAYS_INLINE void ccmp(RegisterID rn, UInt5 imm, int nzcv, Condition cond)

{

CHECK_DATASIZE();

insn(conditionalCompareImmediate(DATASIZE, AddOp_SUB, imm, cond, rn, nzcv));

}

template<int datasize>

ALWAYS_INLINE void cinc(RegisterID rd, RegisterID rn, Condition cond)

{

csinc<datasize>(rd, rn, rn, invert(cond));

}

template<int datasize>

ALWAYS_INLINE void cinv(RegisterID rd, RegisterID rn, Condition cond)

{

csinv<datasize>(rd, rn, rn, invert(cond));

}

template<int datasize>

ALWAYS_INLINE void cls(RegisterID rd, RegisterID rn)

{

CHECK_DATASIZE();

insn(dataProcessing1Source(DATASIZE, DataOp_CLS, rn, rd));

}

template<int datasize>

ALWAYS_INLINE void clz(RegisterID rd, RegisterID rn)

{

CHECK_DATASIZE();

insn(dataProcessing1Source(DATASIZE, DataOp_CLZ, rn, rd));

}

template<int datasize>

ALWAYS_INLINE void cmn(RegisterID rn, UInt12 imm12, int shift = 0)

{

add<datasize, S>(ARM64Registers::zr, rn, imm12, shift);

}

template<int datasize>

ALWAYS_INLINE void cmn(RegisterID rn, RegisterID rm)

{

add<datasize, S>(ARM64Registers::zr, rn, rm);

}

template<int datasize>

ALWAYS_INLINE void cmn(RegisterID rn, RegisterID rm, ExtendType extend, int amount)

{

add<datasize, S>(ARM64Registers::zr, rn, rm, extend, amount);

}

template<int datasize>

ALWAYS_INLINE void cmn(RegisterID rn, RegisterID rm, ShiftType shift, int amount)

{

add<datasize, S>(ARM64Registers::zr, rn, rm, shift, amount);

}

template<int datasize>

ALWAYS_INLINE void cmp(RegisterID rn, UInt12 imm12, int shift = 0)

{

sub<datasize, S>(ARM64Registers::zr, rn, imm12, shift);