/*

* Copyright (C) 2019-2022 Apple Inc. All rights reserved.

*

* Redistribution and use in source and binary forms, with or without

* modification, are permitted provided that the following conditions

* are met:

* 1. Redistributions of source code must retain the above copyright

* notice, this list of conditions and the following disclaimer.

* 2. Redistributions in binary form must reproduce the above copyright

* notice, this list of conditions and the following disclaimer in the

* documentation and/or other materials provided with the distribution.

*

* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY

* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR

* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,

* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,

* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR

* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY

* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

*/

#include "config.h"

#include "WasmAirIRGenerator.h"

#if ENABLE(WEBASSEMBLY_B3JIT)

#include "AirCode.h"

#include "AirGenerate.h"

#include "AirHelpers.h"

#include "AirOpcodeUtils.h"

#include "AllowMacroScratchRegisterUsageIf.h"

#include "B3CheckSpecial.h"

#include "B3CheckValue.h"

#include "B3Commutativity.h"

#include "B3PatchpointSpecial.h"

#include "B3Procedure.h"

#include "B3ProcedureInlines.h"

#include "B3StackmapGenerationParams.h"

#include "BinarySwitch.h"

#include "JSCJSValueInlines.h"

#include "JSWebAssemblyInstance.h"

#include "ScratchRegisterAllocator.h"

#include "WasmCallingConvention.h"

#include "WasmContextInlines.h"

#include "WasmExceptionType.h"

#include "WasmFunctionParser.h"

#include "WasmIRGeneratorHelpers.h"

#include "WasmInstance.h"

#include "WasmMemory.h"

#include "WasmOSREntryData.h"

#include "WasmOpcodeOrigin.h"

#include "WasmOperations.h"

#include "WasmSignatureInlines.h"

#include "WasmThunks.h"

#include <limits>

#include <wtf/Box.h>

#include <wtf/StdLibExtras.h>

namespace JSC { namespace Wasm {

using namespace B3::Air;

struct ConstrainedTmp {

ConstrainedTmp() = default;

ConstrainedTmp(Tmp tmp)

: ConstrainedTmp(tmp, tmp.isReg() ? B3::ValueRep::reg(tmp.reg()) : B3::ValueRep::SomeRegister)

{ }

ConstrainedTmp(Tmp tmp, B3::ValueRep rep)

: tmp(tmp)

, rep(rep)

{

}

explicit operator bool() const { return !!tmp; }

Tmp tmp;

B3::ValueRep rep;

};

class TypedTmp {

public:

constexpr TypedTmp()

: m_tmp()

, m_type(Types::Void)

{

}

TypedTmp(Tmp tmp, Type type)

: m_tmp(tmp)

, m_type(type)

{ }

TypedTmp(const TypedTmp&) = default;

TypedTmp(TypedTmp&&) = default;

TypedTmp& operator=(TypedTmp&&) = default;

TypedTmp& operator=(const TypedTmp&) = default;

bool operator==(const TypedTmp& other) const

{

return m_tmp == other.m_tmp && m_type == other.m_type;

}

bool operator!=(const TypedTmp& other) const

{

return !(*this == other);

}

explicit operator bool() const { return !!tmp(); }

operator Tmp() const { return tmp(); }

operator Arg() const { return Arg(tmp()); }

Tmp tmp() const { return m_tmp; }

Type type() const { return m_type; }

void dump(PrintStream& out) const

{

out.print("(", m_tmp, ", ", m_type.kind, ", ", m_type.index, ")");

}

private:

Tmp m_tmp;

Type m_type;

};

class AirIRGenerator {

public:

using ExpressionType = TypedTmp;

using ResultList = Vector<ExpressionType, 8>;

struct ControlData {

ControlData(B3::Origin, BlockSignature result, ResultList resultTmps, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)

: controlBlockType(type)

, continuation(continuation)

, special(special)

, results(resultTmps)

, returnType(result)

{

}

ControlData(B3::Origin, BlockSignature result, ResultList resultTmps, BlockType type, BasicBlock* continuation, unsigned tryStart, unsigned tryDepth)

: controlBlockType(type)

, continuation(continuation)

, special(nullptr)

, results(resultTmps)

, returnType(result)

, m_tryStart(tryStart)

, m_tryCatchDepth(tryDepth)

{

}

ControlData()

{

}

static bool isIf(const ControlData& control) { return control.blockType() == BlockType::If; }

static bool isTry(const ControlData& control) { return control.blockType() == BlockType::Try; }

static bool isAnyCatch(const ControlData& control) { return control.blockType() == BlockType::Catch; }

static bool isCatch(const ControlData& control) { return isAnyCatch(control) && control.catchKind() == CatchKind::Catch; }

static bool isTopLevel(const ControlData& control) { return control.blockType() == BlockType::TopLevel; }

static bool isLoop(const ControlData& control) { return control.blockType() == BlockType::Loop; }

static bool isBlock(const ControlData& control) { return control.blockType() == BlockType::Block; }

void dump(PrintStream& out) const

{

switch (blockType()) {

case BlockType::If:

out.print("If: ");

break;

case BlockType::Block:

out.print("Block: ");

break;

case BlockType::Loop:

out.print("Loop: ");

break;

case BlockType::TopLevel:

out.print("TopLevel: ");

break;

case BlockType::Try:

out.print("Try: ");

break;

case BlockType::Catch:

out.print("Catch: ");

break;

}

out.print("Continuation: ", *continuation, ", Special: ");

if (special)

out.print(*special);

else

out.print("None");

CommaPrinter comma(", ", " Result Tmps: [");

for (const auto& tmp : results)

out.print(comma, tmp);

if (comma.didPrint())

out.print("]");

}

BlockType blockType() const { return controlBlockType; }

BlockSignature signature() const { return returnType; }

BasicBlock* targetBlockForBranch()

{

if (blockType() == BlockType::Loop)

return special;

return continuation;

}

void convertIfToBlock()

{

ASSERT(blockType() == BlockType::If);

controlBlockType = BlockType::Block;

special = nullptr;

}

SignatureArgCount branchTargetArity() const

{

if (blockType() == BlockType::Loop)

return returnType->argumentCount();

return returnType->returnCount();

}

Type branchTargetType(unsigned i) const

{

ASSERT(i < branchTargetArity());

if (blockType() == BlockType::Loop)

return returnType->argument(i);

return returnType->returnType(i);

}

void convertTryToCatch(unsigned tryEndCallSiteIndex, TypedTmp exception)

{

ASSERT(blockType() == BlockType::Try);

controlBlockType = BlockType::Catch;

m_catchKind = CatchKind::Catch;

m_tryEnd = tryEndCallSiteIndex;

m_exception = exception;

}

void convertTryToCatchAll(unsigned tryEndCallSiteIndex, TypedTmp exception)

{

ASSERT(blockType() == BlockType::Try);

controlBlockType = BlockType::Catch;

m_catchKind = CatchKind::CatchAll;

m_tryEnd = tryEndCallSiteIndex;

m_exception = exception;

}

unsigned tryStart() const

{

ASSERT(controlBlockType == BlockType::Try || controlBlockType == BlockType::Catch);

return m_tryStart;

}

unsigned tryEnd() const

{

ASSERT(controlBlockType == BlockType::Catch);

return m_tryEnd;

}

unsigned tryDepth() const

{

ASSERT(controlBlockType == BlockType::Try || controlBlockType == BlockType::Catch);

return m_tryCatchDepth;

}

CatchKind catchKind() const

{

ASSERT(controlBlockType == BlockType::Catch);

return m_catchKind;

}

TypedTmp exception() const

{

ASSERT(controlBlockType == BlockType::Catch);

return m_exception;

}

private:

friend class AirIRGenerator;

BlockType controlBlockType;

BasicBlock* continuation;

BasicBlock* special;

ResultList results;

BlockSignature returnType;

unsigned m_tryStart;

unsigned m_tryEnd;

unsigned m_tryCatchDepth;

CatchKind m_catchKind;

TypedTmp m_exception;

};

using ControlType = ControlData;

using ControlEntry = FunctionParser<AirIRGenerator>::ControlEntry;

using ControlStack = FunctionParser<AirIRGenerator>::ControlStack;

using Stack = FunctionParser<AirIRGenerator>::Stack;

using TypedExpression = FunctionParser<AirIRGenerator>::TypedExpression;

using ErrorType = String;

using UnexpectedResult = Unexpected<ErrorType>;

using Result = Expected<std::unique_ptr<InternalFunction>, ErrorType>;

using PartialResult = Expected<void, ErrorType>;

static_assert(std::is_same_v<ResultList, FunctionParser<AirIRGenerator>::ResultList>);

static ExpressionType emptyExpression() { return { }; };

template <typename ...Args>

NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const

{

using namespace FailureHelper; // See ADL comment in WasmParser.h.

return UnexpectedResult(makeString("WebAssembly.Module failed compiling: "_s, makeString(args)...));

}

#define WASM_COMPILE_FAIL_IF(condition, ...) do { \

if (UNLIKELY(condition)) \

return fail(__VA_ARGS__); \

} while (0)

AirIRGenerator(const ModuleInformation&, B3::Procedure&, InternalFunction*, Vector<UnlinkedWasmToWasmCall>&, MemoryMode, unsigned functionIndex, TierUpCount*, const Signature&, unsigned& osrEntryScratchBufferSize);

void finalizeEntrypoints();

PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);

PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);

ExpressionType addConstant(Type, uint64_t);

ExpressionType addConstant(BasicBlock*, Type, uint64_t);

ExpressionType addBottom(BasicBlock*, Type);

// References

PartialResult WARN_UNUSED_RETURN addRefIsNull(ExpressionType value, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addRefFunc(uint32_t index, ExpressionType& result);

// Tables

PartialResult WARN_UNUSED_RETURN addTableGet(unsigned, ExpressionType index, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addTableSet(unsigned, ExpressionType index, ExpressionType value);

PartialResult WARN_UNUSED_RETURN addTableInit(unsigned, unsigned, ExpressionType dstOffset, ExpressionType srcOffset, ExpressionType length);

PartialResult WARN_UNUSED_RETURN addElemDrop(unsigned);

PartialResult WARN_UNUSED_RETURN addTableSize(unsigned, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addTableGrow(unsigned, ExpressionType fill, ExpressionType delta, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addTableFill(unsigned, ExpressionType offset, ExpressionType fill, ExpressionType count);

PartialResult WARN_UNUSED_RETURN addTableCopy(unsigned, unsigned, ExpressionType dstOffset, ExpressionType srcOffset, ExpressionType length);

// Locals

PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);

// Globals

PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);

// Memory

PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);

PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);

PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addMemoryFill(ExpressionType dstAddress, ExpressionType targetValue, ExpressionType count);

PartialResult WARN_UNUSED_RETURN addMemoryCopy(ExpressionType dstAddress, ExpressionType srcAddress, ExpressionType count);

PartialResult WARN_UNUSED_RETURN addMemoryInit(unsigned, ExpressionType dstAddress, ExpressionType srcAddress, ExpressionType length);

PartialResult WARN_UNUSED_RETURN addDataDrop(unsigned);

// Atomics

PartialResult WARN_UNUSED_RETURN atomicLoad(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType& result, uint32_t offset);

PartialResult WARN_UNUSED_RETURN atomicStore(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType value, uint32_t offset);

PartialResult WARN_UNUSED_RETURN atomicBinaryRMW(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType value, ExpressionType& result, uint32_t offset);

PartialResult WARN_UNUSED_RETURN atomicCompareExchange(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType expected, ExpressionType value, ExpressionType& result, uint32_t offset);

PartialResult WARN_UNUSED_RETURN atomicWait(ExtAtomicOpType, ExpressionType pointer, ExpressionType value, ExpressionType timeout, ExpressionType& result, uint32_t offset);

PartialResult WARN_UNUSED_RETURN atomicNotify(ExtAtomicOpType, ExpressionType pointer, ExpressionType value, ExpressionType& result, uint32_t offset);

PartialResult WARN_UNUSED_RETURN atomicFence(ExtAtomicOpType, uint8_t flags);

// Saturated truncation.

PartialResult WARN_UNUSED_RETURN truncSaturated(Ext1OpType, ExpressionType operand, ExpressionType& result, Type returnType, Type operandType);

// Basic operators

template<OpType>

PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);

template<OpType>

PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);

// Control flow

ControlData WARN_UNUSED_RETURN addTopLevel(BlockSignature);

PartialResult WARN_UNUSED_RETURN addBlock(BlockSignature, Stack& enclosingStack, ControlType& newBlock, Stack& newStack);

PartialResult WARN_UNUSED_RETURN addLoop(BlockSignature, Stack& enclosingStack, ControlType& block, Stack& newStack, uint32_t loopIndex);

PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, BlockSignature, Stack& enclosingStack, ControlType& result, Stack& newStack);

PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const Stack&);

PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);

PartialResult WARN_UNUSED_RETURN addTry(BlockSignature, Stack& enclosingStack, ControlType& result, Stack& newStack);

PartialResult WARN_UNUSED_RETURN addCatch(unsigned exceptionIndex, const Signature&, Stack&, ControlType&, ResultList&);

PartialResult WARN_UNUSED_RETURN addCatchToUnreachable(unsigned exceptionIndex, const Signature&, ControlType&, ResultList&);

PartialResult WARN_UNUSED_RETURN addCatchAll(Stack&, ControlType&);

PartialResult WARN_UNUSED_RETURN addCatchAllToUnreachable(ControlType&);

PartialResult WARN_UNUSED_RETURN addDelegate(ControlType&, ControlType&);

PartialResult WARN_UNUSED_RETURN addDelegateToUnreachable(ControlType&, ControlType&);

PartialResult WARN_UNUSED_RETURN addThrow(unsigned exceptionIndex, Vector<ExpressionType>& args, Stack&);

PartialResult WARN_UNUSED_RETURN addRethrow(unsigned, ControlType&);

PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const Stack& returnValues);

PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const Stack& returnValues);

PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const Stack& expressionStack);

PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, Stack& expressionStack);

PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&, const Stack& expressionStack = { });

PartialResult WARN_UNUSED_RETURN endTopLevel(BlockSignature, const Stack&) { return { }; }

// Calls

PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature&, Vector<ExpressionType>& args, ResultList& results);

PartialResult WARN_UNUSED_RETURN addCallIndirect(unsigned tableIndex, const Signature&, Vector<ExpressionType>& args, ResultList& results);

PartialResult WARN_UNUSED_RETURN addCallRef(const Signature&, Vector<ExpressionType>& args, ResultList& results);

PartialResult WARN_UNUSED_RETURN addUnreachable();

PartialResult WARN_UNUSED_RETURN emitIndirectCall(TypedTmp calleeInstance, ExpressionType calleeCode, const Signature&, const Vector<ExpressionType>& args, ResultList&);

std::pair<B3::PatchpointValue*, PatchpointExceptionHandle> WARN_UNUSED_RETURN emitCallPatchpoint(BasicBlock*, const Signature&, const ResultList& results, const Vector<TypedTmp>& args, Vector<ConstrainedTmp> extraArgs = { });

PartialResult addShift(Type, B3::Air::Opcode, ExpressionType value, ExpressionType shift, ExpressionType& result);

PartialResult addIntegerSub(B3::Air::Opcode, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);

PartialResult addFloatingPointAbs(B3::Air::Opcode, ExpressionType value, ExpressionType& result);

PartialResult addFloatingPointBinOp(Type, B3::Air::Opcode, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);

void dump(const ControlStack&, const Stack* expressionStack);

void setParser(FunctionParser<AirIRGenerator>* parser) { m_parser = parser; };

void didFinishParsingLocals() { }

void didPopValueFromStack() { }

Tmp emitCatchImpl(CatchKind, ControlType&, unsigned exceptionIndex = 0);

template <size_t inlineCapacity>

PatchpointExceptionHandle preparePatchpointForExceptions(B3::PatchpointValue*, Vector<ConstrainedTmp, inlineCapacity>& args);

const Bag<B3::PatchpointValue*>& patchpoints() const

{

return m_patchpoints;

}

void addStackMap(unsigned callSiteIndex, StackMap&& stackmap)

{

m_stackmaps.add(CallSiteIndex(callSiteIndex), WTFMove(stackmap));

}

StackMaps&& takeStackmaps()

{

return WTFMove(m_stackmaps);

}

Vector<UnlinkedHandlerInfo>&& takeExceptionHandlers()

{

return WTFMove(m_exceptionHandlers);

}

private:

B3::Type toB3ResultType(BlockSignature returnType);

ALWAYS_INLINE void validateInst(Inst& inst)

{

if (ASSERT_ENABLED) {

if (!inst.isValidForm()) {

dataLogLn("Inst validation failed:");

dataLogLn(inst, "\n");

if (inst.origin)

dataLogLn(deepDump(inst.origin), "\n");

CRASH();

}

}

}

static Arg extractArg(const TypedTmp& tmp) { return tmp.tmp(); }

static Arg extractArg(const Tmp& tmp) { return Arg(tmp); }

static Arg extractArg(const Arg& arg) { return arg; }

template<typename... Arguments>

void append(BasicBlock* block, Kind kind, Arguments&&... arguments)

{

// FIXME: Find a way to use origin here.

auto& inst = block->append(kind, nullptr, extractArg(arguments)...);

validateInst(inst);

}

template<typename... Arguments>

void append(Kind kind, Arguments&&... arguments)

{

append(m_currentBlock, kind, std::forward<Arguments>(arguments)...);

}

template<typename... Arguments>

void appendEffectful(B3::Air::Opcode op, Arguments&&... arguments)

{

Kind kind = op;

kind.effects = true;

append(m_currentBlock, kind, std::forward<Arguments>(arguments)...);

}

template<typename... Arguments>

void appendEffectful(BasicBlock* block, B3::Air::Opcode op, Arguments&&... arguments)

{

Kind kind = op;

kind.effects = true;

append(block, kind, std::forward<Arguments>(arguments)...);

}

Tmp newTmp(B3::Bank bank)

{

return m_code.newTmp(bank);

}

TypedTmp g32() { return { newTmp(B3::GP), Types::I32 }; }

TypedTmp g64() { return { newTmp(B3::GP), Types::I64 }; }

TypedTmp gExternref() { return { newTmp(B3::GP), Types::Externref }; }

TypedTmp gFuncref() { return { newTmp(B3::GP), Types::Funcref }; }

TypedTmp gRef(Type type) { return { newTmp(B3::GP), type }; }

TypedTmp f32() { return { newTmp(B3::FP), Types::F32 }; }

TypedTmp f64() { return { newTmp(B3::FP), Types::F64 }; }

TypedTmp tmpForType(Type type)

{

switch (type.kind) {

case TypeKind::I32:

return g32();

case TypeKind::I64:

return g64();

case TypeKind::Funcref:

return gFuncref();

case TypeKind::Ref:

case TypeKind::RefNull:

return gRef(type);

case TypeKind::Externref:

return gExternref();

case TypeKind::F32:

return f32();

case TypeKind::F64:

return f64();

case TypeKind::Void:

return { };

default:

RELEASE_ASSERT_NOT_REACHED();

}

}

ResultList tmpsForSignature(BlockSignature signature)

{

ResultList result(signature->returnCount());

for (unsigned i = 0; i < signature->returnCount(); ++i)

result[i] = tmpForType(signature->returnType(i));

return result;

}

B3::PatchpointValue* addPatchpoint(B3::Type type)

{

auto* result = m_proc.add<B3::PatchpointValue>(type, B3::Origin());

if (UNLIKELY(shouldDumpIRAtEachPhase(B3::AirMode)))

m_patchpoints.add(result);

return result;

}

template <typename ...Args>

void emitPatchpoint(B3::PatchpointValue* patch, Tmp result, Args... theArgs)

{

emitPatchpoint(m_currentBlock, patch, result, std::forward<Args>(theArgs)...);

}

template <typename ...Args>

void emitPatchpoint(BasicBlock* basicBlock, B3::PatchpointValue* patch, Tmp result, Args... theArgs)

{

emitPatchpoint(basicBlock, patch, Vector<Tmp, 8> { result }, Vector<ConstrainedTmp, sizeof...(Args)>::from(theArgs...));

}

void emitPatchpoint(BasicBlock* basicBlock, B3::PatchpointValue* patch, Tmp result)

{

emitPatchpoint(basicBlock, patch, Vector<Tmp, 8> { result }, Vector<ConstrainedTmp>());

}

template <size_t inlineSize>

void emitPatchpoint(BasicBlock* basicBlock, B3::PatchpointValue* patch, Tmp result, Vector<ConstrainedTmp, inlineSize>&& args)

{

emitPatchpoint(basicBlock, patch, Vector<Tmp, 8> { result }, WTFMove(args));

}

template <typename ResultTmpType, size_t inlineSize>

void emitPatchpoint(BasicBlock* basicBlock, B3::PatchpointValue* patch, const Vector<ResultTmpType, 8>& results, Vector<ConstrainedTmp, inlineSize>&& args)

{

if (!m_patchpointSpecial)

m_patchpointSpecial = static_cast<B3::PatchpointSpecial*>(m_code.addSpecial(makeUnique<B3::PatchpointSpecial>()));

auto toTmp = [&] (ResultTmpType tmp) {

if constexpr (std::is_same_v<ResultTmpType, Tmp>)

return tmp;

else

return tmp.tmp();

};

Inst inst(Patch, patch, Arg::special(m_patchpointSpecial));

Vector<Inst, 1> resultMovs;

switch (patch->type().kind()) {

case B3::Void:

break;

default: {

ASSERT(results.size());

for (unsigned i = 0; i < results.size(); ++i) {

switch (patch->resultConstraints[i].kind()) {

case B3::ValueRep::StackArgument: {

Arg arg = Arg::callArg(patch->resultConstraints[i].offsetFromSP());

inst.args.append(arg);

resultMovs.append(Inst(B3::Air::moveForType(m_proc.typeAtOffset(patch->type(), i)), nullptr, arg, toTmp(results[i])));

break;

}

case B3::ValueRep::Register: {

inst.args.append(Tmp(patch->resultConstraints[i].reg()));

resultMovs.append(Inst(B3::Air::relaxedMoveForType(m_proc.typeAtOffset(patch->type(), i)), nullptr, Tmp(patch->resultConstraints[i].reg()), toTmp(results[i])));

break;

}

case B3::ValueRep::SomeRegister: {

inst.args.append(toTmp(results[i]));

break;

}

default:

RELEASE_ASSERT_NOT_REACHED();

}

}

}

}

for (unsigned i = 0; i < args.size(); ++i) {

ConstrainedTmp& tmp = args[i];

// FIXME: This is less than ideal to create dummy values just to satisfy Air's

// validation. We should abstrcat Patch enough so ValueRep's don't need to be

// backed by Values.

// https://bugs.webkit.org/show_bug.cgi?id=194040

B3::Value* dummyValue = m_proc.addConstant(B3::Origin(), tmp.tmp.isGP() ? B3::Int64 : B3::Double, 0);

patch->append(dummyValue, tmp.rep);

switch (tmp.rep.kind()) {

// B3::Value propagates (Late)ColdAny information and later Air will allocate appropriate stack.

case B3::ValueRep::ColdAny:

case B3::ValueRep::LateColdAny:

case B3::ValueRep::SomeRegister:

inst.args.append(tmp.tmp);

break;

case B3::ValueRep::Register:

patch->earlyClobbered().clear(tmp.rep.reg());

append(basicBlock, tmp.tmp.isGP() ? Move : MoveDouble, tmp.tmp, tmp.rep.reg());

inst.args.append(Tmp(tmp.rep.reg()));

break;

case B3::ValueRep::StackArgument: {

Arg arg = Arg::callArg(tmp.rep.offsetFromSP());

append(basicBlock, tmp.tmp.isGP() ? Move : MoveDouble, tmp.tmp, arg);

ASSERT(arg.canRepresent(patch->child(i)->type()));

inst.args.append(arg);

break;

}

default:

RELEASE_ASSERT_NOT_REACHED();

}

}

for (auto valueRep : patch->resultConstraints) {

if (valueRep.isReg())

patch->lateClobbered().clear(valueRep.reg());

}

for (unsigned i = patch->numGPScratchRegisters; i--;)

inst.args.append(g64().tmp());

for (unsigned i = patch->numFPScratchRegisters; i--;)

inst.args.append(f64().tmp());

validateInst(inst);

basicBlock->append(WTFMove(inst));

for (Inst result : resultMovs) {

validateInst(result);

basicBlock->append(WTFMove(result));

}

}

template <typename Branch, typename Generator>

void emitCheck(const Branch& makeBranch, const Generator& generator)

{

// We fail along the truthy edge of 'branch'.

Inst branch = makeBranch();

// FIXME: Make a hashmap of these.

B3::CheckSpecial::Key key(branch);

B3::CheckSpecial* special = static_cast<B3::CheckSpecial*>(m_code.addSpecial(makeUnique<B3::CheckSpecial>(key)));

// FIXME: Remove the need for dummy values

// https://bugs.webkit.org/show_bug.cgi?id=194040

B3::Value* dummyPredicate = m_proc.addConstant(B3::Origin(), B3::Int32, 42);

B3::CheckValue* checkValue = m_proc.add<B3::CheckValue>(B3::Check, B3::Origin(), dummyPredicate);

checkValue->setGenerator(generator);

Inst inst(Patch, checkValue, Arg::special(special));

inst.args.appendVector(branch.args);

m_currentBlock->append(WTFMove(inst));

}

template <typename Func, typename ...Args>

void emitCCall(Func func, TypedTmp result, Args... args)

{

emitCCall(m_currentBlock, func, result, std::forward<Args>(args)...);

}

template <typename Func, typename ...Args>

void emitCCall(BasicBlock* block, Func func, TypedTmp result, Args... theArgs)

{

B3::Type resultType = B3::Void;

if (result) {

switch (result.type().kind) {

case TypeKind::I32:

resultType = B3::Int32;

break;

case TypeKind::I64:

case TypeKind::Externref:

case TypeKind::Funcref:

case TypeKind::Ref:

case TypeKind::RefNull:

resultType = B3::Int64;

break;

case TypeKind::F32:

resultType = B3::Float;

break;

case TypeKind::F64:

resultType = B3::Double;

break;

default:

RELEASE_ASSERT_NOT_REACHED();

}

}

auto makeDummyValue = [&] (Tmp tmp) {

// FIXME: This is less than ideal to create dummy values just to satisfy Air's

// validation. We should abstrcat CCall enough so we're not reliant on arguments

// to the B3::CCallValue.

// https://bugs.webkit.org/show_bug.cgi?id=194040

if (tmp.isGP())

return m_proc.addConstant(B3::Origin(), B3::Int64, 0);

return m_proc.addConstant(B3::Origin(), B3::Double, 0);

};

B3::Value* dummyFunc = m_proc.addConstant(B3::Origin(), B3::Int64, bitwise_cast<uintptr_t>(func));

B3::Value* origin = m_proc.add<B3::CCallValue>(resultType, B3::Origin(), B3::Effects::none(), dummyFunc, makeDummyValue(theArgs)...);

Inst inst(CCall, origin);

Tmp callee = g64();

append(block, Move, Arg::immPtr(tagCFunctionPtr<void*, OperationPtrTag>(func)), callee);

inst.args.append(callee);

if (result)

inst.args.append(result.tmp());

for (Tmp tmp : Vector<Tmp, sizeof...(Args)>::from(theArgs.tmp()...))

inst.args.append(tmp);

block->append(WTFMove(inst));

}

static B3::Air::Opcode moveOpForValueType(Type type)

{

switch (type.kind) {

case TypeKind::I32:

return Move32;

case TypeKind::I64:

case TypeKind::Externref:

case TypeKind::Funcref:

case TypeKind::Ref:

case TypeKind::RefNull:

return Move;

case TypeKind::F32:

return MoveFloat;

case TypeKind::F64:

return MoveDouble;

default:

RELEASE_ASSERT_NOT_REACHED();

}

}

void emitLoad(B3::Air::Opcode op, B3::Type type, Tmp base, size_t offset, Tmp result)

{

if (Arg::isValidAddrForm(offset, B3::widthForType(type)))

append(op, Arg::addr(base, offset), result);

else {

auto temp2 = g64();

append(Move, Arg::bigImm(offset), temp2);

append(Add64, temp2, base, temp2);

append(op, Arg::addr(temp2), result);

}

}

void emitLoad(Tmp base, size_t offset, const TypedTmp& result)

{

emitLoad(moveOpForValueType(result.type()), toB3Type(result.type()), base, offset, result.tmp());

}

void emitThrowException(CCallHelpers&, ExceptionType);

void emitEntryTierUpCheck();

void emitLoopTierUpCheck(uint32_t loopIndex, const Vector<TypedTmp>& liveValues);

void emitWriteBarrierForJSWrapper();

ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);

ExpressionType emitLoadOp(LoadOpType, ExpressionType pointer, uint32_t offset);

void emitStoreOp(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);

ExpressionType emitAtomicLoadOp(ExtAtomicOpType, Type, ExpressionType pointer, uint32_t offset);

void emitAtomicStoreOp(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType value, uint32_t offset);

ExpressionType emitAtomicBinaryRMWOp(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType value, uint32_t offset);

ExpressionType emitAtomicCompareExchange(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType expected, ExpressionType value, uint32_t offset);

void sanitizeAtomicResult(ExtAtomicOpType, Type, Tmp source, Tmp dest);

void sanitizeAtomicResult(ExtAtomicOpType, Type, Tmp result);

TypedTmp appendGeneralAtomic(ExtAtomicOpType, B3::Air::Opcode nonAtomicOpcode, B3::Commutativity, Arg input, Arg addrArg, TypedTmp result);

TypedTmp appendStrongCAS(ExtAtomicOpType, TypedTmp expected, TypedTmp value, Arg addrArg, TypedTmp result);

void unify(const ExpressionType dst, const ExpressionType source);

void unifyValuesWithBlock(const Stack& resultStack, const ResultList& stack);

template <typename IntType>

void emitChecksForModOrDiv(bool isSignedDiv, ExpressionType left, ExpressionType right);

template <typename IntType>

void emitModOrDiv(bool isDiv, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);

enum class MinOrMax { Min, Max };

PartialResult addFloatingPointMinOrMax(Type, MinOrMax, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);

int32_t WARN_UNUSED_RETURN fixupPointerPlusOffset(ExpressionType&, uint32_t);

ExpressionType WARN_UNUSED_RETURN fixupPointerPlusOffsetForAtomicOps(ExtAtomicOpType, ExpressionType, uint32_t);

void restoreWasmContextInstance(BasicBlock*, TypedTmp);

enum class RestoreCachedStackLimit { No, Yes };

void restoreWebAssemblyGlobalState(RestoreCachedStackLimit, const MemoryInformation&, TypedTmp instance, BasicBlock*);

B3::Origin origin();

uint32_t outerLoopIndex() const

{

if (m_outerLoops.isEmpty())

return UINT32_MAX;

return m_outerLoops.last();

}

template <typename Function>

void forEachLiveValue(Function);

FunctionParser<AirIRGenerator>* m_parser { nullptr };

const ModuleInformation& m_info;

const MemoryMode m_mode { MemoryMode::BoundsChecking };

const unsigned m_functionIndex { UINT_MAX };

TierUpCount* m_tierUp { nullptr };

B3::Procedure& m_proc;

Code& m_code;

Vector<uint32_t> m_outerLoops;

BasicBlock* m_currentBlock { nullptr };

BasicBlock* m_rootBlock { nullptr };

BasicBlock* m_mainEntrypointStart { nullptr };

Vector<TypedTmp> m_locals;

Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.

GPRReg m_memoryBaseGPR { InvalidGPRReg };

GPRReg m_boundsCheckingSizeGPR { InvalidGPRReg };

GPRReg m_wasmContextInstanceGPR { InvalidGPRReg };

GPRReg m_prologueWasmContextGPR { InvalidGPRReg };

bool m_makesCalls { false };

HashMap<BlockSignature, B3::Type> m_tupleMap;

// This is only filled if we are dumping IR.

Bag<B3::PatchpointValue*> m_patchpoints;

TypedTmp m_instanceValue; // Always use the accessor below to ensure the instance value is materialized when used.

bool m_usesInstanceValue { false };

TypedTmp instanceValue()

{

m_usesInstanceValue = true;

return m_instanceValue;

}

uint32_t m_maxNumJSCallArguments { 0 };

unsigned m_numImportFunctions;

B3::PatchpointSpecial* m_patchpointSpecial { nullptr };

RefPtr<B3::Air::PrologueGenerator> m_prologueGenerator;

Vector<BasicBlock*> m_catchEntrypoints;

Checked<unsigned> m_tryCatchDepth { 0 };

Checked<unsigned> m_callSiteIndex { 0 };

StackMaps m_stackmaps;

Vector<UnlinkedHandlerInfo> m_exceptionHandlers;

Vector<std::pair<BasicBlock*, Vector<TypedTmp>>> m_loopEntryVariableData;

unsigned& m_osrEntryScratchBufferSize;

};

// Memory accesses in WebAssembly have unsigned 32-bit offsets, whereas they have signed 32-bit offsets in B3.

int32_t AirIRGenerator::fixupPointerPlusOffset(ExpressionType& ptr, uint32_t offset)

{

if (static_cast<uint64_t>(offset) > static_cast<uint64_t>(std::numeric_limits<int32_t>::max())) {

auto previousPtr = ptr;

ptr = g64();

auto constant = g64();

append(Move, Arg::bigImm(offset), constant);

append(Add64, constant, previousPtr, ptr);

return 0;

}

return offset;

}

void AirIRGenerator::restoreWasmContextInstance(BasicBlock* block, TypedTmp instance)

{

if (Context::useFastTLS()) {

auto* patchpoint = addPatchpoint(B3::Void);

if (CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister())

patchpoint->clobber(RegisterSet::macroScratchRegisters());

patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {

AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister());

jit.storeWasmContextInstance(params[0].gpr());

});

emitPatchpoint(block, patchpoint, Tmp(), instance);

return;

}

// FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.

// This prevents us from using ArgumentReg to this (logically) immutable pinned register.

auto* patchpoint = addPatchpoint(B3::Void);

B3::Effects effects = B3::Effects::none();

effects.writesPinned = true;

effects.reads = B3::HeapRange::top();

patchpoint->effects = effects;

patchpoint->clobberLate(RegisterSet(m_wasmContextInstanceGPR));

GPRReg wasmContextInstanceGPR = m_wasmContextInstanceGPR;

patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& param) {

jit.move(param[0].gpr(), wasmContextInstanceGPR);

});

emitPatchpoint(block, patchpoint, Tmp(), instance);

}

AirIRGenerator::AirIRGenerator(const ModuleInformation& info, B3::Procedure& procedure, InternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, MemoryMode mode, unsigned functionIndex, TierUpCount* tierUp, const Signature& signature, unsigned& osrEntryScratchBufferSize)

: m_info(info)

, m_mode(mode)

, m_functionIndex(functionIndex)

, m_tierUp(tierUp)

, m_proc(procedure)

, m_code(m_proc.code())

, m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)

, m_numImportFunctions(info.importFunctionCount())

, m_osrEntryScratchBufferSize(osrEntryScratchBufferSize)

{

m_currentBlock = m_code.addBlock();

m_rootBlock = m_currentBlock;

// FIXME we don't really need to pin registers here if there's no memory. It makes wasm -> wasm thunks simpler for now. https://bugs.webkit.org/show_bug.cgi?id=166623

const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();

m_memoryBaseGPR = pinnedRegs.baseMemoryPointer;

m_code.pinRegister(m_memoryBaseGPR);

m_wasmContextInstanceGPR = pinnedRegs.wasmContextInstancePointer;

if (!Context::useFastTLS())

m_code.pinRegister(m_wasmContextInstanceGPR);

if (mode != MemoryMode::Signaling) {

m_boundsCheckingSizeGPR = pinnedRegs.boundsCheckingSizeRegister;

m_code.pinRegister(m_boundsCheckingSizeGPR);

}

m_prologueWasmContextGPR = Context::useFastTLS() ? wasmCallingConvention().prologueScratchGPRs[1] : m_wasmContextInstanceGPR;

m_prologueGenerator = createSharedTask<B3::Air::PrologueGeneratorFunction>([=, this] (CCallHelpers& jit, B3::Air::Code& code) {

AllowMacroScratchRegisterUsage allowScratch(jit);

code.emitDefaultPrologue(jit);

{

GPRReg calleeGPR = wasmCallingConvention().prologueScratchGPRs[0];

auto moveLocation = jit.moveWithPatch(MacroAssembler::TrustedImmPtr(nullptr), calleeGPR);

jit.addLinkTask([compilation, moveLocation] (LinkBuffer& linkBuffer) {

compilation->calleeMoveLocations.append(linkBuffer.locationOf<WasmEntryPtrTag>(moveLocation));

});

jit.emitPutToCallFrameHeader(calleeGPR, CallFrameSlot::callee);

jit.emitPutToCallFrameHeader(nullptr, CallFrameSlot::codeBlock);

}

{

const Checked<int32_t> wasmFrameSize = m_code.frameSize();

const unsigned minimumParentCheckSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), 1024);

const unsigned extraFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), std::max<uint32_t>(

// This allows us to elide stack checks for functions that are terminal nodes in the call

// tree, (e.g they don't make any calls) and have a small enough frame size. This works by

// having any such terminal node have its parent caller include some extra size in its

// own check for it. The goal here is twofold:

// 1. Emit less code.

// 2. Try to speed things up by skipping stack checks.

minimumParentCheckSize,

// This allows us to elide stack checks in the Wasm -> Embedder call IC stub. Since these will

// spill all arguments to the stack, we ensure that a stack check here covers the

// stack that such a stub would use.

Checked<uint32_t>(m_maxNumJSCallArguments) * sizeof(Register) + jsCallingConvention().headerSizeInBytes

));

const int32_t checkSize = m_makesCalls ? (wasmFrameSize + extraFrameSize).value() : wasmFrameSize.value();

bool needUnderflowCheck = static_cast<unsigned>(checkSize) > Options::reservedZoneSize();

bool needsOverflowCheck = m_makesCalls || wasmFrameSize >= static_cast<int32_t>(minimumParentCheckSize) || needUnderflowCheck;

// This allows leaf functions to not do stack checks if their frame size is within