mozilla-central/js/src/wasm/WasmIonCompile.cpp (file symbol)

Source code

Revision control

Copy as Markdown

Other Tools

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
*
* Copyright 2015 Mozilla Foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "wasm/WasmIonCompile.h"
#include "mozilla/DebugOnly.h"
#include "mozilla/MathAlgorithms.h"
#include <algorithm>
#include "jit/ABIArgGenerator.h"
#include "jit/CodeGenerator.h"
#include "jit/CompileInfo.h"
#include "jit/Ion.h"
#include "jit/IonOptimizationLevels.h"
#include "jit/MIR.h"
#include "jit/ShuffleAnalysis.h"
#include "js/ScalarType.h" // js::Scalar::Type
#include "wasm/WasmBaselineCompile.h"
#include "wasm/WasmBuiltinModule.h"
#include "wasm/WasmBuiltins.h"
#include "wasm/WasmCodegenTypes.h"
#include "wasm/WasmGC.h"
#include "wasm/WasmGcObject.h"
#include "wasm/WasmGenerator.h"
#include "wasm/WasmOpIter.h"
#include "wasm/WasmSignalHandlers.h"
#include "wasm/WasmStubs.h"
#include "wasm/WasmValidate.h"
using namespace js;
using namespace js::jit;
using namespace js::wasm;
using mozilla::IsPowerOfTwo;
using mozilla::Maybe;
using mozilla::Nothing;
using mozilla::Some;
namespace {
using BlockVector = Vector<MBasicBlock*, 8, SystemAllocPolicy>;
using DefVector = Vector<MDefinition*, 8, SystemAllocPolicy>;
// To compile try-catch blocks, we extend the IonCompilePolicy's ControlItem
// from being just an MBasicBlock* to a Control structure collecting additional
// information.
using ControlInstructionVector =
Vector<MControlInstruction*, 8, SystemAllocPolicy>;
struct TryControl {
// Branches to bind to the try's landing pad.
ControlInstructionVector landingPadPatches;
// For `try_table`, the list of tagged catches and labels to branch to.
TryTableCatchVector catches;
// Whether this try is in the body and should catch any thrown exception.
bool inBody;
TryControl() : inBody(false) {}
// Reset the try control for when it is cached in FunctionCompiler.
void reset() {
landingPadPatches.clearAndFree();
catches.clearAndFree();
inBody = false;
}
};
using UniqueTryControl = UniquePtr<TryControl>;
using VectorUniqueTryControl = Vector<UniqueTryControl, 2, SystemAllocPolicy>;
struct Control {
MBasicBlock* block;
UniqueTryControl tryControl;
Control() : block(nullptr), tryControl(nullptr) {}
Control(Control&&) = default;
Control(const Control&) = delete;
};
// [SMDOC] WebAssembly Exception Handling in Ion
// =======================================================
//
// ## Throwing instructions
//
// Wasm exceptions can be thrown by either a throw instruction (local throw),
// or by a wasm call.
//
// ## The "catching try control"
//
// We know we are in try-code if there is a surrounding ControlItem with
// LabelKind::Try. The innermost such control is called the
// "catching try control".
//
// ## Throws without a catching try control
//
// Such throws are implemented with an instance call that triggers the exception
// unwinding runtime. The exception unwinding runtime will not return to the
// function.
//
// ## "landing pad" and "pre-pad" blocks
//
// When an exception is thrown, the unwinder will search for the nearest
// enclosing try block and redirect control flow to it. The code that executes
// before any catch blocks is called the 'landing pad'. The 'landing pad' is
// responsible to:
// 1. Consume the pending exception state from
// Instance::pendingException(Tag)
// 2. Branch to the correct catch block, or else rethrow
//
// There is one landing pad for each try block. The immediate predecessors of
// the landing pad are called 'pre-pad' blocks. There is one pre-pad block per
// throwing instruction.
//
// ## Creating pre-pad blocks
//
// There are two possible sorts of pre-pad blocks, depending on whether we
// are branching after a local throw instruction, or after a wasm call:
//
// - If we encounter a local throw, we create the exception and tag objects,
// store them to Instance::pendingException(Tag), and then jump to the
// landing pad.
//
// - If we encounter a wasm call, we construct a MWasmCallCatchable which is a
// control instruction with either a branch to a fallthrough block or
// to a pre-pad block.
//
// The pre-pad block for a wasm call is empty except for a jump to the
// landing pad. It only exists to avoid critical edges which when split would
// violate the invariants of MWasmCallCatchable. The pending exception state
// is taken care of by the unwinder.
//
// Each pre-pad ends with a pending jump to the landing pad. The pending jumps
// to the landing pad are tracked in `tryPadPatches`. These are called
// "pad patches".
//
// ## Creating the landing pad
//
// When we exit try-code, we check if tryPadPatches has captured any control
// instructions (pad patches). If not, we don't compile any catches and we mark
// the rest as dead code.
//
// If there are pre-pad blocks, we join them to create a landing pad (or just
// "pad"). The pad's last two slots are the caught exception, and the
// exception's tag object.
//
// There are three different forms of try-catch/catch_all Wasm instructions,
// which result in different form of landing pad.
//
// 1. A catchless try, so a Wasm instruction of the form "try ... end".
// - In this case, we end the pad by rethrowing the caught exception.
//
// 2. A single catch_all after a try.
// - If the first catch after a try is a catch_all, then there won't be
// any more catches, but we need the exception and its tag object, in
// case the code in a catch_all contains "rethrow" instructions.
// - The Wasm instruction "rethrow", gets the exception and tag object to
// rethrow from the last two slots of the landing pad which, due to
// validation, is the l'th surrounding ControlItem.
// - We immediately GoTo to a new block after the pad and pop both the
// exception and tag object, as we don't need them anymore in this case.
//
// 3. Otherwise, there is one or more catch code blocks following.
// - In this case, we construct the landing pad by creating a sequence
// of compare and branch blocks that compare the pending exception tag
// object to the tag object of the current tagged catch block. This is
// done incrementally as we visit each tagged catch block in the bytecode
// stream. At every step, we update the ControlItem's block to point to
// the next block to be created in the landing pad sequence. The final
// block will either be a rethrow, if there is no catch_all, or else a
// jump to a catch_all block.
struct IonCompilePolicy {
// We store SSA definitions in the value stack.
using Value = MDefinition*;
using ValueVector = DefVector;
// We store loop headers and then/else blocks in the control flow stack.
// In the case of try-catch control blocks, we collect additional information
// regarding the possible paths from throws and calls to a landing pad, as
// well as information on the landing pad's handlers (its catches).
using ControlItem = Control;
};
using IonOpIter = OpIter<IonCompilePolicy>;
class FunctionCompiler;
// CallCompileState describes a call that is being compiled.
class CallCompileState {
// A generator object that is passed each argument as it is compiled.
WasmABIArgGenerator abi_;
// Accumulates the register arguments while compiling arguments.
MWasmCallBase::Args regArgs_;
// Reserved argument for passing Instance* to builtin instance method calls.
ABIArg instanceArg_;
// The stack area in which the callee will write stack return values, or
// nullptr if no stack results.
MWasmStackResultArea* stackResultArea_ = nullptr;
// Indicates that the call is a return/tail call.
bool returnCall = false;
// Only FunctionCompiler should be directly manipulating CallCompileState.
friend class FunctionCompiler;
};
// Encapsulates the compilation of a single function in an asm.js module. The
// function compiler handles the creation and final backend compilation of the
// MIR graph.
class FunctionCompiler {
struct ControlFlowPatch {
MControlInstruction* ins;
uint32_t index;
ControlFlowPatch(MControlInstruction* ins, uint32_t index)
: ins(ins), index(index) {}
};
using ControlFlowPatchVector = Vector<ControlFlowPatch, 0, SystemAllocPolicy>;
struct PendingBlockTarget {
ControlFlowPatchVector patches;
BranchHint hint = BranchHint::Invalid;
};
using PendingBlockTargetVector =
Vector<PendingBlockTarget, 0, SystemAllocPolicy>;
const ModuleEnvironment& moduleEnv_;
IonOpIter iter_;
uint32_t functionBodyOffset_;
const FuncCompileInput& func_;
const ValTypeVector& locals_;
size_t lastReadCallSite_;
TempAllocator& alloc_;
MIRGraph& graph_;
const CompileInfo& info_;
MIRGenerator& mirGen_;
MBasicBlock* curBlock_;
uint32_t maxStackArgBytes_;
uint32_t loopDepth_;
uint32_t blockDepth_;
PendingBlockTargetVector pendingBlocks_;
// Control flow patches created by `delegate` instructions that target the
// outermost label of this function. These will be bound to a pad that will
// do a rethrow in `emitBodyDelegateThrowPad`.
ControlInstructionVector bodyDelegatePadPatches_;
// Instance pointer argument to the current function.
MWasmParameter* instancePointer_;
MWasmParameter* stackResultPointer_;
// Reference to masm.tryNotes_
wasm::TryNoteVector& tryNotes_;
// Cache of TryControl to minimize heap allocations
VectorUniqueTryControl tryControlCache_;
public:
FunctionCompiler(const ModuleEnvironment& moduleEnv, Decoder& decoder,
const FuncCompileInput& func, const ValTypeVector& locals,
MIRGenerator& mirGen, TryNoteVector& tryNotes)
: moduleEnv_(moduleEnv),
iter_(moduleEnv, decoder),
functionBodyOffset_(decoder.beginOffset()),
func_(func),
locals_(locals),
lastReadCallSite_(0),
alloc_(mirGen.alloc()),
graph_(mirGen.graph()),
info_(mirGen.outerInfo()),
mirGen_(mirGen),
curBlock_(nullptr),
maxStackArgBytes_(0),
loopDepth_(0),
blockDepth_(0),
instancePointer_(nullptr),
stackResultPointer_(nullptr),
tryNotes_(tryNotes) {}
const ModuleEnvironment& moduleEnv() const { return moduleEnv_; }
IonOpIter& iter() { return iter_; }
uint32_t relativeBytecodeOffset() {
return readBytecodeOffset() - functionBodyOffset_;
}
TempAllocator& alloc() const { return alloc_; }
// FIXME(1401675): Replace with BlockType.
uint32_t funcIndex() const { return func_.index; }
const FuncType& funcType() const {
return *moduleEnv_.funcs[func_.index].type;
}
MBasicBlock* getCurBlock() const { return curBlock_; }
BytecodeOffset bytecodeOffset() const { return iter_.bytecodeOffset(); }
BytecodeOffset bytecodeIfNotAsmJS() const {
return moduleEnv_.isAsmJS() ? BytecodeOffset() : iter_.bytecodeOffset();
}
FeatureUsage featureUsage() const { return iter_.featureUsage(); }
// Try to get a free TryControl from the cache, or allocate a new one.
[[nodiscard]] UniqueTryControl newTryControl() {
if (tryControlCache_.empty()) {
return UniqueTryControl(js_new<TryControl>());
}
UniqueTryControl tryControl = std::move(tryControlCache_.back());
tryControlCache_.popBack();
return tryControl;
}
// Release the TryControl to the cache.
void freeTryControl(UniqueTryControl&& tryControl) {
// Ensure that it's in a consistent state
tryControl->reset();
// Ignore any OOM, as we'll fail later
(void)tryControlCache_.append(std::move(tryControl));
}
[[nodiscard]] bool init() {
// Prepare the entry block for MIR generation:
const ArgTypeVector args(funcType());
if (!mirGen_.ensureBallast()) {
return false;
}
if (!newBlock(/* prev */ nullptr, &curBlock_)) {
return false;
}
for (WasmABIArgIter i(args); !i.done(); i++) {
MWasmParameter* ins = MWasmParameter::New(alloc(), *i, i.mirType());
curBlock_->add(ins);
if (args.isSyntheticStackResultPointerArg(i.index())) {
MOZ_ASSERT(stackResultPointer_ == nullptr);
stackResultPointer_ = ins;
} else {
curBlock_->initSlot(info().localSlot(args.naturalIndex(i.index())),
ins);
}
if (!mirGen_.ensureBallast()) {
return false;
}
}
// Set up a parameter that receives the hidden instance pointer argument.
instancePointer_ =
MWasmParameter::New(alloc(), ABIArg(InstanceReg), MIRType::Pointer);
curBlock_->add(instancePointer_);
if (!mirGen_.ensureBallast()) {
return false;
}
for (size_t i = args.lengthWithoutStackResults(); i < locals_.length();
i++) {
ValType slotValType = locals_[i];
#ifndef ENABLE_WASM_SIMD
if (slotValType == ValType::V128) {
return iter().fail("Ion has no SIMD support yet");
}
#endif
MDefinition* zero = constantZeroOfValType(slotValType);
curBlock_->initSlot(info().localSlot(i), zero);
if (!mirGen_.ensureBallast()) {
return false;
}
}
return true;
}
void finish() {
mirGen().initWasmMaxStackArgBytes(maxStackArgBytes_);
MOZ_ASSERT(loopDepth_ == 0);
MOZ_ASSERT(blockDepth_ == 0);
#ifdef DEBUG
for (PendingBlockTarget& targets : pendingBlocks_) {
MOZ_ASSERT(targets.patches.empty());
}
#endif
MOZ_ASSERT(inDeadCode());
MOZ_ASSERT(done(), "all bytes must be consumed");
MOZ_ASSERT(func_.callSiteLineNums.length() == lastReadCallSite_);
}
/************************* Read-only interface (after local scope setup) */
MIRGenerator& mirGen() const { return mirGen_; }
MIRGraph& mirGraph() const { return graph_; }
const CompileInfo& info() const { return info_; }
MDefinition* getLocalDef(unsigned slot) {
if (inDeadCode()) {
return nullptr;
}
return curBlock_->getSlot(info().localSlot(slot));
}
const ValTypeVector& locals() const { return locals_; }
/*********************************************************** Constants ***/
MDefinition* constantF32(float f) {
if (inDeadCode()) {
return nullptr;
}
auto* cst = MWasmFloatConstant::NewFloat32(alloc(), f);
curBlock_->add(cst);
return cst;
}
// Hide all other overloads, to guarantee no implicit argument conversion.
template <typename T>
MDefinition* constantF32(T) = delete;
MDefinition* constantF64(double d) {
if (inDeadCode()) {
return nullptr;
}
auto* cst = MWasmFloatConstant::NewDouble(alloc(), d);
curBlock_->add(cst);
return cst;
}
template <typename T>
MDefinition* constantF64(T) = delete;
MDefinition* constantI32(int32_t i) {
if (inDeadCode()) {
return nullptr;
}
MConstant* constant =
MConstant::New(alloc(), Int32Value(i), MIRType::Int32);
curBlock_->add(constant);
return constant;
}
template <typename T>
MDefinition* constantI32(T) = delete;
MDefinition* constantI64(int64_t i) {
if (inDeadCode()) {
return nullptr;
}
MConstant* constant = MConstant::NewInt64(alloc(), i);
curBlock_->add(constant);
return constant;
}
template <typename T>
MDefinition* constantI64(T) = delete;
// Produce an MConstant of the machine's target int type (Int32 or Int64).
MDefinition* constantTargetWord(intptr_t n) {
return targetIs64Bit() ? constantI64(int64_t(n)) : constantI32(int32_t(n));
}
template <typename T>
MDefinition* constantTargetWord(T) = delete;
#ifdef ENABLE_WASM_SIMD
MDefinition* constantV128(V128 v) {
if (inDeadCode()) {
return nullptr;
}
MWasmFloatConstant* constant = MWasmFloatConstant::NewSimd128(
alloc(), SimdConstant::CreateSimd128((int8_t*)v.bytes));
curBlock_->add(constant);
return constant;
}
template <typename T>
MDefinition* constantV128(T) = delete;
#endif
MDefinition* constantNullRef() {
if (inDeadCode()) {
return nullptr;
}
// MConstant has a lot of baggage so we don't use that here.
MWasmNullConstant* constant = MWasmNullConstant::New(alloc());
curBlock_->add(constant);
return constant;
}
// Produce a zero constant for the specified ValType.
MDefinition* constantZeroOfValType(ValType valType) {
switch (valType.kind()) {
case ValType::I32:
return constantI32(0);
case ValType::I64:
return constantI64(int64_t(0));
#ifdef ENABLE_WASM_SIMD
case ValType::V128:
return constantV128(V128(0));
#endif
case ValType::F32:
return constantF32(0.0f);
case ValType::F64:
return constantF64(0.0);
case ValType::Ref:
return constantNullRef();
default:
MOZ_CRASH();
}
}
/***************************** Code generation (after local scope setup) */
void fence() {
if (inDeadCode()) {
return;
}
MWasmFence* ins = MWasmFence::New(alloc());
curBlock_->add(ins);
}
template <class T>
MDefinition* unary(MDefinition* op) {
if (inDeadCode()) {
return nullptr;
}
T* ins = T::New(alloc(), op);
curBlock_->add(ins);
return ins;
}
template <class T>
MDefinition* unary(MDefinition* op, MIRType type) {
if (inDeadCode()) {
return nullptr;
}
T* ins = T::New(alloc(), op, type);
curBlock_->add(ins);
return ins;
}
template <class T>
MDefinition* binary(MDefinition* lhs, MDefinition* rhs) {
if (inDeadCode()) {
return nullptr;
}
T* ins = T::New(alloc(), lhs, rhs);
curBlock_->add(ins);
return ins;
}
template <class T>
MDefinition* binary(MDefinition* lhs, MDefinition* rhs, MIRType type) {
if (inDeadCode()) {
return nullptr;
}
T* ins = T::New(alloc(), lhs, rhs, type);
curBlock_->add(ins);
return ins;
}
template <class T>
MDefinition* binary(MDefinition* lhs, MDefinition* rhs, MIRType type,
MWasmBinaryBitwise::SubOpcode subOpc) {
if (inDeadCode()) {
return nullptr;
}
T* ins = T::New(alloc(), lhs, rhs, type, subOpc);
curBlock_->add(ins);
return ins;
}
MDefinition* ursh(MDefinition* lhs, MDefinition* rhs, MIRType type) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MUrsh::NewWasm(alloc(), lhs, rhs, type);
curBlock_->add(ins);
return ins;
}
MDefinition* add(MDefinition* lhs, MDefinition* rhs, MIRType type) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MAdd::NewWasm(alloc(), lhs, rhs, type);
curBlock_->add(ins);
return ins;
}
bool mustPreserveNaN(MIRType type) {
return IsFloatingPointType(type) && !moduleEnv().isAsmJS();
}
MDefinition* sub(MDefinition* lhs, MDefinition* rhs, MIRType type) {
if (inDeadCode()) {
return nullptr;
}
// wasm can't fold x - 0.0 because of NaN with custom payloads.
MSub* ins = MSub::NewWasm(alloc(), lhs, rhs, type, mustPreserveNaN(type));
curBlock_->add(ins);
return ins;
}
MDefinition* nearbyInt(MDefinition* input, RoundingMode roundingMode) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MNearbyInt::New(alloc(), input, input->type(), roundingMode);
curBlock_->add(ins);
return ins;
}
MDefinition* minMax(MDefinition* lhs, MDefinition* rhs, MIRType type,
bool isMax) {
if (inDeadCode()) {
return nullptr;
}
if (mustPreserveNaN(type)) {
// Convert signaling NaN to quiet NaNs.
MDefinition* zero = constantZeroOfValType(ValType::fromMIRType(type));
lhs = sub(lhs, zero, type);
rhs = sub(rhs, zero, type);
}
MMinMax* ins = MMinMax::NewWasm(alloc(), lhs, rhs, type, isMax);
curBlock_->add(ins);
return ins;
}
MDefinition* mul(MDefinition* lhs, MDefinition* rhs, MIRType type,
MMul::Mode mode) {
if (inDeadCode()) {
return nullptr;
}
// wasm can't fold x * 1.0 because of NaN with custom payloads.
auto* ins =
MMul::NewWasm(alloc(), lhs, rhs, type, mode, mustPreserveNaN(type));
curBlock_->add(ins);
return ins;
}
MDefinition* div(MDefinition* lhs, MDefinition* rhs, MIRType type,
bool unsignd) {
if (inDeadCode()) {
return nullptr;
}
bool trapOnError = !moduleEnv().isAsmJS();
if (!unsignd && type == MIRType::Int32) {
// Enforce the signedness of the operation by coercing the operands
// to signed. Otherwise, operands that "look" unsigned to Ion but
// are not unsigned to Baldr (eg, unsigned right shifts) may lead to
// the operation being executed unsigned. Applies to mod() as well.
//
// Do this for Int32 only since Int64 is not subject to the same
// issues.
//
// Note the offsets passed to MWasmBuiltinTruncateToInt32 are wrong here,
// but it doesn't matter: they're not codegen'd to calls since inputs
// already are int32.
auto* lhs2 = createTruncateToInt32(lhs);
curBlock_->add(lhs2);
lhs = lhs2;
auto* rhs2 = createTruncateToInt32(rhs);
curBlock_->add(rhs2);
rhs = rhs2;
}
// For x86 and arm we implement i64 div via c++ builtin.
// A call to c++ builtin requires instance pointer.
#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
if (type == MIRType::Int64) {
auto* ins =
MWasmBuiltinDivI64::New(alloc(), lhs, rhs, instancePointer_, unsignd,
trapOnError, bytecodeOffset());
curBlock_->add(ins);
return ins;
}
#endif
auto* ins = MDiv::New(alloc(), lhs, rhs, type, unsignd, trapOnError,
bytecodeOffset(), mustPreserveNaN(type));
curBlock_->add(ins);
return ins;
}
MInstruction* createTruncateToInt32(MDefinition* op) {
if (op->type() == MIRType::Double || op->type() == MIRType::Float32) {
return MWasmBuiltinTruncateToInt32::New(alloc(), op, instancePointer_);
}
return MTruncateToInt32::New(alloc(), op);
}
MDefinition* mod(MDefinition* lhs, MDefinition* rhs, MIRType type,
bool unsignd) {
if (inDeadCode()) {
return nullptr;
}
bool trapOnError = !moduleEnv().isAsmJS();
if (!unsignd && type == MIRType::Int32) {
// See block comment in div().
auto* lhs2 = createTruncateToInt32(lhs);
curBlock_->add(lhs2);
lhs = lhs2;
auto* rhs2 = createTruncateToInt32(rhs);
curBlock_->add(rhs2);
rhs = rhs2;
}
// For x86 and arm we implement i64 mod via c++ builtin.
// A call to c++ builtin requires instance pointer.
#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
if (type == MIRType::Int64) {
auto* ins =
MWasmBuiltinModI64::New(alloc(), lhs, rhs, instancePointer_, unsignd,
trapOnError, bytecodeOffset());
curBlock_->add(ins);
return ins;
}
#endif
// Should be handled separately because we call BuiltinThunk for this case
// and so, need to add the dependency from instancePointer.
if (type == MIRType::Double) {
auto* ins = MWasmBuiltinModD::New(alloc(), lhs, rhs, instancePointer_,
type, bytecodeOffset());
curBlock_->add(ins);
return ins;
}
auto* ins = MMod::New(alloc(), lhs, rhs, type, unsignd, trapOnError,
bytecodeOffset());
curBlock_->add(ins);
return ins;
}
MDefinition* bitnot(MDefinition* op) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MBitNot::New(alloc(), op);
curBlock_->add(ins);
return ins;
}
MDefinition* select(MDefinition* trueExpr, MDefinition* falseExpr,
MDefinition* condExpr) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MWasmSelect::New(alloc(), trueExpr, falseExpr, condExpr);
curBlock_->add(ins);
return ins;
}
MDefinition* extendI32(MDefinition* op, bool isUnsigned) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MExtendInt32ToInt64::New(alloc(), op, isUnsigned);
curBlock_->add(ins);
return ins;
}
MDefinition* signExtend(MDefinition* op, uint32_t srcSize,
uint32_t targetSize) {
if (inDeadCode()) {
return nullptr;
}
MInstruction* ins;
switch (targetSize) {
case 4: {
MSignExtendInt32::Mode mode;
switch (srcSize) {
case 1:
mode = MSignExtendInt32::Byte;
break;
case 2:
mode = MSignExtendInt32::Half;
break;
default:
MOZ_CRASH("Bad sign extension");
}
ins = MSignExtendInt32::New(alloc(), op, mode);
break;
}
case 8: {
MSignExtendInt64::Mode mode;
switch (srcSize) {
case 1:
mode = MSignExtendInt64::Byte;
break;
case 2:
mode = MSignExtendInt64::Half;
break;
case 4:
mode = MSignExtendInt64::Word;
break;
default:
MOZ_CRASH("Bad sign extension");
}
ins = MSignExtendInt64::New(alloc(), op, mode);
break;
}
default: {
MOZ_CRASH("Bad sign extension");
}
}
curBlock_->add(ins);
return ins;
}
MDefinition* convertI64ToFloatingPoint(MDefinition* op, MIRType type,
bool isUnsigned) {
if (inDeadCode()) {
return nullptr;
}
#if defined(JS_CODEGEN_ARM)
auto* ins = MBuiltinInt64ToFloatingPoint::New(
alloc(), op, instancePointer_, type, bytecodeOffset(), isUnsigned);
#else
auto* ins = MInt64ToFloatingPoint::New(alloc(), op, type, bytecodeOffset(),
isUnsigned);
#endif
curBlock_->add(ins);
return ins;
}
MDefinition* rotate(MDefinition* input, MDefinition* count, MIRType type,
bool left) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MRotate::New(alloc(), input, count, type, left);
curBlock_->add(ins);
return ins;
}
template <class T>
MDefinition* truncate(MDefinition* op, TruncFlags flags) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = T::New(alloc(), op, flags, bytecodeOffset());
curBlock_->add(ins);
return ins;
}
#if defined(JS_CODEGEN_ARM)
MDefinition* truncateWithInstance(MDefinition* op, TruncFlags flags) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MWasmBuiltinTruncateToInt64::New(alloc(), op, instancePointer_,
flags, bytecodeOffset());
curBlock_->add(ins);
return ins;
}
#endif
MDefinition* compare(MDefinition* lhs, MDefinition* rhs, JSOp op,
MCompare::CompareType type) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MCompare::NewWasm(alloc(), lhs, rhs, op, type);
curBlock_->add(ins);
return ins;
}
void assign(unsigned slot, MDefinition* def) {
if (inDeadCode()) {
return;
}
curBlock_->setSlot(info().localSlot(slot), def);
}
MDefinition* compareIsNull(MDefinition* ref, JSOp compareOp) {
MDefinition* nullVal = constantNullRef();
if (!nullVal) {
return nullptr;
}
return compare(ref, nullVal, compareOp, MCompare::Compare_WasmAnyRef);
}
[[nodiscard]] bool refAsNonNull(MDefinition* ref) {
if (inDeadCode()) {
return true;
}
auto* ins = MWasmTrapIfNull::New(
alloc(), ref, wasm::Trap::NullPointerDereference, bytecodeOffset());
curBlock_->add(ins);
return true;
}
#ifdef ENABLE_WASM_GC
[[nodiscard]] bool brOnNull(uint32_t relativeDepth, const DefVector& values,
const ResultType& type, MDefinition* condition) {
if (inDeadCode()) {
return true;
}
MBasicBlock* fallthroughBlock = nullptr;
if (!newBlock(curBlock_, &fallthroughBlock)) {
return false;
}
MDefinition* check = compareIsNull(condition, JSOp::Eq);
if (!check) {
return false;
}
MTest* test = MTest::New(alloc(), check, nullptr, fallthroughBlock);
if (!test ||
!addControlFlowPatch(test, relativeDepth, MTest::TrueBranchIndex)) {
return false;
}
if (!pushDefs(values)) {
return false;
}
curBlock_->end(test);
curBlock_ = fallthroughBlock;
return true;
}
[[nodiscard]] bool brOnNonNull(uint32_t relativeDepth,
const DefVector& values,
const ResultType& type,
MDefinition* condition) {
if (inDeadCode()) {
return true;
}
MBasicBlock* fallthroughBlock = nullptr;
if (!newBlock(curBlock_, &fallthroughBlock)) {
return false;
}
MDefinition* check = compareIsNull(condition, JSOp::Ne);
if (!check) {
return false;
}
MTest* test = MTest::New(alloc(), check, nullptr, fallthroughBlock);
if (!test ||
!addControlFlowPatch(test, relativeDepth, MTest::TrueBranchIndex)) {
return false;
}
if (!pushDefs(values)) {
return false;
}
curBlock_->end(test);
curBlock_ = fallthroughBlock;
return true;
}
#endif // ENABLE_WASM_GC
#ifdef ENABLE_WASM_GC
MDefinition* refI31(MDefinition* input) {
auto* ins = MWasmNewI31Ref::New(alloc(), input);
curBlock_->add(ins);
return ins;
}
MDefinition* i31Get(MDefinition* input, FieldWideningOp wideningOp) {
auto* ins = MWasmI31RefGet::New(alloc(), input, wideningOp);
curBlock_->add(ins);
return ins;
}
#endif // ENABLE_WASM_GC
#ifdef ENABLE_WASM_SIMD
// About Wasm SIMD as supported by Ion:
//
// The expectation is that Ion will only ever support SIMD on x86 and x64,
// since ARMv7 will cease to be a tier-1 platform soon, and MIPS64 will never
// implement SIMD.
//
// The division of the operations into MIR nodes reflects that expectation,
// and is a good fit for x86/x64. Should the expectation change we'll
// possibly want to re-architect the SIMD support to be a little more general.
//
// Most SIMD operations map directly to a single MIR node that ultimately ends
// up being expanded in the macroassembler.
//
// Some SIMD operations that do have a complete macroassembler expansion are
// open-coded into multiple MIR nodes here; in some cases that's just
// convenience, in other cases it may also allow them to benefit from Ion
// optimizations. The reason for the expansions will be documented by a
// comment.
// (v128,v128) -> v128 effect-free binary operations
MDefinition* binarySimd128(MDefinition* lhs, MDefinition* rhs,
bool commutative, SimdOp op) {
if (inDeadCode()) {
return nullptr;
}
MOZ_ASSERT(lhs->type() == MIRType::Simd128 &&
rhs->type() == MIRType::Simd128);
auto* ins = MWasmBinarySimd128::New(alloc(), lhs, rhs, commutative, op);
curBlock_->add(ins);
return ins;
}
// (v128,i32) -> v128 effect-free shift operations
MDefinition* shiftSimd128(MDefinition* lhs, MDefinition* rhs, SimdOp op) {
if (inDeadCode()) {
return nullptr;
}
MOZ_ASSERT(lhs->type() == MIRType::Simd128 &&
rhs->type() == MIRType::Int32);
int32_t maskBits;
if (MacroAssembler::MustMaskShiftCountSimd128(op, &maskBits)) {
MDefinition* mask = constantI32(maskBits);
auto* rhs2 = MBitAnd::New(alloc(), rhs, mask, MIRType::Int32);
curBlock_->add(rhs2);
rhs = rhs2;
}
auto* ins = MWasmShiftSimd128::New(alloc(), lhs, rhs, op);
curBlock_->add(ins);
return ins;
}
// (v128,scalar,imm) -> v128
MDefinition* replaceLaneSimd128(MDefinition* lhs, MDefinition* rhs,
uint32_t laneIndex, SimdOp op) {
if (inDeadCode()) {
return nullptr;
}
MOZ_ASSERT(lhs->type() == MIRType::Simd128);
auto* ins = MWasmReplaceLaneSimd128::New(alloc(), lhs, rhs, laneIndex, op);
curBlock_->add(ins);
return ins;
}
// (scalar) -> v128 effect-free unary operations
MDefinition* scalarToSimd128(MDefinition* src, SimdOp op) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MWasmScalarToSimd128::New(alloc(), src, op);
curBlock_->add(ins);
return ins;
}
// (v128) -> v128 effect-free unary operations
MDefinition* unarySimd128(MDefinition* src, SimdOp op) {
if (inDeadCode()) {
return nullptr;
}
MOZ_ASSERT(src->type() == MIRType::Simd128);
auto* ins = MWasmUnarySimd128::New(alloc(), src, op);
curBlock_->add(ins);
return ins;
}
// (v128, imm) -> scalar effect-free unary operations
MDefinition* reduceSimd128(MDefinition* src, SimdOp op, ValType outType,
uint32_t imm = 0) {
if (inDeadCode()) {
return nullptr;
}
MOZ_ASSERT(src->type() == MIRType::Simd128);
auto* ins =
MWasmReduceSimd128::New(alloc(), src, op, outType.toMIRType(), imm);
curBlock_->add(ins);
return ins;
}
// (v128, v128, v128) -> v128 effect-free operations
MDefinition* ternarySimd128(MDefinition* v0, MDefinition* v1, MDefinition* v2,
SimdOp op) {
if (inDeadCode()) {
return nullptr;
}
MOZ_ASSERT(v0->type() == MIRType::Simd128 &&
v1->type() == MIRType::Simd128 &&
v2->type() == MIRType::Simd128);
auto* ins = MWasmTernarySimd128::New(alloc(), v0, v1, v2, op);
curBlock_->add(ins);
return ins;
}
// (v128, v128, imm_v128) -> v128 effect-free operations
MDefinition* shuffleSimd128(MDefinition* v1, MDefinition* v2, V128 control) {
if (inDeadCode()) {
return nullptr;
}
MOZ_ASSERT(v1->type() == MIRType::Simd128);
MOZ_ASSERT(v2->type() == MIRType::Simd128);
auto* ins = BuildWasmShuffleSimd128(
alloc(), reinterpret_cast<int8_t*>(control.bytes), v1, v2);
curBlock_->add(ins);
return ins;
}
// Also see below for SIMD memory references
#endif // ENABLE_WASM_SIMD
/************************************************ Linear memory accesses */
// For detailed information about memory accesses, see "Linear memory
// addresses and bounds checking" in WasmMemory.cpp.
private:
// If the platform does not have a HeapReg, load the memory base from
// instance.
MDefinition* maybeLoadMemoryBase(uint32_t memoryIndex) {
#ifdef WASM_HAS_HEAPREG
if (memoryIndex == 0) {
return nullptr;
}
#endif
return memoryBase(memoryIndex);
}
public:
// A value holding the memory base, whether that's HeapReg or some other
// register.
MDefinition* memoryBase(uint32_t memoryIndex) {
AliasSet aliases = !moduleEnv_.memories[memoryIndex].canMovingGrow()
? AliasSet::None()
: AliasSet::Load(AliasSet::WasmHeapMeta);
#ifdef WASM_HAS_HEAPREG
if (memoryIndex == 0) {
MWasmHeapReg* base = MWasmHeapReg::New(alloc(), aliases);
curBlock_->add(base);
return base;
}
#endif
uint32_t offset =
memoryIndex == 0
? Instance::offsetOfMemory0Base()
: (Instance::offsetInData(
moduleEnv_.offsetOfMemoryInstanceData(memoryIndex) +
offsetof(MemoryInstanceData, base)));
MWasmLoadInstance* base = MWasmLoadInstance::New(
alloc(), instancePointer_, offset, MIRType::Pointer, aliases);
curBlock_->add(base);
return base;
}
private:
// If the bounds checking strategy requires it, load the bounds check limit
// from the instance.
MWasmLoadInstance* maybeLoadBoundsCheckLimit(uint32_t memoryIndex,
MIRType type) {
MOZ_ASSERT(type == MIRType::Int32 || type == MIRType::Int64);
if (moduleEnv_.hugeMemoryEnabled(memoryIndex)) {
return nullptr;
}
uint32_t offset =
memoryIndex == 0
? Instance::offsetOfMemory0BoundsCheckLimit()
: (Instance::offsetInData(
moduleEnv_.offsetOfMemoryInstanceData(memoryIndex) +
offsetof(MemoryInstanceData, boundsCheckLimit)));
AliasSet aliases = !moduleEnv_.memories[memoryIndex].canMovingGrow()
? AliasSet::None()
: AliasSet::Load(AliasSet::WasmHeapMeta);
auto* load = MWasmLoadInstance::New(alloc(), instancePointer_, offset, type,
aliases);
curBlock_->add(load);
return load;
}
// Return true if the access requires an alignment check. If so, sets
// *mustAdd to true if the offset must be added to the pointer before
// checking.
bool needAlignmentCheck(MemoryAccessDesc* access, MDefinition* base,
bool* mustAdd) {
MOZ_ASSERT(!*mustAdd);
// asm.js accesses are always aligned and need no checks.
if (moduleEnv_.isAsmJS() || !access->isAtomic()) {
return false;
}
// If the EA is known and aligned it will need no checks.
if (base->isConstant()) {
// We only care about the low bits, so overflow is OK, as is chopping off
// the high bits of an i64 pointer.
uint32_t ptr = 0;
if (isMem64(access->memoryIndex())) {
ptr = uint32_t(base->toConstant()->toInt64());
} else {
ptr = base->toConstant()->toInt32();
}
if (((ptr + access->offset64()) & (access->byteSize() - 1)) == 0) {
return false;
}
}
// If the offset is aligned then the EA is just the pointer, for
// the purposes of this check.
*mustAdd = (access->offset64() & (access->byteSize() - 1)) != 0;
return true;
}
// Fold a constant base into the offset and make the base 0, provided the
// offset stays below the guard limit. The reason for folding the base into
// the offset rather than vice versa is that a small offset can be ignored
// by both explicit bounds checking and bounds check elimination.
void foldConstantPointer(MemoryAccessDesc* access, MDefinition** base) {
uint32_t offsetGuardLimit = GetMaxOffsetGuardLimit(
moduleEnv_.hugeMemoryEnabled(access->memoryIndex()));
if ((*base)->isConstant()) {
uint64_t basePtr = 0;
if (isMem64(access->memoryIndex())) {
basePtr = uint64_t((*base)->toConstant()->toInt64());
} else {
basePtr = uint64_t(int64_t((*base)->toConstant()->toInt32()));
}
uint64_t offset = access->offset64();
if (offset < offsetGuardLimit && basePtr < offsetGuardLimit - offset) {
offset += uint32_t(basePtr);
access->setOffset32(uint32_t(offset));
*base = isMem64(access->memoryIndex()) ? constantI64(int64_t(0))
: constantI32(0);
}
}
}
// If the offset must be added because it is large or because the true EA must
// be checked, compute the effective address, trapping on overflow.
void maybeComputeEffectiveAddress(MemoryAccessDesc* access,
MDefinition** base, bool mustAddOffset) {
uint32_t offsetGuardLimit = GetMaxOffsetGuardLimit(
moduleEnv_.hugeMemoryEnabled(access->memoryIndex()));
if (access->offset64() >= offsetGuardLimit ||
access->offset64() > UINT32_MAX || mustAddOffset ||
!JitOptions.wasmFoldOffsets) {
*base = computeEffectiveAddress(*base, access);
}
}
MWasmLoadInstance* needBoundsCheck(uint32_t memoryIndex) {
#ifdef JS_64BIT
// For 32-bit base pointers:
//
// If the bounds check uses the full 64 bits of the bounds check limit, then
// the base pointer must be zero-extended to 64 bits before checking and
// wrapped back to 32-bits after Spectre masking. (And it's important that
// the value we end up with has flowed through the Spectre mask.)
//
// If the memory's max size is known to be smaller than 64K pages exactly,
// we can use a 32-bit check and avoid extension and wrapping.
static_assert(0x100000000 % PageSize == 0);
bool mem32LimitIs64Bits =
isMem32(memoryIndex) &&
!moduleEnv_.memories[memoryIndex].boundsCheckLimitIs32Bits() &&
MaxMemoryPages(moduleEnv_.memories[memoryIndex].indexType()) >=
Pages(0x100000000 / PageSize);
#else
// On 32-bit platforms we have no more than 2GB memory and the limit for a
// 32-bit base pointer is never a 64-bit value.
bool mem32LimitIs64Bits = false;
#endif
return maybeLoadBoundsCheckLimit(memoryIndex,
mem32LimitIs64Bits || isMem64(memoryIndex)
? MIRType::Int64
: MIRType::Int32);
}
void performBoundsCheck(uint32_t memoryIndex, MDefinition** base,
MWasmLoadInstance* boundsCheckLimit) {
// At the outset, actualBase could be the result of pretty much any integer
// operation, or it could be the load of an integer constant. If its type
// is i32, we may assume the value has a canonical representation for the
// platform, see doc block in MacroAssembler.h.
MDefinition* actualBase = *base;
// Extend an i32 index value to perform a 64-bit bounds check if the memory
// can be 4GB or larger.
bool extendAndWrapIndex =
isMem32(memoryIndex) && boundsCheckLimit->type() == MIRType::Int64;
if (extendAndWrapIndex) {
auto* extended = MWasmExtendU32Index::New(alloc(), actualBase);
curBlock_->add(extended);
actualBase = extended;
}
auto target = memoryIndex == 0 ? MWasmBoundsCheck::Memory0
: MWasmBoundsCheck::Unknown;
auto* ins = MWasmBoundsCheck::New(alloc(), actualBase, boundsCheckLimit,
bytecodeOffset(), target);
curBlock_->add(ins);
actualBase = ins;
// If we're masking, then we update *base to create a dependency chain
// through the masked index. But we will first need to wrap the index
// value if it was extended above.
if (JitOptions.spectreIndexMasking) {
if (extendAndWrapIndex) {
auto* wrapped = MWasmWrapU32Index::New(alloc(), actualBase);
curBlock_->add(wrapped);
actualBase = wrapped;
}
*base = actualBase;
}
}
// Perform all necessary checking before a wasm heap access, based on the
// attributes of the access and base pointer.
//
// For 64-bit indices on platforms that are limited to indices that fit into
// 32 bits (all 32-bit platforms and mips64), this returns a bounds-checked
// `base` that has type Int32. Lowering code depends on this and will assert
// that the base has this type. See the end of this function.
void checkOffsetAndAlignmentAndBounds(MemoryAccessDesc* access,
MDefinition** base) {
MOZ_ASSERT(!inDeadCode());
MOZ_ASSERT(!moduleEnv_.isAsmJS());
// Attempt to fold an offset into a constant base pointer so as to simplify
// the addressing expression. This may update *base.
foldConstantPointer(access, base);
// Determine whether an alignment check is needed and whether the offset
// must be checked too.
bool mustAddOffsetForAlignmentCheck = false;
bool alignmentCheck =
needAlignmentCheck(access, *base, &mustAddOffsetForAlignmentCheck);
// If bounds checking or alignment checking requires it, compute the
// effective address: add the offset into the pointer and trap on overflow.
// This may update *base.
maybeComputeEffectiveAddress(access, base, mustAddOffsetForAlignmentCheck);
// Emit the alignment check if necessary; it traps if it fails.
if (alignmentCheck) {
curBlock_->add(MWasmAlignmentCheck::New(
alloc(), *base, access->byteSize(), bytecodeOffset()));
}
// Emit the bounds check if necessary; it traps if it fails. This may
// update *base.
MWasmLoadInstance* boundsCheckLimit =
needBoundsCheck(access->memoryIndex());
if (boundsCheckLimit) {
performBoundsCheck(access->memoryIndex(), base, boundsCheckLimit);
}
#ifndef JS_64BIT
if (isMem64(access->memoryIndex())) {
// We must have had an explicit bounds check (or one was elided if it was
// proved redundant), and on 32-bit systems the index will for sure fit in
// 32 bits: the max memory is 2GB. So chop the index down to 32-bit to
// simplify the back-end.
MOZ_ASSERT((*base)->type() == MIRType::Int64);
MOZ_ASSERT(!moduleEnv_.hugeMemoryEnabled(access->memoryIndex()));
auto* chopped = MWasmWrapU32Index::New(alloc(), *base);
MOZ_ASSERT(chopped->type() == MIRType::Int32);
curBlock_->add(chopped);
*base = chopped;
}
#endif
}
bool isSmallerAccessForI64(ValType result, const MemoryAccessDesc* access) {
if (result == ValType::I64 && access->byteSize() <= 4) {
// These smaller accesses should all be zero-extending.
MOZ_ASSERT(!isSignedIntType(access->type()));
return true;
}
return false;
}
public:
bool isMem32(uint32_t memoryIndex) {
return moduleEnv_.memories[memoryIndex].indexType() == IndexType::I32;
}
bool isMem64(uint32_t memoryIndex) {
return moduleEnv_.memories[memoryIndex].indexType() == IndexType::I64;
}
bool hugeMemoryEnabled(uint32_t memoryIndex) {
return moduleEnv_.hugeMemoryEnabled(memoryIndex);
}
// Add the offset into the pointer to yield the EA; trap on overflow.
MDefinition* computeEffectiveAddress(MDefinition* base,
MemoryAccessDesc* access) {
if (inDeadCode()) {
return nullptr;
}
uint64_t offset = access->offset64();
if (offset == 0) {
return base;
}
auto* ins = MWasmAddOffset::New(alloc(), base, offset, bytecodeOffset());
curBlock_->add(ins);
access->clearOffset();
return ins;
}
MDefinition* load(MDefinition* base, MemoryAccessDesc* access,
ValType result) {
if (inDeadCode()) {
return nullptr;
}
MDefinition* memoryBase = maybeLoadMemoryBase(access->memoryIndex());
MInstruction* load = nullptr;
if (moduleEnv_.isAsmJS()) {
MOZ_ASSERT(access->offset64() == 0);
MWasmLoadInstance* boundsCheckLimit =
maybeLoadBoundsCheckLimit(access->memoryIndex(), MIRType::Int32);
load = MAsmJSLoadHeap::New(alloc(), memoryBase, base, boundsCheckLimit,
access->type());
} else {
checkOffsetAndAlignmentAndBounds(access, &base);
#ifndef JS_64BIT
MOZ_ASSERT(base->type() == MIRType::Int32);
#endif
load = MWasmLoad::New(alloc(), memoryBase, base, *access,
result.toMIRType());
}
if (!load) {
return nullptr;
}
curBlock_->add(load);
return load;
}
void store(MDefinition* base, MemoryAccessDesc* access, MDefinition* v) {
if (inDeadCode()) {
return;
}
MDefinition* memoryBase = maybeLoadMemoryBase(access->memoryIndex());
MInstruction* store = nullptr;
if (moduleEnv_.isAsmJS()) {
MOZ_ASSERT(access->offset64() == 0);
MWasmLoadInstance* boundsCheckLimit =
maybeLoadBoundsCheckLimit(access->memoryIndex(), MIRType::Int32);
store = MAsmJSStoreHeap::New(alloc(), memoryBase, base, boundsCheckLimit,
access->type(), v);
} else {
checkOffsetAndAlignmentAndBounds(access, &base);
#ifndef JS_64BIT
MOZ_ASSERT(base->type() == MIRType::Int32);
#endif
store = MWasmStore::New(alloc(), memoryBase, base, *access, v);
}
if (!store) {
return;
}
curBlock_->add(store);
}
MDefinition* atomicCompareExchangeHeap(MDefinition* base,
MemoryAccessDesc* access,
ValType result, MDefinition* oldv,
MDefinition* newv) {
if (inDeadCode()) {
return nullptr;
}
checkOffsetAndAlignmentAndBounds(access, &base);
#ifndef JS_64BIT
MOZ_ASSERT(base->type() == MIRType::Int32);
#endif
if (isSmallerAccessForI64(result, access)) {
auto* cvtOldv =
MWrapInt64ToInt32::New(alloc(), oldv, /*bottomHalf=*/true);
curBlock_->add(cvtOldv);
oldv = cvtOldv;
auto* cvtNewv =
MWrapInt64ToInt32::New(alloc(), newv, /*bottomHalf=*/true);
curBlock_->add(cvtNewv);
newv = cvtNewv;
}
MDefinition* memoryBase = maybeLoadMemoryBase(access->memoryIndex());
MInstruction* cas = MWasmCompareExchangeHeap::New(
alloc(), bytecodeOffset(), memoryBase, base, *access, oldv, newv,
instancePointer_);
if (!cas) {
return nullptr;
}
curBlock_->add(cas);
if (isSmallerAccessForI64(result, access)) {
cas = MExtendInt32ToInt64::New(alloc(), cas, true);
curBlock_->add(cas);
}
return cas;
}
MDefinition* atomicExchangeHeap(MDefinition* base, MemoryAccessDesc* access,
ValType result, MDefinition* value) {
if (inDeadCode()) {
return nullptr;
}
checkOffsetAndAlignmentAndBounds(access, &base);
#ifndef JS_64BIT
MOZ_ASSERT(base->type() == MIRType::Int32);
#endif
if (isSmallerAccessForI64(result, access)) {
auto* cvtValue =
MWrapInt64ToInt32::New(alloc(), value, /*bottomHalf=*/true);
curBlock_->add(cvtValue);
value = cvtValue;
}
MDefinition* memoryBase = maybeLoadMemoryBase(access->memoryIndex());
MInstruction* xchg =
MWasmAtomicExchangeHeap::New(alloc(), bytecodeOffset(), memoryBase,
base, *access, value, instancePointer_);
if (!xchg) {
return nullptr;
}
curBlock_->add(xchg);
if (isSmallerAccessForI64(result, access)) {
xchg = MExtendInt32ToInt64::New(alloc(), xchg, true);
curBlock_->add(xchg);
}
return xchg;
}
MDefinition* atomicBinopHeap(AtomicOp op, MDefinition* base,
MemoryAccessDesc* access, ValType result,
MDefinition* value) {
if (inDeadCode()) {
return nullptr;
}
checkOffsetAndAlignmentAndBounds(access, &base);
#ifndef JS_64BIT
MOZ_ASSERT(base->type() == MIRType::Int32);
#endif
if (isSmallerAccessForI64(result, access)) {
auto* cvtValue =
MWrapInt64ToInt32::New(alloc(), value, /*bottomHalf=*/true);
curBlock_->add(cvtValue);
value = cvtValue;
}
MDefinition* memoryBase = maybeLoadMemoryBase(access->memoryIndex());
MInstruction* binop =
MWasmAtomicBinopHeap::New(alloc(), bytecodeOffset(), op, memoryBase,
base, *access, value, instancePointer_);
if (!binop) {
return nullptr;
}
curBlock_->add(binop);
if (isSmallerAccessForI64(result, access)) {
binop = MExtendInt32ToInt64::New(alloc(), binop, true);
curBlock_->add(binop);
}
return binop;
}
#ifdef ENABLE_WASM_SIMD
MDefinition* loadSplatSimd128(Scalar::Type viewType,
const LinearMemoryAddress<MDefinition*>& addr,
wasm::SimdOp splatOp) {
if (inDeadCode()) {
return nullptr;
}
MemoryAccessDesc access(addr.memoryIndex, viewType, addr.align, addr.offset,
bytecodeIfNotAsmJS(),
hugeMemoryEnabled(addr.memoryIndex));
// Generate better code (on x86)
// If AVX2 is enabled, more broadcast operators are available.
if (viewType == Scalar::Float64
# if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
|| (js::jit::CPUInfo::IsAVX2Present() &&
(viewType == Scalar::Uint8 || viewType == Scalar::Uint16 ||
viewType == Scalar::Float32))
# endif
) {
access.setSplatSimd128Load();
return load(addr.base, &access, ValType::V128);
}
ValType resultType = ValType::I32;
if (viewType == Scalar::Float32) {
resultType = ValType::F32;
splatOp = wasm::SimdOp::F32x4Splat;
}
auto* scalar = load(addr.base, &access, resultType);
if (!inDeadCode() && !scalar) {
return nullptr;
}
return scalarToSimd128(scalar, splatOp);
}
MDefinition* loadExtendSimd128(const LinearMemoryAddress<MDefinition*>& addr,
wasm::SimdOp op) {
if (inDeadCode()) {
return nullptr;
}
// Generate better code (on x86) by loading as a double with an
// operation that sign extends directly.
MemoryAccessDesc access(addr.memoryIndex, Scalar::Float64, addr.align,
addr.offset, bytecodeIfNotAsmJS(),
hugeMemoryEnabled(addr.memoryIndex));
access.setWidenSimd128Load(op);
return load(addr.base, &access, ValType::V128);
}
MDefinition* loadZeroSimd128(Scalar::Type viewType, size_t numBytes,
const LinearMemoryAddress<MDefinition*>& addr) {
if (inDeadCode()) {
return nullptr;
}
MemoryAccessDesc access(addr.memoryIndex, viewType, addr.align, addr.offset,
bytecodeIfNotAsmJS(),
hugeMemoryEnabled(addr.memoryIndex));
access.setZeroExtendSimd128Load();
return load(addr.base, &access, ValType::V128);
}
MDefinition* loadLaneSimd128(uint32_t laneSize,
const LinearMemoryAddress<MDefinition*>& addr,
uint32_t laneIndex, MDefinition* src) {
if (inDeadCode()) {
return nullptr;
}
MemoryAccessDesc access(addr.memoryIndex, Scalar::Simd128, addr.align,
addr.offset, bytecodeIfNotAsmJS(),
hugeMemoryEnabled(addr.memoryIndex));
MDefinition* memoryBase = maybeLoadMemoryBase(access.memoryIndex());
MDefinition* base = addr.base;
MOZ_ASSERT(!moduleEnv_.isAsmJS());
checkOffsetAndAlignmentAndBounds(&access, &base);
# ifndef JS_64BIT
MOZ_ASSERT(base->type() == MIRType::Int32);
# endif
MInstruction* load = MWasmLoadLaneSimd128::New(
alloc(), memoryBase, base, access, laneSize, laneIndex, src);
if (!load) {
return nullptr;
}
curBlock_->add(load);
return load;
}
void storeLaneSimd128(uint32_t laneSize,
const LinearMemoryAddress<MDefinition*>& addr,
uint32_t laneIndex, MDefinition* src) {
if (inDeadCode()) {
return;
}
MemoryAccessDesc access(addr.memoryIndex, Scalar::Simd128, addr.align,
addr.offset, bytecodeIfNotAsmJS(),
hugeMemoryEnabled(addr.memoryIndex));
MDefinition* memoryBase = maybeLoadMemoryBase(access.memoryIndex());
MDefinition* base = addr.base;
MOZ_ASSERT(!moduleEnv_.isAsmJS());
checkOffsetAndAlignmentAndBounds(&access, &base);
# ifndef JS_64BIT
MOZ_ASSERT(base->type() == MIRType::Int32);
# endif
MInstruction* store = MWasmStoreLaneSimd128::New(
alloc(), memoryBase, base, access, laneSize, laneIndex, src);
if (!store) {
return;
}
curBlock_->add(store);
}
#endif // ENABLE_WASM_SIMD
/************************************************ Global variable accesses */
MDefinition* loadGlobalVar(unsigned instanceDataOffset, bool isConst,
bool isIndirect, MIRType type) {
if (inDeadCode()) {
return nullptr;
}
MInstruction* load;
if (isIndirect) {
// Pull a pointer to the value out of Instance::globalArea, then
// load from that pointer. Note that the pointer is immutable
// even though the value it points at may change, hence the use of
// |true| for the first node's |isConst| value, irrespective of
// the |isConst| formal parameter to this method. The latter
// applies to the denoted value as a whole.
auto* cellPtr = MWasmLoadInstanceDataField::New(
alloc(), MIRType::Pointer, instanceDataOffset,
/*isConst=*/true, instancePointer_);
curBlock_->add(cellPtr);
load = MWasmLoadGlobalCell::New(alloc(), type, cellPtr);
} else {
// Pull the value directly out of Instance::globalArea.
load = MWasmLoadInstanceDataField::New(alloc(), type, instanceDataOffset,
isConst, instancePointer_);
}
curBlock_->add(load);
return load;
}
[[nodiscard]] bool storeGlobalVar(uint32_t lineOrBytecode,
uint32_t instanceDataOffset,
bool isIndirect, MDefinition* v) {
if (inDeadCode()) {
return true;
}
if (isIndirect) {
// Pull a pointer to the value out of Instance::globalArea, then
// store through that pointer.
auto* valueAddr = MWasmLoadInstanceDataField::New(
alloc(), MIRType::Pointer, instanceDataOffset,
/*isConst=*/true, instancePointer_);
curBlock_->add(valueAddr);
// Handle a store to a ref-typed field specially
if (v->type() == MIRType::WasmAnyRef) {
// Load the previous value for the post-write barrier
auto* prevValue =
MWasmLoadGlobalCell::New(alloc(), MIRType::WasmAnyRef, valueAddr);
curBlock_->add(prevValue);
// Store the new value
auto* store =
MWasmStoreRef::New(alloc(), instancePointer_, valueAddr,
/*valueOffset=*/0, v, AliasSet::WasmGlobalCell,
WasmPreBarrierKind::Normal);
curBlock_->add(store);
// Call the post-write barrier
return postBarrierPrecise(lineOrBytecode, valueAddr, prevValue);
}
auto* store = MWasmStoreGlobalCell::New(alloc(), v, valueAddr);
curBlock_->add(store);
return true;
}
// Or else store the value directly in Instance::globalArea.
// Handle a store to a ref-typed field specially
if (v->type() == MIRType::WasmAnyRef) {
// Compute the address of the ref-typed global
auto* valueAddr = MWasmDerivedPointer::New(
alloc(), instancePointer_,
wasm::Instance::offsetInData(instanceDataOffset));
curBlock_->add(valueAddr);
// Load the previous value for the post-write barrier
auto* prevValue =
MWasmLoadGlobalCell::New(alloc(), MIRType::WasmAnyRef, valueAddr);
curBlock_->add(prevValue);
// Store the new value
auto* store =
MWasmStoreRef::New(alloc(), instancePointer_, valueAddr,
/*valueOffset=*/0, v, AliasSet::WasmInstanceData,
WasmPreBarrierKind::Normal);
curBlock_->add(store);
// Call the post-write barrier
return postBarrierPrecise(lineOrBytecode, valueAddr, prevValue);
}
auto* store = MWasmStoreInstanceDataField::New(alloc(), instanceDataOffset,
v, instancePointer_);
curBlock_->add(store);
return true;
}
MDefinition* loadTableField(uint32_t tableIndex, unsigned fieldOffset,
MIRType type) {
uint32_t instanceDataOffset = wasm::Instance::offsetInData(
moduleEnv_.offsetOfTableInstanceData(tableIndex) + fieldOffset);
auto* load =
MWasmLoadInstance::New(alloc(), instancePointer_, instanceDataOffset,
type, AliasSet::Load(AliasSet::WasmTableMeta));
curBlock_->add(load);
return load;
}
MDefinition* loadTableLength(uint32_t tableIndex) {
return loadTableField(tableIndex, offsetof(TableInstanceData, length),
MIRType::Int32);
}
MDefinition* loadTableElements(uint32_t tableIndex) {
return loadTableField(tableIndex, offsetof(TableInstanceData, elements),
MIRType::Pointer);
}
MDefinition* tableGetAnyRef(uint32_t tableIndex, MDefinition* index) {
// Load the table length and perform a bounds check with spectre index
// masking
auto* length = loadTableLength(tableIndex);
auto* check = MWasmBoundsCheck::New(
alloc(), index, length, bytecodeOffset(), MWasmBoundsCheck::Unknown);
curBlock_->add(check);
if (JitOptions.spectreIndexMasking) {
index = check;
}
// Load the table elements and load the element
auto* elements = loadTableElements(tableIndex);
auto* element = MWasmLoadTableElement::New(alloc(), elements, index);
curBlock_->add(element);
return element;
}
[[nodiscard]] bool tableSetAnyRef(uint32_t tableIndex, MDefinition* index,
MDefinition* value,
uint32_t lineOrBytecode) {
// Load the table length and perform a bounds check with spectre index
// masking
auto* length = loadTableLength(tableIndex);
auto* check = MWasmBoundsCheck::New(
alloc(), index, length, bytecodeOffset(), MWasmBoundsCheck::Unknown);
curBlock_->add(check);
if (JitOptions.spectreIndexMasking) {
index = check;
}
// Load the table elements
auto* elements = loadTableElements(tableIndex);
// Load the previous value
auto* prevValue = MWasmLoadTableElement::New(alloc(), elements, index);
curBlock_->add(prevValue);
// Compute the value's location for the post barrier
auto* loc =
MWasmDerivedIndexPointer::New(alloc(), elements, index, ScalePointer);
curBlock_->add(loc);
// Store the new value
auto* store = MWasmStoreRef::New(
alloc(), instancePointer_, loc, /*valueOffset=*/0, value,
AliasSet::WasmTableElement, WasmPreBarrierKind::Normal);
curBlock_->add(store);
// Perform the post barrier
return postBarrierPrecise(lineOrBytecode, loc, prevValue);
}
void addInterruptCheck() {
if (inDeadCode()) {
return;
}
curBlock_->add(
MWasmInterruptCheck::New(alloc(), instancePointer_, bytecodeOffset()));
}
// Perform a post-write barrier to update the generational store buffer. This
// version will remove a previous store buffer entry if it is no longer
// needed.
[[nodiscard]] bool postBarrierPrecise(uint32_t lineOrBytecode,
MDefinition* valueAddr,
MDefinition* value) {
return emitInstanceCall2(lineOrBytecode, SASigPostBarrierPrecise, valueAddr,
value);
}
// Perform a post-write barrier to update the generational store buffer. This
// version will remove a previous store buffer entry if it is no longer
// needed.
[[nodiscard]] bool postBarrierPreciseWithOffset(uint32_t lineOrBytecode,
MDefinition* valueBase,
uint32_t valueOffset,
MDefinition* value) {
MDefinition* valueOffsetDef = constantI32(int32_t(valueOffset));
if (!valueOffsetDef) {
return false;
}
return emitInstanceCall3(lineOrBytecode, SASigPostBarrierPreciseWithOffset,
valueBase, valueOffsetDef, value);
}
// Perform a post-write barrier to update the generational store buffer. This
// version is the most efficient and only requires the address to store the
// value and the new value. It does not remove a previous store buffer entry
// if it is no longer needed, you must use a precise post-write barrier for
// that.
[[nodiscard]] bool postBarrierImmediate(uint32_t lineOrBytecode,
MDefinition* object,
MDefinition* valueBase,
uint32_t valueOffset,
MDefinition* newValue) {
auto* barrier = MWasmPostWriteBarrierImmediate::New(
alloc(), instancePointer_, object, valueBase, valueOffset, newValue);
if (!barrier) {
return false;
}
curBlock_->add(barrier);
return true;
}
[[nodiscard]] bool postBarrierIndex(uint32_t lineOrBytecode,
MDefinition* object,
MDefinition* valueBase,
MDefinition* index, uint32_t scale,
MDefinition* newValue) {
auto* barrier = MWasmPostWriteBarrierIndex::New(
alloc(), instancePointer_, object, valueBase, index, scale, newValue);
if (!barrier) {
return false;
}
curBlock_->add(barrier);
return true;
}
/***************************************************************** Calls */
// The IonMonkey backend maintains a single stack offset (from the stack
// pointer to the base of the frame) by adding the total amount of spill
// space required plus the maximum stack required for argument passing.
// Since we do not use IonMonkey's MPrepareCall/MPassArg/MCall, we must
// manually accumulate, for the entire function, the maximum required stack
// space for argument passing. (This is passed to the CodeGenerator via
// MIRGenerator::maxWasmStackArgBytes.) This is just be the maximum of the
// stack space required for each individual call (as determined by the call
// ABI).
// Operations that modify a CallCompileState.
[[nodiscard]] bool passInstance(MIRType instanceType,
CallCompileState* args) {
if (inDeadCode()) {
return true;
}
// Should only pass an instance once. And it must be a non-GC pointer.
MOZ_ASSERT(args->instanceArg_ == ABIArg());
MOZ_ASSERT(instanceType == MIRType::Pointer);
args->instanceArg_ = args->abi_.next(MIRType::Pointer);
return true;
}
// Do not call this directly. Call one of the passArg() variants instead.
[[nodiscard]] bool passArgWorker(MDefinition* argDef, MIRType type,
CallCompileState* call) {
ABIArg arg = call->abi_.next(type);
switch (arg.kind()) {
#ifdef JS_CODEGEN_REGISTER_PAIR
case ABIArg::GPR_PAIR: {
auto mirLow =
MWrapInt64ToInt32::New(alloc(), argDef, /* bottomHalf = */ true);
curBlock_->add(mirLow);
auto mirHigh =
MWrapInt64ToInt32::New(alloc(), argDef, /* bottomHalf = */ false);
curBlock_->add(mirHigh);
return call->regArgs_.append(
MWasmCallBase::Arg(AnyRegister(arg.gpr64().low), mirLow)) &&
call->regArgs_.append(
MWasmCallBase::Arg(AnyRegister(arg.gpr64().high), mirHigh));
}
#endif
case ABIArg::GPR:
case ABIArg::FPU:
return call->regArgs_.append(MWasmCallBase::Arg(arg.reg(), argDef));
case ABIArg::Stack: {
auto* mir =
MWasmStackArg::New(alloc(), arg.offsetFromArgBase(), argDef);
curBlock_->add(mir);
return true;
}
case ABIArg::Uninitialized:
MOZ_ASSERT_UNREACHABLE("Uninitialized ABIArg kind");
}
MOZ_CRASH("Unknown ABIArg kind.");
}
template <typename VecT>
[[nodiscard]] bool passArgs(const DefVector& argDefs, const VecT& types,
CallCompileState* call) {
MOZ_ASSERT(argDefs.length() == types.length());
for (uint32_t i = 0; i < argDefs.length(); i++) {
MDefinition* def = argDefs[i];
ValType type = types[i];
if (!passArg(def, type, call)) {
return false;
}
}
return true;
}
[[nodiscard]] bool passArg(MDefinition* argDef, MIRType type,
CallCompileState* call) {
if (inDeadCode()) {
return true;
}
return passArgWorker(argDef, type, call);
}
[[nodiscard]] bool passArg(MDefinition* argDef, ValType type,
CallCompileState* call) {
if (inDeadCode()) {
return true;
}
return passArgWorker(argDef, type.toMIRType(), call);
}
void markReturnCall(CallCompileState* call) { call->returnCall = true; }
// If the call returns results on the stack, prepare a stack area to receive
// them, and pass the address of the stack area to the callee as an additional
// argument.
[[nodiscard]] bool passStackResultAreaCallArg(const ResultType& resultType,
CallCompileState* call) {
if (inDeadCode()) {
return true;
}
ABIResultIter iter(resultType);
while (!iter.done() && iter.cur().inRegister()) {
iter.next();
}
if (iter.done()) {
// No stack results.
return true;
}
auto* stackResultArea = MWasmStackResultArea::New(alloc());
if (!stackResultArea) {
return false;
}
if (!stackResultArea->init(alloc(), iter.remaining())) {
return false;
}
for (uint32_t base = iter.index(); !iter.done(); iter.next()) {
MWasmStackResultArea::StackResult loc(iter.cur().stackOffset(),
iter.cur().type().toMIRType());
stackResultArea->initResult(iter.index() - base, loc);
}
curBlock_->add(stackResultArea);
MDefinition* def = call->returnCall ? (MDefinition*)stackResultPointer_
: (MDefinition*)stackResultArea;
if (!passArg(def, MIRType::Pointer, call)) {
return false;
}
call->stackResultArea_ = stackResultArea;
return true;
}
[[nodiscard]] bool finishCall(CallCompileState* call) {
if (inDeadCode()) {
return true;
}
if (!call->regArgs_.append(
MWasmCallBase::Arg(AnyRegister(InstanceReg), instancePointer_))) {
return false;
}
uint32_t stackBytes = call->abi_.stackBytesConsumedSoFar();
maxStackArgBytes_ = std::max(maxStackArgBytes_, stackBytes);
return true;
}
// Wrappers for creating various kinds of calls.
[[nodiscard]] bool collectUnaryCallResult(MIRType type,
MDefinition** result) {
MInstruction* def;
switch (type) {
case MIRType::Int32:
def = MWasmRegisterResult::New(alloc(), MIRType::Int32, ReturnReg);
break;
case MIRType::Int64:
def = MWasmRegister64Result::New(alloc(), ReturnReg64);
break;
case MIRType::Float32:
def = MWasmFloatRegisterResult::New(alloc(), type, ReturnFloat32Reg);
break;
case MIRType::Double:
def = MWasmFloatRegisterResult::New(alloc(), type, ReturnDoubleReg);
break;
#ifdef ENABLE_WASM_SIMD
case MIRType::Simd128:
def = MWasmFloatRegisterResult::New(alloc(), type, ReturnSimd128Reg);
break;
#endif
case MIRType::WasmAnyRef:
def = MWasmRegisterResult::New(alloc(), MIRType::WasmAnyRef, ReturnReg);
break;
default:
MOZ_CRASH("unexpected MIRType result for builtin call");
}
if (!def) {
return false;
}
curBlock_->add(def);
*result = def;
return true;
}
[[nodiscard]] bool collectCallResults(const ResultType& type,
MWasmStackResultArea* stackResultArea,
DefVector* results) {
if (!results->reserve(type.length())) {
return false;
}
// The result iterator goes in the order in which results would be popped
// off; we want the order in which they would be pushed.
ABIResultIter iter(type);
uint32_t stackResultCount = 0;
while (!iter.done()) {
if (iter.cur().onStack()) {
stackResultCount++;
}
iter.next();
}
for (iter.switchToPrev(); !iter.done(); iter.prev()) {
if (!mirGen().ensureBallast()) {
return false;
}
const ABIResult& result = iter.cur();
MInstruction* def;
if (result.inRegister()) {
switch (result.type().kind()) {
case wasm::ValType::I32:
def =
MWasmRegisterResult::New(alloc(), MIRType::Int32, result.gpr());
break;
case wasm::ValType::I64:
def = MWasmRegister64Result::New(alloc(), result.gpr64());
break;
case wasm::ValType::F32:
def = MWasmFloatRegisterResult::New(alloc(), MIRType::Float32,
result.fpr());
break;
case wasm::ValType::F64:
def = MWasmFloatRegisterResult::New(alloc(), MIRType::Double,
result.fpr());
break;
case wasm::ValType::Ref:
def = MWasmRegisterResult::New(alloc(), MIRType::WasmAnyRef,
result.gpr());
break;
case wasm::ValType::V128:
#ifdef ENABLE_WASM_SIMD
def = MWasmFloatRegisterResult::New(alloc(), MIRType::Simd128,
result.fpr());
#else
return this->iter().fail("Ion has no SIMD support yet");
#endif
}
} else {
MOZ_ASSERT(stackResultArea);
MOZ_ASSERT(stackResultCount);
uint32_t idx = --stackResultCount;
def = MWasmStackResult::New(alloc(), stackResultArea, idx);
}
if (!def) {
return false;
}
curBlock_->add(def);
results->infallibleAppend(def);
}
MOZ_ASSERT(results->length() == type.length());
return true;
}
[[nodiscard]] bool catchableCall(const CallSiteDesc& desc,
const CalleeDesc& callee,
const MWasmCallBase::Args& args,
const ArgTypeVector& argTypes,
MDefinition* indexOrRef = nullptr) {
MWasmCallTryDesc tryDesc;
if (!beginTryCall(&tryDesc)) {
return false;
}
MInstruction* ins;
if (tryDesc.inTry) {
ins = MWasmCallCatchable::New(alloc(), desc, callee, args,
StackArgAreaSizeUnaligned(argTypes),
tryDesc, indexOrRef);
} else {
ins = MWasmCallUncatchable::New(alloc(), desc, callee, args,
StackArgAreaSizeUnaligned(argTypes),
indexOrRef);
}
if (!ins) {
return false;
}
curBlock_->add(ins);
return finishTryCall(&tryDesc);
}
[[nodiscard]] bool callDirect(const FuncType& funcType, uint32_t funcIndex,
uint32_t lineOrBytecode,
const CallCompileState& call,
DefVector* results) {
MOZ_ASSERT(!inDeadCode());
CallSiteDesc desc(lineOrBytecode, CallSiteDesc::Func);
ResultType resultType = ResultType::Vector(funcType.results());
auto callee = CalleeDesc::function(funcIndex);
ArgTypeVector args(funcType);
if (!catchableCall(desc, callee, call.regArgs_, args)) {
return false;
}
return collectCallResults(resultType, call.stackResultArea_, results);
}
[[nodiscard]] bool returnCallDirect(const FuncType& funcType,
uint32_t funcIndex,
uint32_t lineOrBytecode,
const CallCompileState& call,
DefVector* results) {
MOZ_ASSERT(!inDeadCode());
CallSiteDesc desc(lineOrBytecode, CallSiteDesc::ReturnFunc);
auto callee = CalleeDesc::function(funcIndex);
ArgTypeVector args(funcType);
auto ins = MWasmReturnCall::New(alloc(), desc, callee, call.regArgs_,
StackArgAreaSizeUnaligned(args), nullptr);
if (!ins) {
return false;
}
curBlock_->end(ins);
curBlock_ = nullptr;
return true;
}
[[nodiscard]] bool returnCallImport(unsigned globalDataOffset,
uint32_t lineOrBytecode,
const CallCompileState& call,
const FuncType& funcType,
DefVector* results) {
MOZ_ASSERT(!inDeadCode());
CallSiteDesc desc(lineOrBytecode, CallSiteDesc::Import);
auto callee = CalleeDesc::import(globalDataOffset);
ArgTypeVector args(funcType);
auto* ins = MWasmReturnCall::New(alloc(), desc, callee, call.regArgs_,
StackArgAreaSizeUnaligned(args), nullptr);
if (!ins) {
return false;
}
curBlock_->end(ins);
curBlock_ = nullptr;
return true;
}
[[nodiscard]] bool returnCallIndirect(uint32_t funcTypeIndex,
uint32_t tableIndex, MDefinition* index,
uint32_t lineOrBytecode,
const CallCompileState& call,
DefVector* results) {
MOZ_ASSERT(!inDeadCode());
const FuncType& funcType = (*moduleEnv_.types)[funcTypeIndex].funcType();
CallIndirectId callIndirectId =
CallIndirectId::forFuncType(moduleEnv_, funcTypeIndex);
CalleeDesc callee;
MOZ_ASSERT(callIndirectId.kind() != CallIndirectIdKind::AsmJS);
const TableDesc& table = moduleEnv_.tables[tableIndex];
callee =
CalleeDesc::wasmTable(moduleEnv_, table, tableIndex, callIndirectId);
CallSiteDesc desc(lineOrBytecode, CallSiteDesc::Indirect);
ArgTypeVector args(funcType);
auto* ins = MWasmReturnCall::New(alloc(), desc, callee, call.regArgs_,
StackArgAreaSizeUnaligned(args), index);
if (!ins) {
return false;
}
curBlock_->end(ins);
curBlock_ = nullptr;
return true;
}
[[nodiscard]] bool callIndirect(uint32_t funcTypeIndex, uint32_t tableIndex,
MDefinition* index, uint32_t lineOrBytecode,
const CallCompileState& call,
DefVector* results) {
MOZ_ASSERT(!inDeadCode());
const FuncType& funcType = (*moduleEnv_.types)[funcTypeIndex].funcType();
CallIndirectId callIndirectId =
CallIndirectId::forFuncType(moduleEnv_, funcTypeIndex);
CalleeDesc callee;
if (moduleEnv_.isAsmJS()) {
MOZ_ASSERT(tableIndex == 0);
MOZ_ASSERT(callIndirectId.kind() == CallIndirectIdKind::AsmJS);
uint32_t tableIndex = moduleEnv_.asmJSSigToTableIndex[funcTypeIndex];
const TableDesc& table = moduleEnv_.tables[tableIndex];
MOZ_ASSERT(IsPowerOfTwo(table.initialLength));
MDefinition* mask = constantI32(int32_t(table.initialLength - 1));
MBitAnd* maskedIndex = MBitAnd::New(alloc(), index, mask, MIRType::Int32);
curBlock_->add(maskedIndex);
index = maskedIndex;
callee = CalleeDesc::asmJSTable(moduleEnv_, tableIndex);
} else {
MOZ_ASSERT(callIndirectId.kind() != CallIndirectIdKind::AsmJS);
const TableDesc& table = moduleEnv_.tables[tableIndex];
callee =
CalleeDesc::wasmTable(moduleEnv_, table, tableIndex, callIndirectId);
}
CallSiteDesc desc(lineOrBytecode, CallSiteDesc::Indirect);
ArgTypeVector args(funcType);
ResultType resultType = ResultType::Vector(funcType.results());
if (!catchableCall(desc, callee, call.regArgs_, args, index)) {
return false;
}
return collectCallResults(resultType, call.stackResultArea_, results);
}
[[nodiscard]] bool callImport(unsigned instanceDataOffset,
uint32_t lineOrBytecode,
const CallCompileState& call,
const FuncType& funcType, DefVector* results) {
MOZ_ASSERT(!inDeadCode());
CallSiteDesc desc(lineOrBytecode, CallSiteDesc::Import);
auto callee = CalleeDesc::import(instanceDataOffset);
ArgTypeVector args(funcType);
ResultType resultType = ResultType::Vector(funcType.results());
if (!catchableCall(desc, callee, call.regArgs_, args)) {
return false;
}
return collectCallResults(resultType, call.stackResultArea_, results);
}
[[nodiscard]] bool builtinCall(const SymbolicAddressSignature& builtin,
uint32_t lineOrBytecode,
const CallCompileState& call,
MDefinition** def) {
if (inDeadCode()) {
*def = nullptr;
return true;
}
MOZ_ASSERT(builtin.failureMode == FailureMode::Infallible);
CallSiteDesc desc(lineOrBytecode, CallSiteDesc::Symbolic);
auto callee = CalleeDesc::builtin(builtin.identity);
auto* ins = MWasmCallUncatchable::New(alloc(), desc, callee, call.regArgs_,
StackArgAreaSizeUnaligned(builtin));
if (!ins) {
return false;
}
curBlock_->add(ins);
return collectUnaryCallResult(builtin.retType, def);
}
[[nodiscard]] bool builtinInstanceMethodCall(
const SymbolicAddressSignature& builtin, uint32_t lineOrBytecode,
const CallCompileState& call, MDefinition** def = nullptr) {
MOZ_ASSERT_IF(!def, builtin.retType == MIRType::None);
if (inDeadCode()) {
if (def) {
*def = nullptr;
}
return true;
}
CallSiteDesc desc(lineOrBytecode, CallSiteDesc::Symbolic);
MWasmCallTryDesc tryDesc;
if (!beginTryCall(&tryDesc)) {
return false;
}
MInstruction* ins;
if (tryDesc.inTry) {
ins = MWasmCallCatchable::NewBuiltinInstanceMethodCall(
alloc(), desc, builtin.identity, builtin.failureMode,
call.instanceArg_, call.regArgs_, StackArgAreaSizeUnaligned(builtin),
tryDesc);
} else {
ins = MWasmCallUncatchable::NewBuiltinInstanceMethodCall(
alloc(), desc, builtin.identity, builtin.failureMode,
call.instanceArg_, call.regArgs_, StackArgAreaSizeUnaligned(builtin));
}
if (!ins) {
return false;
}
curBlock_->add(ins);
if (!finishTryCall(&tryDesc)) {
return false;
}
if (!def) {
return true;
}
return collectUnaryCallResult(builtin.retType, def);
}
#ifdef ENABLE_WASM_GC
[[nodiscard]] bool callRef(const FuncType& funcType, MDefinition* ref,
uint32_t lineOrBytecode,
const CallCompileState& call, DefVector* results) {
MOZ_ASSERT(!inDeadCode());
CalleeDesc callee = CalleeDesc::wasmFuncRef();
CallSiteDesc desc(lineOrBytecode, CallSiteDesc::FuncRef);
ArgTypeVector args(funcType);
ResultType resultType = ResultType::Vector(funcType.results());
if (!catchableCall(desc, callee, call.regArgs_, args, ref)) {
return false;
}
return collectCallResults(resultType, call.stackResultArea_, results);
}
# ifdef ENABLE_WASM_TAIL_CALLS
[[nodiscard]] bool returnCallRef(const FuncType& funcType, MDefinition* ref,
uint32_t lineOrBytecode,
const CallCompileState& call,
DefVector* results) {
MOZ_ASSERT(!inDeadCode());
CalleeDesc callee = CalleeDesc::wasmFuncRef();
CallSiteDesc desc(lineOrBytecode, CallSiteDesc::FuncRef);
ArgTypeVector args(funcType);
auto* ins = MWasmReturnCall::New(alloc(), desc, callee, call.regArgs_,
StackArgAreaSizeUnaligned(args), ref);
if (!ins) {
return false;
}
curBlock_->end(ins);
curBlock_ = nullptr;
return true;
}
# endif // ENABLE_WASM_TAIL_CALLS
#endif // ENABLE_WASM_GC
/*********************************************** Control flow generation */
inline bool inDeadCode() const { return curBlock_ == nullptr; }
[[nodiscard]] bool returnValues(const DefVector& values) {
if (inDeadCode()) {
return true;
}
if (values.empty()) {
curBlock_->end(MWasmReturnVoid::New(alloc(), instancePointer_));
} else {
ResultType resultType = ResultType::Vector(funcType().results());
ABIResultIter iter(resultType);
// Switch to iterate in FIFO order instead of the default LIFO.
while (!iter.done()) {
iter.next();
}
iter.switchToPrev();
for (uint32_t i = 0; !iter.done(); iter.prev(), i++) {
if (!mirGen().ensureBallast()) {
return false;
}
const ABIResult& result = iter.cur();
if (result.onStack()) {
MOZ_ASSERT(iter.remaining() > 1);
if (result.type().isRefRepr()) {
auto* store = MWasmStoreRef::New(
alloc(), instancePointer_, stackResultPointer_,
result.stackOffset(), values[i], AliasSet::WasmStackResult,
WasmPreBarrierKind::None);
curBlock_->add(store);
} else {
auto* store = MWasmStoreStackResult::New(
alloc(), stackResultPointer_, result.stackOffset(), values[i]);
curBlock_->add(store);
}
} else {
MOZ_ASSERT(iter.remaining() == 1);
MOZ_ASSERT(i + 1 == values.length());
curBlock_->end(
MWasmReturn::New(alloc(), values[i], instancePointer_));
}
}
}
curBlock_ = nullptr;
return true;
}
void unreachableTrap() {
if (inDeadCode()) {
return;
}
auto* ins =
MWasmTrap::New(alloc(), wasm::Trap::Unreachable, bytecodeOffset());
curBlock_->end(ins);
curBlock_ = nullptr;
}
private:
static uint32_t numPushed(MBasicBlock* block) {
return block->stackDepth() - block->info().firstStackSlot();
}
public:
[[nodiscard]] bool pushDefs(const DefVector& defs) {
if (inDeadCode()) {
return true;
}
MOZ_ASSERT(numPushed(curBlock_) == 0);
if (!curBlock_->ensureHasSlots(defs.length())) {
return false;
}
for (MDefinition* def : defs) {
MOZ_ASSERT(def->type() != MIRType::None);
curBlock_->push(def);
}
return true;
}
[[nodiscard]] bool popPushedDefs(DefVector* defs) {
size_t n = numPushed(curBlock_);
if (!defs->resizeUninitialized(n)) {
return false;
}
for (; n > 0; n--) {
MDefinition* def = curBlock_->pop();
MOZ_ASSERT(def->type() != MIRType::Value);
(*defs)[n - 1] = def;
}
return true;
}
private:
[[nodiscard]] bool addJoinPredecessor(const DefVector& defs,
MBasicBlock** joinPred) {
*joinPred = curBlock_;
if (inDeadCode()) {
return true;
}
return pushDefs(defs);
}
public:
[[nodiscard]] bool branchAndStartThen(MDefinition* cond,
MBasicBlock** elseBlock) {
if (inDeadCode()) {
*elseBlock = nullptr;
} else {
MBasicBlock* thenBlock;
if (!newBlock(curBlock_, &thenBlock)) {
return false;
}
if (!newBlock(curBlock_, elseBlock)) {
return false;
}
curBlock_->end(MTest::New(alloc(), cond, thenBlock, *elseBlock));
curBlock_ = thenBlock;
mirGraph().moveBlockToEnd(curBlock_);
}
return startBlock();
}
[[nodiscard]] bool switchToElse(MBasicBlock* elseBlock,
MBasicBlock** thenJoinPred) {
DefVector values;
if (!finishBlock(&values)) {
return false;
}
if (!elseBlock) {
*thenJoinPred = nullptr;
} else {
if (!addJoinPredecessor(values, thenJoinPred)) {
return false;
}
curBlock_ = elseBlock;
mirGraph().moveBlockToEnd(curBlock_);
}
return startBlock();
}
[[nodiscard]] bool joinIfElse(MBasicBlock* thenJoinPred, DefVector* defs) {
DefVector values;
if (!finishBlock(&values)) {
return false;
}
if (!thenJoinPred && inDeadCode()) {
return true;
}
MBasicBlock* elseJoinPred;
if (!addJoinPredecessor(values, &elseJoinPred)) {
return false;
}
mozilla::Array<MBasicBlock*, 2> blocks;
size_t numJoinPreds = 0;
if (thenJoinPred) {
blocks[numJoinPreds++] = thenJoinPred;
}
if (elseJoinPred) {
blocks[numJoinPreds++] = elseJoinPred;
}
if (numJoinPreds == 0) {
return true;
}
MBasicBlock* join;
if (!goToNewBlock(blocks[0], &join)) {
return false;
}
for (size_t i = 1; i < numJoinPreds; ++i) {
if (!goToExistingBlock(blocks[i], join)) {
return false;
}
}
curBlock_ = join;
return popPushedDefs(defs);
}
[[nodiscard]] bool startBlock() {
MOZ_ASSERT_IF(blockDepth_ < pendingBlocks_.length(),
pendingBlocks_[blockDepth_].patches.empty());
blockDepth_++;
return true;
}
[[nodiscard]] bool finishBlock(DefVector* defs) {
MOZ_ASSERT(blockDepth_);
uint32_t topLabel = --blockDepth_;
return bindBranches(topLabel, defs);
}
[[nodiscard]] bool startLoop(MBasicBlock** loopHeader, size_t paramCount) {
*loopHeader = nullptr;
blockDepth_++;
loopDepth_++;
if (inDeadCode()) {
return true;
}
// Create the loop header.
MOZ_ASSERT(curBlock_->loopDepth() == loopDepth_ - 1);
*loopHeader = MBasicBlock::New(mirGraph(), info(), curBlock_,
MBasicBlock::PENDING_LOOP_HEADER);
if (!*loopHeader) {
return false;
}
(*loopHeader)->setLoopDepth(loopDepth_);
mirGraph().addBlock(*loopHeader);
curBlock_->end(MGoto::New(alloc(), *loopHeader));
DefVector loopParams;
if (!iter().getResults(paramCount, &loopParams)) {
return false;
}
for (size_t i = 0; i < paramCount; i++) {
MPhi* phi = MPhi::New(alloc(), loopParams[i]->type());
if (!phi) {
return false;
}
if (!phi->reserveLength(2)) {
return false;
}
(*loopHeader)->addPhi(phi);
phi->addInput(loopParams[i]);
loopParams[i] = phi;
}
iter().setResults(paramCount, loopParams);
MBasicBlock* body;
if (!goToNewBlock(*loopHeader, &body)) {
return false;
}
curBlock_ = body;
return true;
}
private:
void fixupRedundantPhis(MBasicBlock* b) {
for (size_t i = 0, depth = b->stackDepth(); i < depth; i++) {
MDefinition* def = b->getSlot(i);
if (def->isUnused()) {
b->setSlot(i, def->toPhi()->getOperand(0));
}
}
}
[[nodiscard]] bool setLoopBackedge(MBasicBlock* loopEntry,
MBasicBlock* loopBody,
MBasicBlock* backedge, size_t paramCount) {
if (!loopEntry->setBackedgeWasm(backedge, paramCount)) {
return false;
}
// Flag all redundant phis as unused.
for (MPhiIterator phi = loopEntry->phisBegin(); phi != loopEntry->phisEnd();
phi++) {
MOZ_ASSERT(phi->numOperands() == 2);
if (phi->getOperand(0) == phi->getOperand(1)) {
phi->setUnused();
}
}
// Fix up phis stored in the slots Vector of pending blocks.
for (PendingBlockTarget& pendingBlockTarget : pendingBlocks_) {
for (ControlFlowPatch& p : pendingBlockTarget.patches) {
MBasicBlock* block = p.ins->block();
if (block->loopDepth() >= loopEntry->loopDepth()) {
fixupRedundantPhis(block);
}
}
}
// The loop body, if any, might be referencing recycled phis too.
if (loopBody) {
fixupRedundantPhis(loopBody);
}
// Pending jumps to an enclosing try-catch may reference the recycled phis.
// We have to search above all enclosing try blocks, as a delegate may move
// patches around.
for (uint32_t depth = 0; depth < iter().controlStackDepth(); depth++) {
LabelKind kind = iter().controlKind(depth);
if (kind != LabelKind::Try && kind != LabelKind::TryTable &&
kind != LabelKind::Body) {
continue;
}
Control& control = iter().controlItem(depth);
if (!control.tryControl) {
continue;
}
for (MControlInstruction* patch : control.tryControl->landingPadPatches) {
MBasicBlock* block = patch->block();
if (block->loopDepth() >= loopEntry->loopDepth()) {
fixupRedundantPhis(block);
}
}
}
for (MControlInstruction* patch : bodyDelegatePadPatches_) {
MBasicBlock* block = patch->block();
if (block->loopDepth() >= loopEntry->loopDepth()) {
fixupRedundantPhis(block);
}
}
// Discard redundant phis and add to the free list.
for (MPhiIterator phi = loopEntry->phisBegin();
phi != loopEntry->phisEnd();) {
MPhi* entryDef = *phi++;
if (!entryDef->isUnused()) {
continue;
}
entryDef->justReplaceAllUsesWith(entryDef->getOperand(0));
loopEntry->discardPhi(entryDef);
mirGraph().addPhiToFreeList(entryDef);
}
return true;
}
public:
[[nodiscard]] bool closeLoop(MBasicBlock* loopHeader,
DefVector* loopResults) {
MOZ_ASSERT(blockDepth_ >= 1);
MOZ_ASSERT(loopDepth_);
uint32_t headerLabel = blockDepth_ - 1;
if (!loopHeader) {
MOZ_ASSERT(inDeadCode());
MOZ_ASSERT(headerLabel >= pendingBlocks_.length() ||
pendingBlocks_[headerLabel].patches.empty());
blockDepth_--;
loopDepth_--;
return true;
}
// Op::Loop doesn't have an implicit backedge so temporarily set
// aside the end of the loop body to bind backedges.
MBasicBlock* loopBody = curBlock_;
curBlock_ = nullptr;
// As explained in bug 1253544, Ion apparently has an invariant that
// there is only one backedge to loop headers. To handle wasm's ability
// to have multiple backedges to the same loop header, we bind all those
// branches as forward jumps to a single backward jump. This is
// unfortunate but the optimizer is able to fold these into single jumps
// to backedges.
DefVector backedgeValues;
if (!bindBranches(headerLabel, &backedgeValues)) {
return false;
}
MOZ_ASSERT(loopHeader->loopDepth() == loopDepth_);
if (curBlock_) {
// We're on the loop backedge block, created by bindBranches.
for (size_t i = 0, n = numPushed(curBlock_); i != n; i++) {
curBlock_->pop();
}
if (!pushDefs(backedgeValues)) {
return false;
}
MOZ_ASSERT(curBlock_->loopDepth() == loopDepth_);
curBlock_->end(MGoto::New(alloc(), loopHeader));
if (!setLoopBackedge(loopHeader, loopBody, curBlock_,
backedgeValues.length())) {
return false;
}
}
curBlock_ = loopBody;
loopDepth_--;
// If the loop depth still at the inner loop body, correct it.
if (curBlock_ && curBlock_->loopDepth() != loopDepth_) {
MBasicBlock* out;
if (!goToNewBlock(curBlock_, &out)) {
return false;
}
curBlock_ = out;
}
blockDepth_ -= 1;
return inDeadCode() || popPushedDefs(loopResults);
}
[[nodiscard]] bool addControlFlowPatch(
MControlInstruction* ins, uint32_t relative, uint32_t index,
BranchHint branchHint = BranchHint::Invalid) {
MOZ_ASSERT(relative < blockDepth_);
uint32_t absolute = blockDepth_ - 1 - relative;
if (absolute >= pendingBlocks_.length() &&
!pendingBlocks_.resize(absolute + 1)) {
return false;
}
pendingBlocks_[absolute].hint = branchHint;
return pendingBlocks_[absolute].patches.append(
ControlFlowPatch(ins, index));
}
[[nodiscard]] bool br(uint32_t relativeDepth, const DefVector& values) {
if (inDeadCode()) {
return true;
}
MGoto* jump = MGoto::New(alloc());
if (!addControlFlowPatch(jump, relativeDepth, MGoto::TargetIndex)) {
return false;
}
if (!pushDefs(values)) {
return false;
}
curBlock_->end(jump);
curBlock_ = nullptr;
return true;
}
[[nodiscard]] bool brIf(uint32_t relativeDepth, const DefVector& values,
MDefinition* condition, BranchHint branchHint) {
if (inDeadCode()) {
return true;
}
MBasicBlock* joinBlock = nullptr;
if (!newBlock(curBlock_, &joinBlock)) {
return false;
}
MTest* test = MTest::New(alloc(), condition, nullptr, joinBlock);
if (!addControlFlowPatch(test, relativeDepth, MTest::TrueBranchIndex,
branchHint)) {
return false;
}
if (!pushDefs(values)) {
return false;
}
curBlock_->end(test);
curBlock_ = joinBlock;
return true;
}
[[nodiscard]] bool brTable(MDefinition* operand, uint32_t defaultDepth,
const Uint32Vector& depths,
const DefVector& values) {
if (inDeadCode()) {
return true;
}
size_t numCases = depths.length();
MOZ_ASSERT(numCases <= INT32_MAX);
MOZ_ASSERT(numCases);
MTableSwitch* table =
MTableSwitch::New(alloc(), operand, 0, int32_t(numCases - 1));
size_t defaultIndex;
if (!table->addDefault(nullptr, &defaultIndex)) {
return false;
}
if (!addControlFlowPatch(table, defaultDepth, defaultIndex)) {
return false;
}
using IndexToCaseMap =
HashMap<uint32_t, uint32_t, DefaultHasher<uint32_t>, SystemAllocPolicy>;
IndexToCaseMap indexToCase;
if (!indexToCase.put(defaultDepth, defaultIndex)) {
return false;
}
for (size_t i = 0; i < numCases; i++) {
if (!mirGen_.ensureBallast()) {
return false;
}
uint32_t depth = depths[i];
size_t caseIndex;
IndexToCaseMap::AddPtr p = indexToCase.lookupForAdd(depth);
if (!p) {
if (!table->addSuccessor(nullptr, &caseIndex)) {
return false;
}
if (!addControlFlowPatch(table, depth, caseIndex)) {
return false;
}
if (!indexToCase.add(p, depth, caseIndex)) {
return false;
}
} else {
caseIndex = p->value();
}
if (!table->addCase(caseIndex)) {
return false;
}
}
if (!pushDefs(values)) {
return false;
}
curBlock_->end(table);
curBlock_ = nullptr;
return true;
}
/********************************************************** Exceptions ***/
bool inTryBlockFrom(uint32_t fromRelativeDepth, uint32_t* relativeDepth) {
return iter().controlFindInnermostFrom(
[](LabelKind kind, const Control& control) {
return control.tryControl != nullptr && control.tryControl->inBody;
},
fromRelativeDepth, relativeDepth);
}
bool inTryBlock(uint32_t* relativeDepth) {
return inTryBlockFrom(0, relativeDepth);
}
bool inTryCode() {
uint32_t relativeDepth;
return inTryBlock(&relativeDepth);
}
MDefinition* loadTag(uint32_t tagIndex) {
MWasmLoadInstanceDataField* tag = MWasmLoadInstanceDataField::New(
alloc(), MIRType::WasmAnyRef,
moduleEnv_.offsetOfTagInstanceData(tagIndex), true, instancePointer_);
curBlock_->add(tag);
return tag;
}
void loadPendingExceptionState(MInstruction** exception, MInstruction** tag) {
*exception = MWasmLoadInstance::New(
alloc(), instancePointer_, wasm::Instance::offsetOfPendingException(),
MIRType::WasmAnyRef, AliasSet::Load(AliasSet::WasmPendingException));
curBlock_->add(*exception);
*tag = MWasmLoadInstance::New(
alloc(), instancePointer_,
wasm::Instance::offsetOfPendingExceptionTag(), MIRType::WasmAnyRef,
AliasSet::Load(AliasSet::WasmPendingException));
curBlock_->add(*tag);
}
[[nodiscard]] bool setPendingExceptionState(MDefinition* exception,
MDefinition* tag) {
// Set the pending exception object
auto* exceptionAddr = MWasmDerivedPointer::New(
alloc(), instancePointer_, Instance::offsetOfPendingException());
curBlock_->add(exceptionAddr);
auto* setException = MWasmStoreRef::New(
alloc(), instancePointer_, exceptionAddr, /*valueOffset=*/0, exception,
AliasSet::WasmPendingException, WasmPreBarrierKind::Normal);
curBlock_->add(setException);
if (!postBarrierPrecise(/*lineOrBytecode=*/0, exceptionAddr, exception)) {
return false;
}
// Set the pending exception tag object
auto* exceptionTagAddr = MWasmDerivedPointer::New(
alloc(), instancePointer_, Instance::offsetOfPendingExceptionTag());
curBlock_->add(exceptionTagAddr);
auto* setExceptionTag = MWasmStoreRef::New(
alloc(), instancePointer_, exceptionTagAddr, /*valueOffset=*/0, tag,
AliasSet::WasmPendingException, WasmPreBarrierKind::Normal);
curBlock_->add(setExceptionTag);
return postBarrierPrecise(/*lineOrBytecode=*/0, exceptionTagAddr, tag);
}
[[nodiscard]] bool addPadPatch(MControlInstruction* ins,
size_t relativeTryDepth) {
Control& control = iter().controlItem(relativeTryDepth);
return control.tryControl->landingPadPatches.emplaceBack(ins);
}
[[nodiscard]] bool endWithPadPatch(uint32_t relativeTryDepth) {
MGoto* jumpToLandingPad = MGoto::New(alloc());
curBlock_->end(jumpToLandingPad);
return addPadPatch(jumpToLandingPad, relativeTryDepth);
}
[[nodiscard]] bool delegatePadPatches(const ControlInstructionVector& patches,
uint32_t relativeDepth) {
if (patches.empty()) {
return true;
}
// Find where we are delegating the pad patches to.
ControlInstructionVector* targetPatches;
uint32_t targetRelativeDepth;
if (inTryBlockFrom(relativeDepth, &targetRelativeDepth)) {
targetPatches = &iter()
.controlItem(targetRelativeDepth)
.tryControl->landingPadPatches;
} else {
MOZ_ASSERT(relativeDepth <= blockDepth_ - 1);
targetPatches = &bodyDelegatePadPatches_;
}
// Append the delegate's pad patches to the target's.
for (MControlInstruction* ins : patches) {
if (!targetPatches->emplaceBack(ins)) {
return false;
}
}
return true;
}
[[nodiscard]] bool beginTryCall(MWasmCallTryDesc* call) {
call->inTry = inTryBlock(&call->relativeTryDepth);
if (!call->inTry) {
return true;
}
// Allocate a try note
if (!tryNotes_.append(wasm::TryNote())) {
return false;
}
call->tryNoteIndex = tryNotes_.length() - 1;
// Allocate blocks for fallthrough and exceptions
return newBlock(curBlock_, &call->fallthroughBlock) &&
newBlock(curBlock_, &call->prePadBlock);
}
[[nodiscard]] bool finishTryCall(MWasmCallTryDesc* call) {
if (!call->inTry) {
return true;
}
// Switch to the prePadBlock
MBasicBlock* callBlock = curBlock_;
curBlock_ = call->prePadBlock;
// Mark this as the landing pad for the call
curBlock_->add(
MWasmCallLandingPrePad::New(alloc(), callBlock, call->tryNoteIndex));
// End with a pending jump to the landing pad
if (!endWithPadPatch(call->relativeTryDepth)) {
return false;
}
// Compilation continues in the fallthroughBlock.
curBlock_ = call->fallthroughBlock;
return true;
}
// Create a landing pad for a try block if there are any throwing
// instructions. This is also used for the implicit rethrow landing pad used
// for delegate instructions that target the outermost label.
[[nodiscard]] bool createTryLandingPadIfNeeded(
ControlInstructionVector& landingPadPatches, MBasicBlock** landingPad) {
// If there are no pad-patches for this try control, it means there are no
// instructions in the try code that could throw an exception. In this
// case, all the catches are dead code, and the try code ends up equivalent
// to a plain wasm block.
if (landingPadPatches.empty()) {
*landingPad = nullptr;
return true;
}
// Otherwise, if there are (pad-) branches from places in the try code that
// may throw an exception, bind these branches to a new landing pad
// block. This is done similarly to what is done in bindBranches.
MControlInstruction* ins = landingPadPatches[0];
MBasicBlock* pred = ins->block();
if (!newBlock(pred, landingPad)) {
return false;
}
ins->replaceSuccessor(0, *landingPad);
for (size_t i = 1; i < landingPadPatches.length(); i++) {
ins = landingPadPatches[i];
pred = ins->block();
if (!(*landingPad)->addPredecessor(alloc(), pred)) {
return false;
}
ins->replaceSuccessor(0, *landingPad);
}
// Set up the slots in the landing pad block.
if (!setupLandingPadSlots(landingPad)) {
return false;
}
// Clear the now bound pad patches.
landingPadPatches.clear();
return true;
}
[[nodiscard]] bool createTryTableLandingPad(TryControl* tryControl) {
MBasicBlock* landingPad;
if (!createTryLandingPadIfNeeded(tryControl->landingPadPatches,
&landingPad)) {
return false;
}
// If there is no landing pad created, no exceptions were possibly thrown
// and we don't need to do anything here.
if (!landingPad) {
return true;
}
MBasicBlock* originalBlock = curBlock_;
curBlock_ = landingPad;
bool hadCatchAll = false;
for (const TryTableCatch& tryTableCatch : tryControl->catches) {
MOZ_ASSERT(numPushed(curBlock_) == 2);
// Handle a catch_all by jumping to the target block
if (tryTableCatch.tagIndex == CatchAllIndex) {
// Get the exception from the slots we pushed when adding
// control flow patches.
curBlock_->pop();
MDefinition* exception = curBlock_->pop();
// Capture the exnref value if we need to
DefVector values;
if (tryTableCatch.captureExnRef && !values.append(exception)) {
return false;
}
// Branch to the catch_all code
if (!br(tryTableCatch.labelRelativeDepth, values)) {
return false;
}
// Break from the loop and skip the implicit rethrow that's needed
// if we didn't have a catch_all
hadCatchAll = true;
break;
}
// Handle a tagged catch by doing a compare and branch on the tag index,
// jumping to a catch block if they match, or else to a fallthrough block
// to continue the landing pad.
MBasicBlock* catchBlock = nullptr;
MBasicBlock* fallthroughBlock = nullptr;
if (!newBlock(curBlock_, &catchBlock) ||
!newBlock(curBlock_, &fallthroughBlock)) {
return false;
}
// Get the exception and its tag from the slots we pushed when adding
// control flow patches.
MDefinition* exceptionTag = curBlock_->pop();
curBlock_->pop();
// Branch to the catch block if the exception's tag matches this catch
// block's tag.
MDefinition* catchTag = loadTag(tryTableCatch.tagIndex);
MDefinition* matchesCatchTag = compare(exceptionTag, catchTag, JSOp::Eq,
MCompare::Compare_WasmAnyRef);
curBlock_->end(
MTest::New(alloc(), matchesCatchTag, catchBlock, fallthroughBlock));
// Set up the catch block by extracting the values from the exception
// object.
curBlock_ = catchBlock;
// Remove the tag and exception slots from the block, they are no
// longer necessary.
curBlock_->pop();
MDefinition* exception = curBlock_->pop();
MOZ_ASSERT(numPushed(curBlock_) == 0);
// Extract the exception values for the catch block
DefVector values;
if (!loadExceptionValues(exception, tryTableCatch.tagIndex, &values)) {
return false;
}
if (tryTableCatch.captureExnRef && !values.append(exception)) {
return false;
}
if (!br(tryTableCatch.labelRelativeDepth, values)) {
return false;
}
curBlock_ = fallthroughBlock;
}
// If there was no catch_all, we must rethrow this exception.
if (!hadCatchAll) {
MOZ_ASSERT(numPushed(curBlock_) == 2);
MDefinition* tag = curBlock_->pop();
MDefinition* exception = curBlock_->pop();
MOZ_ASSERT(numPushed(curBlock_) == 0);
if (!throwFrom(exception, tag)) {
return false;
}
}
curBlock_ = originalBlock;
return true;
}
// Consume the pending exception state from instance, and set up the slots
// of the landing pad with the exception state.
[[nodiscard]] bool setupLandingPadSlots(MBasicBlock** landingPad) {
MBasicBlock* prevBlock = curBlock_;
curBlock_ = *landingPad;
// Load the pending exception and tag
MInstruction* exception;
MInstruction* tag;
loadPendingExceptionState(&exception, &tag);
// Clear the pending exception and tag
auto* null = constantNullRef();
if (!setPendingExceptionState(null, null)) {
return false;
}
// Push the exception and its tag on the stack to make them available
// to the landing pad blocks.
if (!curBlock_->ensureHasSlots(2)) {
return false;
}
curBlock_->push(exception);
curBlock_->push(tag);
*landingPad = curBlock_;
curBlock_ = prevBlock;
return true;
}
[[nodiscard]] bool startTry() {
Control& control = iter().controlItem();
control.block = curBlock_;
control.tryControl = newTryControl();
if (!control.tryControl) {
return false;
}
control.tryControl->inBody = true;
return startBlock();
}
[[nodiscard]] bool startTryTable(TryTableCatchVector&& catches) {
Control& control = iter().controlItem();
control.block = curBlock_;
control.tryControl = newTryControl();
if (!control.tryControl) {
return false;
}
control.tryControl->inBody = true;
control.tryControl->catches = std::move(catches);
return startBlock();
}
[[nodiscard]] bool joinTryOrCatchBlock(Control& control) {
// If the try or catch block ended with dead code, there is no need to
// do any control flow join.
if (inDeadCode()) {
return true;
}
// This is a split path which we'll need to join later, using a control
// flow patch.
MOZ_ASSERT(!curBlock_->hasLastIns());
MGoto* jump = MGoto::New(alloc());
if (!addControlFlowPatch(jump, 0, MGoto::TargetIndex)) {
return false;
}
// Finish the current block with the control flow patch instruction.
curBlock_->end(jump);
return true;
}
// Finish the previous block (either a try or catch block) and then setup a
// new catch block.
[[nodiscard]] bool switchToCatch(Control& control, LabelKind fromKind,
uint32_t tagIndex) {
// Mark this control node as being no longer in the body of the try
control.tryControl->inBody = false;
// If there is no control block, then either:
// - the entry of the try block is dead code, or
// - there is no landing pad for the try-catch.
// In either case, any catch will be dead code.
if (!control.block) {
MOZ_ASSERT(inDeadCode());
return true;
}
// Join the previous try or catch block with a patch to the future join of
// the whole try-catch block.
if (!joinTryOrCatchBlock(control)) {
return false;
}
// If we are switching from the try block, create the landing pad. This is
// guaranteed to happen once and only once before processing catch blocks.
if (fromKind == LabelKind::Try) {
MBasicBlock* padBlock = nullptr;
if (!createTryLandingPadIfNeeded(control.tryControl->landingPadPatches,
&padBlock)) {
return false;
}
// Set the control block for this try-catch to the landing pad.
control.block = padBlock;
}
// If there is no landing pad, then this and following catches are dead
// code.
if (!control.block) {
curBlock_ = nullptr;
return true;
}
// Switch to the landing pad.
curBlock_ = control.block;
// Handle a catch_all by immediately jumping to a new block. We require a
// new block (as opposed to just emitting the catch_all code in the current
// block) because rethrow requires the exception/tag to be present in the
// landing pad's slots, while the catch_all block must not have the
// exception/tag in slots.
if (tagIndex == CatchAllIndex) {
MBasicBlock* catchAllBlock = nullptr;
if (!goToNewBlock(curBlock_, &catchAllBlock)) {
return false;
}
// Compilation will continue in the catch_all block.
curBlock_ = catchAllBlock;
// Remove the tag and exception slots from the block, they are no
// longer necessary.
curBlock_->pop();
curBlock_->pop();
return true;
}
// Handle a tagged catch by doing a compare and branch on the tag index,
// jumping to a catch block if they match, or else to a fallthrough block
// to continue the landing pad.
MBasicBlock* catchBlock = nullptr;
MBasicBlock* fallthroughBlock = nullptr;
if (!newBlock(curBlock_, &catchBlock) ||
!newBlock(curBlock_, &fallthroughBlock)) {
return false;
}
// Get the exception and its tag from the slots we pushed when adding
// control flow patches.
MDefinition* exceptionTag = curBlock_->pop();
MDefinition* exception = curBlock_->pop();
// Branch to the catch block if the exception's tag matches this catch
// block's tag.
MDefinition* catchTag = loadTag(tagIndex);
MDefinition* matchesCatchTag =
compare(exceptionTag, catchTag, JSOp::Eq, MCompare::Compare_WasmAnyRef);
curBlock_->end(
MTest::New(alloc(), matchesCatchTag, catchBlock, fallthroughBlock));
// The landing pad will continue in the fallthrough block
control.block = fallthroughBlock;
// Set up the catch block by extracting the values from the exception
// object.
curBlock_ = catchBlock;
// Remove the tag and exception slots from the block, they are no
// longer necessary.
curBlock_->pop();
exception = curBlock_->pop();
// Extract the exception values for the catch block
DefVector values;
if (!loadExceptionValues(exception, tagIndex, &values)) {
return false;
}
iter().setResults(values.length(), values);
return true;
}
[[nodiscard]] bool loadExceptionValues(MDefinition* exception,
uint32_t tagIndex, DefVector* values) {
SharedTagType tagType = moduleEnv().tags[tagIndex].type;
const ValTypeVector& params = tagType->argTypes();
const TagOffsetVector& offsets = tagType->argOffsets();
// Get the data pointer from the exception object
auto* data = MWasmLoadField::New(
alloc(), exception, WasmExceptionObject::offsetOfData(),
MIRType::Pointer, MWideningOp::None, AliasSet::Load(AliasSet::Any));
if (!data) {
return false;
}
curBlock_->add(data);
// Presize the values vector to the number of params
if (!values->reserve(params.length())) {
return false;
}
// Load each value from the data pointer
for (size_t i = 0; i < params.length(); i++) {
if (!mirGen_.ensureBallast()) {
return false;
}
auto* load = MWasmLoadFieldKA::New(
alloc(), exception, data, offsets[i], params[i].toMIRType(),
MWideningOp::None, AliasSet::Load(AliasSet::Any));
if (!load || !values->append(load)) {
return false;
}
curBlock_->add(load);
}
return true;
}
[[nodiscard]] bool finishTryCatch(LabelKind kind, Control& control,
DefVector* defs) {
switch (kind) {
case LabelKind::Try: {
// This is a catchless try, we must delegate all throwing instructions
// to the nearest enclosing try block if one exists, or else to the
// body block which will handle it in emitBodyDelegateThrowPad. We
// specify a relativeDepth of '1' to delegate outside of the still
// active try block.
uint32_t relativeDepth = 1;
if (!delegatePadPatches(control.tryControl->landingPadPatches,
relativeDepth)) {
return false;
}
break;
}
case LabelKind::Catch: {
MOZ_ASSERT(!control.tryControl->inBody);
// This is a try without a catch_all, we must have a rethrow at the end
// of the landing pad (if any).
MBasicBlock* padBlock = control.block;
if (padBlock) {
MBasicBlock* prevBlock = curBlock_;
curBlock_ = padBlock;
MDefinition* tag = curBlock_->pop();
MDefinition* exception = curBlock_->pop();
if (!throwFrom(exception, tag)) {
return false;
}
curBlock_ = prevBlock;
}
break;
}
case LabelKind::CatchAll: {
MOZ_ASSERT(!control.tryControl->inBody);
// This is a try with a catch_all, and requires no special handling.
break;
}
default:
MOZ_CRASH();
}
// Finish the block, joining the try and catch blocks
return finishBlock(defs);
}
[[nodiscard]] bool finishTryTable(Control& control, DefVector* defs) {
// Mark this control as no longer in the body of the try
control.tryControl->inBody = false;
// Create a landing pad for all of the catches
if (!createTryTableLandingPad(control.tryControl.get())) {
return false;
}
// Finish the block, joining the try and catch blocks
return finishBlock(defs);
}
[[nodiscard]] bool emitBodyDelegateThrowPad(Control& control) {
// Create a landing pad for any throwing instructions
MBasicBlock* padBlock;
if (!createTryLandingPadIfNeeded(bodyDelegatePadPatches_, &padBlock)) {
return false;
}
// If no landing pad was necessary, then we don't need to do anything here
if (!padBlock) {
return true;
}
// Switch to the landing pad and rethrow the exception
MBasicBlock* prevBlock = curBlock_;
curBlock_ = padBlock;
MDefinition* tag = curBlock_->pop();
MDefinition* exception = curBlock_->pop();
if (!throwFrom(exception, tag)) {
return false;
}
curBlock_ = prevBlock;
return true;
}
[[nodiscard]] bool emitNewException(MDefinition* tag,
MDefinition** exception) {
return emitInstanceCall1(readBytecodeOffset(), SASigExceptionNew, tag,
exception);
}
[[nodiscard]] bool emitThrow(uint32_t tagIndex, const DefVector& argValues) {
if (inDeadCode()) {
return true;
}
uint32_t bytecodeOffset = readBytecodeOffset();
// Load the tag
MDefinition* tag = loadTag(tagIndex);
if (!tag) {
return false;
}
// Allocate an exception object
MDefinition* exception;
if (!emitNewException(tag, &exception)) {
return false;
}
// Load the data pointer from the object
auto* data = MWasmLoadField::New(
alloc(), exception, WasmExceptionObject::offsetOfData(),
MIRType::Pointer, MWideningOp::None, AliasSet::Load(AliasSet::Any));
if (!data) {
return false;
}
curBlock_->add(data);
// Store the params into the data pointer
SharedTagType tagType = moduleEnv_.tags[tagIndex].type;
for (size_t i = 0; i < tagType->argOffsets().length(); i++) {
if (!mirGen_.ensureBallast()) {
return false;
}
ValType type = tagType->argTypes()[i];
uint32_t offset = tagType->argOffsets()[i];
if (!type.isRefRepr()) {
auto* store = MWasmStoreFieldKA::New(alloc(), exception, data, offset,
argValues[i], MNarrowingOp::None,
AliasSet::Store(AliasSet::Any));
if (!store) {
return false;
}
curBlock_->add(store);
continue;
}
// Store the new value
auto* store = MWasmStoreFieldRefKA::New(
alloc(), instancePointer_, exception, data, offset, argValues[i],
AliasSet::Store(AliasSet::Any), Nothing(), WasmPreBarrierKind::None);
if (!store) {
return false;
}
curBlock_->add(store);
// Call the post-write barrier
if (!postBarrierImmediate(bytecodeOffset, exception, data, offset,
argValues[i])) {
return false;
}
}
// Throw the exception
return throwFrom(exception, tag);
}
[[nodiscard]] bool emitThrowRef(MDefinition* exnRef) {
if (inDeadCode()) {
return true;
}
// The exception must be non-null
if (!refAsNonNull(exnRef)) {
return false;
}
// Call Instance::throwException to perform tag unpacking and throw the
// exception
if (!emitInstanceCall1(readBytecodeOffset(), SASigThrowException, exnRef)) {
return false;
}
unreachableTrap();
curBlock_ = nullptr;
return true;
}
[[nodiscard]] bool throwFrom(MDefinition* exn, MDefinition* tag) {
if (inDeadCode()) {
return true;
}
// Check if there is a local catching try control, and if so, then add a
// pad-patch to its tryPadPatches.
uint32_t relativeTryDepth;
if (inTryBlock(&relativeTryDepth)) {
// Set the pending exception state, the landing pad will read from this
if (!setPendingExceptionState(exn, tag)) {
return false;
}
// End with a pending jump to the landing pad
if (!endWithPadPatch(relativeTryDepth)) {
return false;
}
curBlock_ = nullptr;
return true;
}
// If there is no surrounding catching block, call an instance method to
// throw the exception.
if (!emitInstanceCall1(readBytecodeOffset(), SASigThrowException, exn)) {
return false;
}
unreachableTrap();
curBlock_ = nullptr;
return true;
}
[[nodiscard]] bool emitRethrow(uint32_t relativeDepth) {
if (inDeadCode()) {
return true;
}
Control& control = iter().controlItem(relativeDepth);
MBasicBlock* pad = control.block;
MOZ_ASSERT(pad);
MOZ_ASSERT(pad->nslots() > 1);
MOZ_ASSERT(iter().controlKind(relativeDepth) == LabelKind::Catch ||
iter().controlKind(relativeDepth) == LabelKind::CatchAll);
// The exception will always be the last slot in the landing pad.
size_t exnSlotPosition = pad->nslots() - 2;
MDefinition* tag = pad->getSlot(exnSlotPosition + 1);
MDefinition* exception = pad->getSlot(exnSlotPosition);
MOZ_ASSERT(exception->type() == MIRType::WasmAnyRef &&
tag->type() == MIRType::WasmAnyRef);
return throwFrom(exception, tag);
}
/*********************************************** Instance call helpers ***/
// Do not call this function directly -- it offers no protection against
// mis-counting of arguments. Instead call one of
// ::emitInstanceCall{0,1,2,3,4,5,6}.
//
// Emits a call to the Instance function indicated by `callee`. This is
// assumed to take an Instance pointer as its first argument. The remaining
// args are taken from `args`, which is assumed to hold `numArgs` entries.
// If `result` is non-null, the MDefinition* holding the return value is
// written to `*result`.
[[nodiscard]] bool emitInstanceCallN(uint32_t lineOrBytecode,
const SymbolicAddressSignature& callee,
MDefinition** args, size_t numArgs,
MDefinition** result = nullptr) {
// Check that the first formal parameter is plausibly an Instance pointer.
MOZ_ASSERT(callee.numArgs > 0);
MOZ_ASSERT(callee.argTypes[0] == MIRType::Pointer);
// Check we agree on the number of args.
MOZ_ASSERT(numArgs + 1 /* the instance pointer */ == callee.numArgs);
// Check we agree on whether a value is returned.
MOZ_ASSERT((result == nullptr) == (callee.retType == MIRType::None));
// If we are in dead code, it can happen that some of the `args` entries
// are nullptr, which will look like an OOM to the logic below. So exit
// at this point. `passInstance`, `passArg`, `finishCall` and
// `builtinInstanceMethodCall` all do nothing in dead code, so it's valid
// to exit here.
if (inDeadCode()) {
if (result) {
*result = nullptr;
}
return true;
}
// Check all args for signs of OOMness before attempting to allocating any
// more memory.
for (size_t i = 0; i < numArgs; i++) {
if (!args[i]) {
if (result) {
*result = nullptr;
}
return false;
}
}
// Finally, construct the call.
CallCompileState ccsArgs;
if (!passInstance(callee.argTypes[0], &ccsArgs)) {
return false;
}
for (size_t i = 0; i < numArgs; i++) {
if (!passArg(args[i], callee.argTypes[i + 1], &ccsArgs)) {
return false;
}
}
if (!finishCall(&ccsArgs)) {
return false;
}
return builtinInstanceMethodCall(callee, lineOrBytecode, ccsArgs, result);
}
[[nodiscard]] bool emitInstanceCall0(uint32_t lineOrBytecode,
const SymbolicAddressSignature& callee,
MDefinition** result = nullptr) {
MDefinition* args[0] = {};
return emitInstanceCallN(lineOrBytecode, callee, args, 0, result);
}
[[nodiscard]] bool emitInstanceCall1(uint32_t lineOrBytecode,
const SymbolicAddressSignature& callee,
MDefinition* arg1,
MDefinition** result = nullptr) {
MDefinition* args[1] = {arg1};
return emitInstanceCallN(lineOrBytecode, callee, args, 1, result);
}
[[nodiscard]] bool emitInstanceCall2(uint32_t lineOrBytecode,
const SymbolicAddressSignature& callee,
MDefinition* arg1, MDefinition* arg2,
MDefinition** result = nullptr) {
MDefinition* args[2] = {arg1, arg2};
return emitInstanceCallN(lineOrBytecode, callee, args, 2, result);
}
[[nodiscard]] bool emitInstanceCall3(uint32_t lineOrBytecode,
const SymbolicAddressSignature& callee,
MDefinition* arg1, MDefinition* arg2,
MDefinition* arg3,
MDefinition** result = nullptr) {
MDefinition* args[3] = {arg1, arg2, arg3};
return emitInstanceCallN(lineOrBytecode, callee, args, 3, result);
}
[[nodiscard]] bool emitInstanceCall4(uint32_t lineOrBytecode,
const SymbolicAddressSignature& callee,
MDefinition* arg1, MDefinition* arg2,
MDefinition* arg3, MDefinition* arg4,
MDefinition** result = nullptr) {
MDefinition* args[4] = {arg1, arg2, arg3, arg4};
return emitInstanceCallN(lineOrBytecode, callee, args, 4, result);
}
[[nodiscard]] bool emitInstanceCall5(uint32_t lineOrBytecode,
const SymbolicAddressSignature& callee,
MDefinition* arg1, MDefinition* arg2,
MDefinition* arg3, MDefinition* arg4,
MDefinition* arg5,
MDefinition** result = nullptr) {
MDefinition* args[5] = {arg1, arg2, arg3, arg4, arg5};
return emitInstanceCallN(lineOrBytecode, callee, args, 5, result);
}
[[nodiscard]] bool emitInstanceCall6(uint32_t lineOrBytecode,
const SymbolicAddressSignature& callee,
MDefinition* arg1, MDefinition* arg2,
MDefinition* arg3, MDefinition* arg4,
MDefinition* arg5, MDefinition* arg6,
MDefinition** result = nullptr) {
MDefinition* args[6] = {arg1, arg2, arg3, arg4, arg5, arg6};
return emitInstanceCallN(lineOrBytecode, callee, args, 6, result);
}
/******************************** WasmGC: low level load/store helpers ***/
// Given a (StorageType, FieldExtension) pair, produce the (MIRType,
// MWideningOp) pair that will give the correct operation for reading the
// value from memory.
static void fieldLoadInfoToMIR(StorageType type, FieldWideningOp wideningOp,
MIRType* mirType, MWideningOp* mirWideningOp) {
switch (type.kind()) {
case StorageType::I8: {
switch (wideningOp) {
case FieldWideningOp::Signed:
*mirType = MIRType::Int32;
*mirWideningOp = MWideningOp::FromS8;
return;
case FieldWideningOp::Unsigned:
*mirType = MIRType::Int32;
*mirWideningOp = MWideningOp::FromU8;
return;
default:
MOZ_CRASH();
}
}
case StorageType::I16: {
switch (wideningOp) {
case FieldWideningOp::Signed:
*mirType = MIRType::Int32;
*mirWideningOp = MWideningOp::FromS16;
return;
case FieldWideningOp::Unsigned:
*mirType = MIRType::Int32;
*mirWideningOp = MWideningOp::FromU16;
return;
default:
MOZ_CRASH();
}
}
default: {
switch (wideningOp) {
case FieldWideningOp::None:
*mirType = type.toMIRType();
*mirWideningOp = MWideningOp::None;
return;
default:
MOZ_CRASH();
}
}
}
}
// Given a StorageType, return the Scale required when accessing array
// elements of this type.
static Scale scaleFromFieldType(StorageType type) {
if (type.kind() == StorageType::V128) {
// V128 is accessed differently, so this scale will not be used.
return Scale::Invalid;
}
return ShiftToScale(type.indexingShift());
}
// Given a StorageType, produce the MNarrowingOp required for writing the
// value to memory.
static MNarrowingOp fieldStoreInfoToMIR(StorageType type) {
switch (type.kind()) {
case StorageType::I8:
return MNarrowingOp::To8;
case StorageType::I16:
return MNarrowingOp::To16;
default:
return MNarrowingOp::None;
}
}
// Generate a write of `value` at address `base + offset`, where `offset` is
// known at JIT time. If the written value is a reftype, the previous value
// at `base + offset` will be retrieved and handed off to the post-write
// barrier. `keepAlive` will be referenced by the instruction so as to hold
// it live (from the GC's point of view).
[[nodiscard]] bool writeGcValueAtBasePlusOffset(
uint32_t lineOrBytecode, StorageType type, MDefinition* keepAlive,
AliasSet::Flag aliasBitset, MDefinition* value, MDefinition* base,
uint32_t offset, bool needsTrapInfo, WasmPreBarrierKind preBarrierKind) {
MOZ_ASSERT(aliasBitset != 0);
MOZ_ASSERT(keepAlive->type() == MIRType::WasmAnyRef);
MOZ_ASSERT(type.widenToValType().toMIRType() == value->type());
MNarrowingOp narrowingOp = fieldStoreInfoToMIR(type);
if (!type.isRefRepr()) {
MaybeTrapSiteInfo maybeTrap;
if (needsTrapInfo) {
maybeTrap.emplace(getTrapSiteInfo());
}
auto* store = MWasmStoreFieldKA::New(
alloc(), keepAlive, base, offset, value, narrowingOp,
AliasSet::Store(aliasBitset), maybeTrap);
if (!store) {
return false;
}
curBlock_->add(store);
return true;
}
// Otherwise it's a ref store. Load the previous value so we can show it
// to the post-write barrier.
//
// Optimisation opportunity: for the case where this field write results
// from struct.new, the old value is always zero. So we should synthesise
// a suitable zero constant rather than reading it from the object. See
// also bug 1799999.
MOZ_ASSERT(narrowingOp == MNarrowingOp::None);
MOZ_ASSERT(type.widenToValType() == type.valType());
// Store the new value
auto* store = MWasmStoreFieldRefKA::New(
alloc(), instancePointer_, keepAlive, base, offset, value,
AliasSet::Store(aliasBitset), mozilla::Some(getTrapSiteInfo()),
preBarrierKind);
if (!store) {
return false;
}
curBlock_->add(store);
// Call the post-write barrier
return postBarrierImmediate(lineOrBytecode, keepAlive, base, offset, value);
}
// Generate a write of `value` at address `base + index * scale`, where
// `scale` is known at JIT-time. If the written value is a reftype, the
// previous value at `base + index * scale` will be retrieved and handed off
// to the post-write barrier. `keepAlive` will be referenced by the
// instruction so as to hold it live (from the GC's point of view).
[[nodiscard]] bool writeGcValueAtBasePlusScaledIndex(
uint32_t lineOrBytecode, StorageType type, MDefinition* keepAlive,
AliasSet::Flag aliasBitset, MDefinition* value, MDefinition* base,
uint32_t scale, MDefinition* index, WasmPreBarrierKind preBarrierKind) {
MOZ_ASSERT(aliasBitset != 0);
MOZ_ASSERT(keepAlive->type() == MIRType::WasmAnyRef);
MOZ_ASSERT(type.widenToValType().toMIRType() == value->type());
MOZ_ASSERT(scale == 1 || scale == 2 || scale == 4 || scale == 8 ||
scale == 16);
MNarrowingOp narrowingOp = fieldStoreInfoToMIR(type);
if (!type.isRefRepr()) {
MaybeTrapSiteInfo maybeTrap;
Scale scale = scaleFromFieldType(type);
auto* store = MWasmStoreElementKA::New(
alloc(), keepAlive, base, index, value, narrowingOp, scale,
AliasSet::Store(aliasBitset), maybeTrap);
if (!store) {
return false;
}
curBlock_->add(store);
return true;
}
// Otherwise it's a ref store.
MOZ_ASSERT(narrowingOp == MNarrowingOp::None);
MOZ_ASSERT(type.widenToValType() == type.valType());
// Store the new value
auto* store = MWasmStoreElementRefKA::New(
alloc(), instancePointer_, keepAlive, base, index, value,
AliasSet::Store(aliasBitset), mozilla::Some(getTrapSiteInfo()),
preBarrierKind);
if (!store) {
return false;
}
curBlock_->add(store);
return postBarrierIndex(lineOrBytecode, keepAlive, base, index,
sizeof(void*), value);
}
// Generate a read from address `base + offset`, where `offset` is known at
// JIT time. The loaded value will be widened as described by `type` and
// `fieldWideningOp`. `keepAlive` will be referenced by the instruction so as
// to hold it live (from the GC's point of view).
[[nodiscard]] MDefinition* readGcValueAtBasePlusOffset(
StorageType type, FieldWideningOp fieldWideningOp, MDefinition* keepAlive,
AliasSet::Flag aliasBitset, MDefinition* base, uint32_t offset,
bool needsTrapInfo) {
MOZ_ASSERT(aliasBitset != 0);
MOZ_ASSERT(keepAlive->type() == MIRType::WasmAnyRef);
MIRType mirType;
MWideningOp mirWideningOp;
fieldLoadInfoToMIR(type, fieldWideningOp, &mirType, &mirWideningOp);
MaybeTrapSiteInfo maybeTrap;
if (needsTrapInfo) {
maybeTrap.emplace(getTrapSiteInfo());
}
auto* load = MWasmLoadFieldKA::New(alloc(), keepAlive, base, offset,
mirType, mirWideningOp,
AliasSet::Load(aliasBitset), maybeTrap);
if (!load) {
return nullptr;
}
curBlock_->add(load);
return load;
}
// Generate a read from address `base + index * scale`, where `scale` is
// known at JIT-time. The loaded value will be widened as described by
// `type` and `fieldWideningOp`. `keepAlive` will be referenced by the
// instruction so as to hold it live (from the GC's point of view).
[[nodiscard]] MDefinition* readGcArrayValueAtIndex(
StorageType type, FieldWideningOp fieldWideningOp, MDefinition* keepAlive,
AliasSet::Flag aliasBitset, MDefinition* base, MDefinition* index) {
MOZ_ASSERT(aliasBitset != 0);
MOZ_ASSERT(keepAlive->type() == MIRType::WasmAnyRef);
MIRType mirType;
MWideningOp mirWideningOp;
fieldLoadInfoToMIR(type, fieldWideningOp, &mirType, &mirWideningOp);
Scale scale = scaleFromFieldType(type);
auto* load = MWasmLoadElementKA::New(
alloc(), keepAlive, base, index, mirType, mirWideningOp, scale,
AliasSet::Load(aliasBitset), mozilla::Some(getTrapSiteInfo()));
if (!load) {
return nullptr;
}
curBlock_->add(load);
return load;
}
/************************************************ WasmGC: type helpers ***/
// Returns an MDefinition holding the supertype vector for `typeIndex`.
[[nodiscard]] MDefinition* loadSuperTypeVector(uint32_t typeIndex) {
uint32_t stvOffset = moduleEnv().offsetOfSuperTypeVector(typeIndex);
auto* load =
MWasmLoadInstanceDataField::New(alloc(), MIRType::Pointer, stvOffset,
/*isConst=*/true, instancePointer_);
if (!load) {
return nullptr;
}
curBlock_->add(load);
return load;
}
[[nodiscard]] MDefinition* loadTypeDefInstanceData(uint32_t typeIndex) {
size_t offset = Instance::offsetInData(
moduleEnv_.offsetOfTypeDefInstanceData(typeIndex));
auto* result = MWasmDerivedPointer::New(alloc(), instancePointer_, offset);
if (!result) {
return nullptr;
}
curBlock_->add(result);
return result;
}
/********************************************** WasmGC: struct helpers ***/
[[nodiscard]] MDefinition* createStructObject(uint32_t typeIndex,
bool zeroFields) {
const TypeDef& typeDef = (*moduleEnv().types)[typeIndex];
gc::AllocKind allocKind = WasmStructObject::allocKindForTypeDef(&typeDef);
bool isOutline =
WasmStructObject::requiresOutlineBytes(typeDef.structType().size_);
// Allocate an uninitialized struct. This requires the type definition
// for the struct.
MDefinition* typeDefData = loadTypeDefInstanceData(typeIndex);
if (!typeDefData) {
return nullptr;
}
auto* structObject =
MWasmNewStructObject::New(alloc(), instancePointer_, typeDefData,
isOutline, zeroFields, allocKind);
if (!structObject) {
return nullptr;
}
curBlock_->add(structObject);
return structObject;
}
// Helper function for EmitStruct{New,Set}: given a MIR pointer to a
// WasmStructObject, a MIR pointer to a value, and a field descriptor,
// generate MIR to write the value to the relevant field in the object.
[[nodiscard]] bool writeValueToStructField(
uint32_t lineOrBytecode, const StructField& field,
MDefinition* structObject, MDefinition* value,
WasmPreBarrierKind preBarrierKind) {
StorageType fieldType = field.type;
uint32_t fieldOffset = field.offset;
bool areaIsOutline;
uint32_t areaOffset;
WasmStructObject::fieldOffsetToAreaAndOffset(fieldType, fieldOffset,
&areaIsOutline, &areaOffset);
// Make `base` point at the first byte of either the struct object as a
// whole or of the out-of-line data area. And adjust `areaOffset`
// accordingly.
MDefinition* base;
bool needsTrapInfo;
if (areaIsOutline) {
auto* load = MWasmLoadField::New(
alloc(), structObject, WasmStructObject::offsetOfOutlineData(),
MIRType::Pointer, MWideningOp::None,
AliasSet::Load(AliasSet::WasmStructOutlineDataPointer),
mozilla::Some(getTrapSiteInfo()));
if (!load) {
return false;
}
curBlock_->add(load);
base = load;
needsTrapInfo = false;
} else {
base = structObject;
needsTrapInfo = true;
areaOffset += WasmStructObject::offsetOfInlineData();
}
// The transaction is to happen at `base + areaOffset`, so to speak.
// After this point we must ignore `fieldOffset`.
// The alias set denoting the field's location, although lacking a
// Load-vs-Store indication at this point.
AliasSet::Flag fieldAliasSet = areaIsOutline
? AliasSet::WasmStructOutlineDataArea
: AliasSet::WasmStructInlineDataArea;
return writeGcValueAtBasePlusOffset(lineOrBytecode, fieldType, structObject,
fieldAliasSet, value, base, areaOffset,
needsTrapInfo, preBarrierKind);
}
// Helper function for EmitStructGet: given a MIR pointer to a
// WasmStructObject, a field descriptor and a field widening operation,
// generate MIR to read the value from the relevant field in the object.
[[nodiscard]] MDefinition* readValueFromStructField(
const StructField& field, FieldWideningOp wideningOp,
MDefinition* structObject) {
StorageType fieldType = field.type;
uint32_t fieldOffset = field.offset;
bool areaIsOutline;
uint32_t areaOffset;
WasmStructObject::fieldOffsetToAreaAndOffset(fieldType, fieldOffset,
&areaIsOutline, &areaOffset);
// Make `base` point at the first byte of either the struct object as a
// whole or of the out-of-line data area. And adjust `areaOffset`
// accordingly.
MDefinition* base;
bool needsTrapInfo;
if (areaIsOutline) {
auto* loadOOLptr = MWasmLoadField::New(
alloc(), structObject, WasmStructObject::offsetOfOutlineData(),
MIRType::Pointer, MWideningOp::None,
AliasSet::Load(AliasSet::WasmStructOutlineDataPointer),
mozilla::Some(getTrapSiteInfo()));
if (!loadOOLptr) {
return nullptr;
}
curBlock_->add(loadOOLptr);
base = loadOOLptr;
needsTrapInfo = false;
} else {
base = structObject;
needsTrapInfo = true;
areaOffset += WasmStructObject::offsetOfInlineData();
}
// The transaction is to happen at `base + areaOffset`, so to speak.
// After this point we must ignore `fieldOffset`.
// The alias set denoting the field's location, although lacking a
// Load-vs-Store indication at this point.
AliasSet::Flag fieldAliasSet = areaIsOutline
? AliasSet::WasmStructOutlineDataArea
: AliasSet::WasmStructInlineDataArea;
return readGcValueAtBasePlusOffset(fieldType, wideningOp, structObject,
fieldAliasSet, base, areaOffset,
needsTrapInfo);
}
/********************************* WasmGC: address-arithmetic helpers ***/
inline bool targetIs64Bit() const {
#ifdef JS_64BIT
return true;
#else
return false;
#endif
}
// Generate MIR to unsigned widen `val` out to the target word size. If
// `val` is already at the target word size, this is a no-op. The only
// other allowed case is where `val` is Int32 and we're compiling for a
// 64-bit target, in which case a widen is generated.
[[nodiscard]] MDefinition* unsignedWidenToTargetWord(MDefinition* val) {
if (targetIs64Bit()) {
if (val->type() == MIRType::Int32) {
auto* ext = MExtendInt32ToInt64::New(alloc(), val, /*isUnsigned=*/true);
if (!ext) {
return nullptr;
}
curBlock_->add(ext);
return ext;
}
MOZ_ASSERT(val->type() == MIRType::Int64);
return val;
}
MOZ_ASSERT(val->type() == MIRType::Int32);
return val;
}
/********************************************** WasmGC: array helpers ***/
// Given `arrayObject`, the address of a WasmArrayObject, generate MIR to
// return the contents of the WasmArrayObject::numElements_ field.
// Adds trap site info for the null check.
[[nodiscard]] MDefinition* getWasmArrayObjectNumElements(
MDefinition* arrayObject) {
MOZ_ASSERT(arrayObject->type() == MIRType::WasmAnyRef);
auto* numElements = MWasmLoadField::New(
alloc(), arrayObject, WasmArrayObject::offsetOfNumElements(),
MIRType::Int32, MWideningOp::None,
AliasSet::Load(AliasSet::WasmArrayNumElements),
mozilla::Some(getTrapSiteInfo()));
if (!numElements) {
return nullptr;
}
curBlock_->add(numElements);
return numElements;
}
// Given `arrayObject`, the address of a WasmArrayObject, generate MIR to
// return the contents of the WasmArrayObject::data_ field.
[[nodiscard]] MDefinition* getWasmArrayObjectData(MDefinition* arrayObject) {
MOZ_ASSERT(arrayObject->type() == MIRType::WasmAnyRef);
auto* data = MWasmLoadField::New(
alloc(), arrayObject, WasmArrayObject::offsetOfData(),
MIRType::WasmArrayData, MWideningOp::None,
AliasSet::Load(AliasSet::WasmArrayDataPointer),
mozilla::Some(getTrapSiteInfo()));
if (!data) {
return nullptr;
}
curBlock_->add(data);
return data;
}
// Given a JIT-time-known type index `typeIndex` and a run-time known number
// of elements `numElements`, create MIR to allocate a new wasm array,
// possibly initialized with `typeIndex`s default value.
[[nodiscard]] MDefinition* createArrayObject(uint32_t lineOrBytecode,
uint32_t typeIndex,
MDefinition* numElements,
uint32_t elemSize,
bool zeroFields) {
// Get the type definition for the array as a whole.
MDefinition* typeDefData = loadTypeDefInstanceData(typeIndex);
if (!typeDefData) {
return nullptr;
}
auto* arrayObject = MWasmNewArrayObject::New(
alloc(), instancePointer_, numElements, typeDefData, elemSize,
zeroFields, bytecodeOffset());
if (!arrayObject) {
return nullptr;
}
curBlock_->add(arrayObject);
return arrayObject;
}
// This emits MIR to perform several actions common to array loads and
// stores. Given `arrayObject`, that points to a WasmArrayObject, and an
// index value `index`, it:
//
// * Generates a trap if the array pointer is null
// * Gets the size of the array
// * Emits a bounds check of `index` against the array size
// * Retrieves the OOL object pointer from the array
// * Includes check for null via signal handler.
//
// The returned value is for the OOL object pointer.
[[nodiscard]] MDefinition* setupForArrayAccess(MDefinition* arrayObject,
MDefinition* index) {
MOZ_ASSERT(arrayObject->type() == MIRType::WasmAnyRef);
MOZ_ASSERT(index->type() == MIRType::Int32);
// Check for null is done in getWasmArrayObjectNumElements.
// Get the size value for the array.
MDefinition* numElements = getWasmArrayObjectNumElements(arrayObject);
if (!numElements) {
return nullptr;
}
// Create a bounds check.
auto* boundsCheck =
MWasmBoundsCheck::New(alloc(), index, numElements, bytecodeOffset(),
MWasmBoundsCheck::Target::Unknown);
if (!boundsCheck) {
return nullptr;
}
curBlock_->add(boundsCheck);
// Get the address of the first byte of the (OOL) data area.
return getWasmArrayObjectData(arrayObject);
}
[[nodiscard]] bool fillArray(uint32_t lineOrBytecode,
const ArrayType& arrayType,
MDefinition* arrayObject, MDefinition* index,
MDefinition* numElements, MDefinition* val,
WasmPreBarrierKind preBarrierKind) {
mozilla::DebugOnly<MIRType> valMIRType = val->type();
StorageType elemType = arrayType.elementType_;
MOZ_ASSERT(elemType.widenToValType().toMIRType() == valMIRType);
uint32_t elemSize = elemType.size();
MOZ_ASSERT(elemSize >= 1 && elemSize <= 16);
// Make `arrayBase` point at the first byte of the (OOL) data area.
MDefinition* arrayBase = getWasmArrayObjectData(arrayObject);
if (!arrayBase) {
return false;
}
// We have:
// arrayBase : TargetWord
// index : Int32
// numElements : Int32
// val : <any StorageType>
// $elemSize = arrayType.elementType_.size(); 1, 2, 4, 8 or 16
//
// Generate MIR:
// <in current block>
// limit : Int32 = index + numElements
// if (limit == index) goto after; // skip loop if trip count == 0
// loop:
// indexPhi = phi(index, indexNext)
// arrayBase[index * $elemSize] = val
// indexNext = indexPhi + 1
// if (indexNext <u limit) goto loop;
// after:
//
// We construct the loop "manually" rather than using
// FunctionCompiler::{startLoop,closeLoop} as the latter have awareness of
// the wasm view of loops, whereas the loop we're building here is not a
// wasm-level loop.
// ==== Create the "loop" and "after" blocks ====
MBasicBlock* loopBlock;
if (!newBlock(curBlock_, &loopBlock, MBasicBlock::LOOP_HEADER)) {
return false;
}
MBasicBlock* afterBlock;
if (!newBlock(loopBlock, &afterBlock)) {
return false;
}
// ==== Fill in the remainder of the block preceding the loop ====
MAdd* limit = MAdd::NewWasm(alloc(), index, numElements, MIRType::Int32);
if (!limit) {
return false;
}
curBlock_->add(limit);
// Use JSOp::StrictEq, not ::Eq, so that the comparison (and eventually
// the entire initialisation loop) will be folded out in the case where
// the number of elements is zero. See MCompare::tryFoldEqualOperands.
MDefinition* limitEqualsBase =
compare(limit, index, JSOp::StrictEq, MCompare::Compare_UInt32);
if (!limitEqualsBase) {
return false;
}
MTest* skipIfLimitEqualsBase =
MTest::New(alloc(), limitEqualsBase, afterBlock, loopBlock);
if (!skipIfLimitEqualsBase) {
return false;
}
curBlock_->end(skipIfLimitEqualsBase);
if (!afterBlock->addPredecessor(alloc(), curBlock_)) {
return false;
}
// ==== Fill in the loop block as best we can ====
curBlock_ = loopBlock;
MPhi* indexPhi = MPhi::New(alloc(), MIRType::Int32);
if (!indexPhi) {
return false;
}
if (!indexPhi->reserveLength(2)) {
return false;
}
indexPhi->addInput(index);
curBlock_->addPhi(indexPhi);
curBlock_->setLoopDepth(loopDepth_ + 1);
if (!writeGcValueAtBasePlusScaledIndex(
lineOrBytecode, elemType, arrayObject, AliasSet::WasmArrayDataArea,
val, arrayBase, elemSize, indexPhi, preBarrierKind)) {
return false;
}
auto* indexNext =
MAdd::NewWasm(alloc(), indexPhi, constantI32(1), MIRType::Int32);
if (!indexNext) {
return false;
}
curBlock_->add(indexNext);
indexPhi->addInput(indexNext);
MDefinition* indexNextLtuLimit =
compare(indexNext, limit, JSOp::Lt, MCompare::Compare_UInt32);
if (!indexNextLtuLimit) {
return false;
}
auto* continueIfIndexNextLtuLimit =
MTest::New(alloc(), indexNextLtuLimit, loopBlock, afterBlock);
if (!continueIfIndexNextLtuLimit) {
return false;
}
curBlock_->end(continueIfIndexNextLtuLimit);
if (!loopBlock->addPredecessor(alloc(), loopBlock)) {
return false;
}
// ==== Loop block completed ====
curBlock_ = afterBlock;
return true;
}
// This routine generates all MIR required for `array.new`. The returned
// value is for the newly created array.
[[nodiscard]] MDefinition* createArrayNewCallAndLoop(uint32_t lineOrBytecode,
uint32_t typeIndex,
MDefinition* numElements,
MDefinition* fillValue) {
const ArrayType& arrayType = (*moduleEnv_.types)[typeIndex].arrayType();
// Create the array object, uninitialized.
MDefinition* arrayObject =
createArrayObject(lineOrBytecode, typeIndex, numElements,
arrayType.elementType_.size(), /*zeroFields=*/false);
if (!arrayObject) {
return nullptr;
}
// Optimisation opportunity: if the fill value is zero, maybe we should
// likewise skip over the initialisation loop entirely (and, if the zero
// value is visible at JIT time, the loop will be removed). For the
// reftyped case, that would be a big win since each iteration requires a
// call to the post-write barrier routine.
if (!fillArray(lineOrBytecode, arrayType, arrayObject, constantI32(0),
numElements, fillValue, WasmPreBarrierKind::None)) {
return nullptr;
}
return arrayObject;
}
[[nodiscard]] bool createArrayFill(uint32_t lineOrBytecode,
uint32_t typeIndex,
MDefinition* arrayObject,
MDefinition* index, MDefinition* val,
MDefinition* numElements) {
MOZ_ASSERT(arrayObject->type() == MIRType::WasmAnyRef);
MOZ_ASSERT(index->type() == MIRType::Int32);
MOZ_ASSERT(numElements->type() == MIRType::Int32);
const ArrayType& arrayType = (*moduleEnv_.types)[typeIndex].arrayType();
// Check for null is done in getWasmArrayObjectNumElements.
// Get the array's actual size.
MDefinition* actualNumElements = getWasmArrayObjectNumElements(arrayObject);
if (!actualNumElements) {
return false;
}
// Create a bounds check.
auto* boundsCheck = MWasmBoundsCheckRange32::New(
alloc(), index, numElements, actualNumElements, bytecodeOffset());
if (!boundsCheck) {
return false;
}
curBlock_->add(boundsCheck);
return fillArray(lineOrBytecode, arrayType, arrayObject, index, numElements,
val, WasmPreBarrierKind::Normal);
}
/*********************************************** WasmGC: other helpers ***/
// Generate MIR that causes a trap of kind `trapKind` if `arg` is zero.
// Currently `arg` may only be a MIRType::Int32, but that requirement could
// be relaxed if needed in future.
[[nodiscard]] bool trapIfZero(wasm::Trap trapKind, MDefinition* arg) {
MOZ_ASSERT(arg->type() == MIRType::Int32);
MBasicBlock* trapBlock = nullptr;
if (!newBlock(curBlock_, &trapBlock)) {
return false;
}
auto* trap = MWasmTrap::New(alloc(), trapKind, bytecodeOffset());
if (!trap) {
return false;
}
trapBlock->end(trap);
MBasicBlock* joinBlock = nullptr;
if (!newBlock(curBlock_, &joinBlock)) {
return false;
}
auto* test = MTest::New(alloc(), arg, joinBlock, trapBlock);
if (!test) {
return false;
}
curBlock_->end(test);
curBlock_ = joinBlock;
return true;
}
[[nodiscard]] MDefinition* isRefSubtypeOf(MDefinition* ref,
RefType sourceType,
RefType destType) {
MInstruction* isSubTypeOf = nullptr;
if (destType.isTypeRef()) {
uint32_t typeIndex = moduleEnv_.types->indexOf(*destType.typeDef());
MDefinition* superSTV = loadSuperTypeVector(typeIndex);
isSubTypeOf = MWasmRefIsSubtypeOfConcrete::New(alloc(), ref, superSTV,
sourceType, destType);
} else {
isSubTypeOf =
MWasmRefIsSubtypeOfAbstract::New(alloc(), ref, sourceType, destType);
}
MOZ_ASSERT(isSubTypeOf);
curBlock_->add(isSubTypeOf);
return isSubTypeOf;
}
// Generate MIR that attempts to downcast `ref` to `castToTypeDef`. If the
// downcast fails, we trap. If it succeeds, then `ref` can be assumed to
// have a type that is a subtype of (or the same as) `castToTypeDef` after
// this point.
[[nodiscard]] bool refCast(MDefinition* ref, RefType sourceType,
RefType destType) {
MDefinition* success = isRefSubtypeOf(ref, sourceType, destType);
if (!success) {
return false;
}
// Trap if `success` is zero. If it's nonzero, we have established that
// `ref <: castToTypeDef`.
return trapIfZero(wasm::Trap::BadCast, success);
}
// Generate MIR that computes a boolean value indicating whether or not it
// is possible to downcast `ref` to `destType`.
[[nodiscard]] MDefinition* refTest(MDefinition* ref, RefType sourceType,
RefType destType) {
return isRefSubtypeOf(ref, sourceType, destType);
}
// Generates MIR for br_on_cast and br_on_cast_fail.
[[nodiscard]] bool brOnCastCommon(bool onSuccess, uint32_t labelRelativeDepth,
RefType sourceType, RefType destType,
const ResultType& labelType,
const DefVector& values) {
if (inDeadCode()) {
return true;
}
MBasicBlock* fallthroughBlock = nullptr;
if (!newBlock(curBlock_, &fallthroughBlock)) {
return false;
}
// `values` are the values in the top block-value on the stack. Since the
// argument to `br_on_cast{_fail}` is at the top of the stack, it is the
// last element in `values`.
//
// For both br_on_cast and br_on_cast_fail, the OpIter validation routines
// ensure that `values` is non-empty (by rejecting the case
// `labelType->length() < 1`) and that the last value in `values` is
// reftyped.
MOZ_RELEASE_ASSERT(values.length() > 0);
MDefinition* ref = values.back();
MOZ_ASSERT(ref->type() == MIRType::WasmAnyRef);
MDefinition* success = isRefSubtypeOf(ref, sourceType, destType);
if (!success) {
return false;
}
MTest* test;
if (onSuccess) {
test = MTest::New(alloc(), success, nullptr, fallthroughBlock);
if (!test || !addControlFlowPatch(test, labelRelativeDepth,
MTest::TrueBranchIndex)) {
return false;
}
} else {
test = MTest::New(alloc(), success, fallthroughBlock, nullptr);
if (!test || !addControlFlowPatch(test, labelRelativeDepth,
MTest::FalseBranchIndex)) {
return false;
}
}
if (!pushDefs(values)) {
return false;
}
curBlock_->end(test);
curBlock_ = fallthroughBlock;
return true;
}
[[nodiscard]] bool brOnNonStruct(const DefVector& values) {
if (inDeadCode()) {
return true;
}
MBasicBlock* fallthroughBlock = nullptr;
if (!newBlock(curBlock_, &fallthroughBlock)) {
return false;
}
MOZ_ASSERT(values.length() > 0);
MOZ_ASSERT(values.back()->type() == MIRType::WasmAnyRef);
MGoto* jump = MGoto::New(alloc(), fallthroughBlock);
if (!jump) {
return false;
}
if (!pushDefs(values)) {
return false;
}
curBlock_->end(jump);
curBlock_ = fallthroughBlock;
return true;
}
/************************************************************ DECODING ***/
// AsmJS adds a line number to `callSiteLineNums` for certain operations that
// are represented by a JS call, such as math builtins. We use these line
// numbers when calling builtins. This method will read from
// `callSiteLineNums` when we are using AsmJS, or else return the current
// bytecode offset.
//
// This method MUST be called from opcodes that AsmJS will emit a call site
// line number for, or else the arrays will get out of sync. Other opcodes
// must use `readBytecodeOffset` below.
uint32_t readCallSiteLineOrBytecode() {
if (!func_.callSiteLineNums.empty()) {
return func_.callSiteLineNums[lastReadCallSite_++];
}
return iter_.lastOpcodeOffset();
}
// Return the current bytecode offset.
uint32_t readBytecodeOffset() { return iter_.lastOpcodeOffset(); }
TrapSiteInfo getTrapSiteInfo() {
return TrapSiteInfo(wasm::BytecodeOffset(readBytecodeOffset()));
}
#if DEBUG
bool done() const { return iter_.done(); }
#endif
/*************************************************************************/
private:
[[nodiscard]] bool newBlock(MBasicBlock* pred, MBasicBlock** block,
MBasicBlock::Kind kind = MBasicBlock::NORMAL) {
*block = MBasicBlock::New(mirGraph(), info(), pred, kind);
if (!*block) {
return false;
}
mirGraph().addBlock(*block);
(*block)->setLoopDepth(loopDepth_);
return true;
}
[[nodiscard]] bool goToNewBlock(MBasicBlock* pred, MBasicBlock** block) {
if (!newBlock(pred, block)) {
return false;
}
pred->end(MGoto::New(alloc(), *block));
return true;
}
[[nodiscard]] bool goToExistingBlock(MBasicBlock* prev, MBasicBlock* next) {
MOZ_ASSERT(prev);
MOZ_ASSERT(next);
prev->end(MGoto::New(alloc(), next));
return next->addPredecessor(alloc(), prev);
}
[[nodiscard]] bool bindBranches(uint32_t absolute, DefVector* defs) {
if (absolute >= pendingBlocks_.length() ||
pendingBlocks_[absolute].patches.empty()) {
return inDeadCode() || popPushedDefs(defs);
}
ControlFlowPatchVector& patches = pendingBlocks_[absolute].patches;
MControlInstruction* ins = patches[0].ins;
MBasicBlock* pred = ins->block();
MBasicBlock* join = nullptr;
if (!newBlock(pred, &join)) {
return false;
}
// Use branch hinting information if any.
if (pendingBlocks_[absolute].hint != BranchHint::Invalid) {
join->setBranchHinting(pendingBlocks_[absolute].hint);
}
pred->mark();
ins->replaceSuccessor(patches[0].index, join);
for (size_t i = 1; i < patches.length(); i++) {
ins = patches[i].ins;
pred = ins->block();
if (!pred->isMarked()) {
if (!join->addPredecessor(alloc(), pred)) {
return false;
}
pred->mark();
}
ins->replaceSuccessor(patches[i].index, join);
}
MOZ_ASSERT_IF(curBlock_, !curBlock_->isMarked());
for (uint32_t i = 0; i < join->numPredecessors(); i++) {
join->getPredecessor(i)->unmark();
}
if (curBlock_ && !goToExistingBlock(curBlock_, join)) {
return false;
}
curBlock_ = join;
if (!popPushedDefs(defs)) {
return false;
}
patches.clear();
return true;
}
};
template <>
MDefinition* FunctionCompiler::unary<MToFloat32>(MDefinition* op) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MToFloat32::New(alloc(), op, mustPreserveNaN(op->type()));
curBlock_->add(ins);
return ins;
}
template <>
MDefinition* FunctionCompiler::unary<MWasmBuiltinTruncateToInt32>(
MDefinition* op) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MWasmBuiltinTruncateToInt32::New(alloc(), op, instancePointer_,
bytecodeOffset());
curBlock_->add(ins);
return ins;
}
template <>
MDefinition* FunctionCompiler::unary<MNot>(MDefinition* op) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MNot::NewInt32(alloc(), op);
curBlock_->add(ins);
return ins;
}
template <>
MDefinition* FunctionCompiler::unary<MAbs>(MDefinition* op, MIRType type) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MAbs::NewWasm(alloc(), op, type);
curBlock_->add(ins);
return ins;
}
} // end anonymous namespace
static bool EmitI32Const(FunctionCompiler& f) {
int32_t i32;
if (!f.iter().readI32Const(&i32)) {
return false;
}
f.iter().setResult(f.constantI32(i32));
return true;
}
static bool EmitI64Const(FunctionCompiler& f) {
int64_t i64;
if (!f.iter().readI64Const(&i64)) {
return false;
}
f.iter().setResult(f.constantI64(i64));
return true;
}
static bool EmitF32Const(FunctionCompiler& f) {
float f32;
if (!f.iter().readF32Const(&f32)) {
return false;
}
f.iter().setResult(f.constantF32(f32));
return true;
}
static bool EmitF64Const(FunctionCompiler& f) {
double f64;
if (!f.iter().readF64Const(&f64)) {
return false;
}
f.iter().setResult(f.constantF64(f64));
return true;
}
static bool EmitBlock(FunctionCompiler& f) {
ResultType params;
return f.iter().readBlock(&params) && f.startBlock();
}
static bool EmitLoop(FunctionCompiler& f) {
ResultType params;
if (!f.iter().readLoop(&params)) {
return false;
}
MBasicBlock* loopHeader;
if (!f.startLoop(&loopHeader, params.length())) {
return false;
}
f.addInterruptCheck();
f.iter().controlItem().block = loopHeader;
return true;
}
static bool EmitIf(FunctionCompiler& f) {
BranchHint branchHint =
f.iter().getBranchHint(f.funcIndex(), f.relativeBytecodeOffset());
ResultType params;
MDefinition* condition = nullptr;
if (!f.iter().readIf(&params, &condition)) {
return false;
}
MBasicBlock* elseBlock;
if (!f.branchAndStartThen(condition, &elseBlock)) {
return false;
}
// Store the branch hint in the basic block.
if (branchHint != BranchHint::Invalid) {
f.getCurBlock()->setBranchHinting(branchHint);
}
f.iter().controlItem().block = elseBlock;
return true;
}
static bool EmitElse(FunctionCompiler& f) {
ResultType paramType;
ResultType resultType;
DefVector thenValues;
if (!f.iter().readElse(&paramType, &resultType, &thenValues)) {
return false;
}
if (!f.pushDefs(thenValues)) {
return false;
}
Control& control = f.iter().controlItem();
return f.switchToElse(control.block, &control.block);
}
static bool EmitEnd(FunctionCompiler& f) {
LabelKind kind;
ResultType type;
DefVector preJoinDefs;
DefVector resultsForEmptyElse;
if (!f.iter().readEnd(&kind, &type, &preJoinDefs, &resultsForEmptyElse)) {
return false;
}
Control& control = f.iter().controlItem();
MBasicBlock* block = control.block;
if (!f.pushDefs(preJoinDefs)) {
return false;
}
// Every label case is responsible to pop the control item at the appropriate
// time for the label case
DefVector postJoinDefs;
switch (kind) {
case LabelKind::Body:
MOZ_ASSERT(!control.tryControl);
if (!f.emitBodyDelegateThrowPad(control)) {
return false;
}
if (!f.finishBlock(&postJoinDefs)) {
return false;
}
if (!f.returnValues(postJoinDefs)) {
return false;
}
f.iter().popEnd();
MOZ_ASSERT(f.iter().controlStackEmpty());
return f.iter().endFunction(f.iter().end());
case LabelKind::Block:
MOZ_ASSERT(!control.tryControl);
if (!f.finishBlock(&postJoinDefs)) {
return false;
}
f.iter().popEnd();
break;
case LabelKind::Loop:
MOZ_ASSERT(!control.tryControl);
if (!f.closeLoop(block, &postJoinDefs)) {
return false;
}
f.iter().popEnd();
break;
case LabelKind::Then: {
MOZ_ASSERT(!control.tryControl);
// If we didn't see an Else, create a trivial else block so that we create
// a diamond anyway, to preserve Ion invariants.
if (!f.switchToElse(block, &block)) {
return false;
}
if (!f.pushDefs(resultsForEmptyElse)) {
return false;
}
if (!f.joinIfElse(block, &postJoinDefs)) {
return false;
}
f.iter().popEnd();
break;
}
case LabelKind::Else:
MOZ_ASSERT(!control.tryControl);
if (!f.joinIfElse(block, &postJoinDefs)) {
return false;
}
f.iter().popEnd();
break;
case LabelKind::Try:
case LabelKind::Catch:
case LabelKind::CatchAll:
MOZ_ASSERT(control.tryControl);
if (!f.finishTryCatch(kind, control, &postJoinDefs)) {
return false;
}
f.freeTryControl(std::move(control.tryControl));
f.iter().popEnd();
break;
case LabelKind::TryTable:
MOZ_ASSERT(control.tryControl);
if (!f.finishTryTable(control, &postJoinDefs)) {
return false;
}
f.freeTryControl(std::move(control.tryControl));
f.iter().popEnd();
break;
}
MOZ_ASSERT_IF(!f.inDeadCode(), postJoinDefs.length() == type.length());
f.iter().setResults(postJoinDefs.length(), postJoinDefs);
return true;
}
static bool EmitBr(FunctionCompiler& f) {
uint32_t relativeDepth;
ResultType type;
DefVector values;
if (!f.iter().readBr(&relativeDepth, &type, &values)) {
return false;
}
return f.br(relativeDepth, values);
}
static bool EmitBrIf(FunctionCompiler& f) {
uint32_t relativeDepth;
ResultType type;
DefVector values;
MDefinition* condition;
BranchHint branchHint =
f.iter().getBranchHint(f.funcIndex(), f.relativeBytecodeOffset());
if (!f.iter().readBrIf(&relativeDepth, &type, &values, &condition)) {
return false;
}
return f.brIf(relativeDepth, values, condition, branchHint);
}
static bool EmitBrTable(FunctionCompiler& f) {
Uint32Vector depths;
uint32_t defaultDepth;
ResultType branchValueType;
DefVector branchValues;
MDefinition* index;
if (!f.iter().readBrTable(&depths, &defaultDepth, &branchValueType,
&branchValues, &index)) {
return false;
}
// If all the targets are the same, or there are no targets, we can just
// use a goto. This is not just an optimization: MaybeFoldConditionBlock
// assumes that tables have more than one successor.
bool allSameDepth = true;
for (uint32_t depth : depths) {
if (depth != defaultDepth) {
allSameDepth = false;
break;
}
}
if (allSameDepth) {
return f.br(defaultDepth, branchValues);
}
return f.brTable(index, defaultDepth, depths, branchValues);
}
static bool EmitReturn(FunctionCompiler& f) {
DefVector values;
if (!f.iter().readReturn(&values)) {
return false;
}
return f.returnValues(values);
}
static bool EmitUnreachable(FunctionCompiler& f) {
if (!f.iter().readUnreachable()) {
return false;
}
f.unreachableTrap();
return true;
}
static bool EmitTry(FunctionCompiler& f) {
ResultType params;
if (!f.iter().readTry(&params)) {
return false;
}
return f.startTry();
}
static bool EmitCatch(FunctionCompiler& f) {
LabelKind kind;
uint32_t tagIndex;
ResultType paramType, resultType;
DefVector tryValues;
if (!f.iter().readCatch(&kind, &tagIndex, &paramType, &resultType,
&tryValues)) {
return false;
}
// Pushing the results of the previous block, to properly join control flow
// after the try and after each handler, as well as potential control flow
// patches from other instrunctions. This is similar to what is done for
// if-then-else control flow and for most other control control flow joins.
if (!f.pushDefs(tryValues)) {
return false;
}
return f.switchToCatch(f.iter().controlItem(), kind, tagIndex);
}
static bool EmitCatchAll(FunctionCompiler& f) {
LabelKind kind;
ResultType paramType, resultType;
DefVector tryValues;
if (!f.iter().readCatchAll(&kind, &paramType, &resultType, &tryValues)) {
return false;
}
// Pushing the results of the previous block, to properly join control flow
// after the try and after each handler, as well as potential control flow
// patches from other instrunctions.
if (!f.pushDefs(tryValues)) {
return false;
}
return f.switchToCatch(f.iter().controlItem(), kind, CatchAllIndex);
}
static bool EmitTryTable(FunctionCompiler& f) {
ResultType params;
TryTableCatchVector catches;
if (!f.iter().readTryTable(&params, &catches)) {
return false;
}
return f.startTryTable(std::move(catches));
}
static bool EmitDelegate(FunctionCompiler& f) {
uint32_t relativeDepth;
ResultType resultType;
DefVector tryValues;
if (!f.iter().readDelegate(&relativeDepth, &resultType, &tryValues)) {
return false;
}
Control& control = f.iter().controlItem();
MBasicBlock* block = control.block;
MOZ_ASSERT(control.tryControl);
// Unless the entire try-delegate is dead code, delegate any pad-patches from
// this try to the next try-block above relativeDepth.
if (block) {
ControlInstructionVector& delegatePadPatches =
control.tryControl->landingPadPatches;
if (!f.delegatePadPatches(delegatePadPatches, relativeDepth)) {
return false;
}
}
f.freeTryControl(std::move(control.tryControl));
f.iter().popDelegate();
// Push the results of the previous block, and join control flow with
// potential control flow patches from other instrunctions in the try code.
// This is similar to what is done for EmitEnd.
if (!f.pushDefs(tryValues)) {
return false;
}
DefVector postJoinDefs;
if (!f.finishBlock(&postJoinDefs)) {
return false;
}
MOZ_ASSERT_IF(!f.inDeadCode(), postJoinDefs.length() == resultType.length());
f.iter().setResults(postJoinDefs.length(), postJoinDefs);
return true;
}
static bool EmitThrow(FunctionCompiler& f) {
uint32_t tagIndex;
DefVector argValues;
if (!f.iter().readThrow(&tagIndex, &argValues)) {
return false;
}
return f.emitThrow(tagIndex, argValues);
}
static bool EmitThrowRef(FunctionCompiler& f) {
MDefinition* exnRef;
if (!f.iter().readThrowRef(&exnRef)) {
return false;
}
return f.emitThrowRef(exnRef);
}
static bool EmitRethrow(FunctionCompiler& f) {
uint32_t relativeDepth;
if (!f.iter().readRethrow(&relativeDepth)) {
return false;
}
return f.emitRethrow(relativeDepth);
}
static bool EmitCallArgs(FunctionCompiler& f, const FuncType& funcType,
const DefVector& args, CallCompileState* call) {
for (size_t i = 0, n = funcType.args().length(); i < n; ++i) {
if (!f.mirGen().ensureBallast()) {
return false;
}
if (!f.passArg(args[i], funcType.args()[i], call)) {
return false;
}
}
ResultType resultType = ResultType::Vector(funcType.results());
if (!f.passStackResultAreaCallArg(resultType, call)) {
return false;
}
return f.finishCall(call);
}
static bool EmitCall(FunctionCompiler& f, bool asmJSFuncDef) {
uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
uint32_t funcIndex;
DefVector args;
if (asmJSFuncDef) {
if (!f.iter().readOldCallDirect(f.moduleEnv().numFuncImports, &funcIndex,
&args)) {
return false;
}
} else {
if (!f.iter().readCall(&funcIndex, &args)) {
return false;
}
}
if (f.inDeadCode()) {
return true;
}
const FuncType& funcType = *f.moduleEnv().funcs[funcIndex].type;
CallCompileState call;
if (!EmitCallArgs(f, funcType, args, &call)) {
return false;
}
DefVector results;
if (f.moduleEnv().funcIsImport(funcIndex)) {
uint32_t instanceDataOffset =
f.moduleEnv().offsetOfFuncImportInstanceData(funcIndex);
if (!f.callImport(instanceDataOffset, lineOrBytecode, call, funcType,
&results)) {
return false;
}
} else {
if (!f.callDirect(funcType, funcIndex, lineOrBytecode, call, &results)) {
return false;
}
}
f.iter().setResults(results.length(), results);
return true;
}
static bool EmitCallIndirect(FunctionCompiler& f, bool oldStyle) {
uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
uint32_t funcTypeIndex;
uint32_t tableIndex;
MDefinition* callee;
DefVector args;
if (oldStyle) {
tableIndex = 0;
if (!f.iter().readOldCallIndirect(&funcTypeIndex, &callee, &args)) {
return false;
}
} else {
if (!f.iter().readCallIndirect(&funcTypeIndex, &tableIndex, &callee,
&args)) {
return false;
}
}
if (f.inDeadCode()) {
return true;
}
const FuncType& funcType = (*f.moduleEnv().types)[funcTypeIndex].funcType();
CallCompileState call;
if (!EmitCallArgs(f, funcType, args, &call)) {
return false;
}
DefVector results;
if (!f.callIndirect(funcTypeIndex, tableIndex, callee, lineOrBytecode, call,
&results)) {
return false;
}
f.iter().setResults(results.length(), results);
return true;
}
#ifdef ENABLE_WASM_TAIL_CALLS
static bool EmitReturnCall(FunctionCompiler& f) {
uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
uint32_t funcIndex;
DefVector args;
if (!f.iter().readReturnCall(&funcIndex, &args)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
const FuncType& funcType = *f.moduleEnv().funcs[funcIndex].type;
CallCompileState call;
f.markReturnCall(&call);
if (!EmitCallArgs(f, funcType, args, &call)) {
return false;
}
DefVector results;
if (f.moduleEnv().funcIsImport(funcIndex)) {
uint32_t globalDataOffset =
f.moduleEnv().offsetOfFuncImportInstanceData(funcIndex);
if (!f.returnCallImport(globalDataOffset, lineOrBytecode, call, funcType,
&results)) {
return false;
}
} else {
if (!f.returnCallDirect(funcType, funcIndex, lineOrBytecode, call,
&results)) {
return false;
}
}
return true;
}
static bool EmitReturnCallIndirect(FunctionCompiler& f) {
uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
uint32_t funcTypeIndex;
uint32_t tableIndex;
MDefinition* callee;
DefVector args;
if (!f.iter().readReturnCallIndirect(&funcTypeIndex, &tableIndex, &callee,
&args)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
const FuncType& funcType = (*f.moduleEnv().types)[funcTypeIndex].funcType();
CallCompileState call;
f.markReturnCall(&call);
if (!EmitCallArgs(f, funcType, args, &call)) {
return false;
}
DefVector results;
return f.returnCallIndirect(funcTypeIndex, tableIndex, callee, lineOrBytecode,
call, &results);
}
#endif
#if defined(ENABLE_WASM_TAIL_CALLS) && defined(ENABLE_WASM_GC)
static bool EmitReturnCallRef(FunctionCompiler& f) {
uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
const FuncType* funcType;
MDefinition* callee;
DefVector args;
if (!f.iter().readReturnCallRef(&funcType, &callee, &args)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
CallCompileState call;
f.markReturnCall(&call);
if (!EmitCallArgs(f, *funcType, args, &call)) {
return false;
}
DefVector results;
return f.returnCallRef(*funcType, callee, lineOrBytecode, call, &results);
}
#endif
static bool EmitGetLocal(FunctionCompiler& f) {
uint32_t id;
if (!f.iter().readGetLocal(f.locals(), &id)) {
return false;
}
f.iter().setResult(f.getLocalDef(id));
return true;
}
static bool EmitSetLocal(FunctionCompiler& f) {
uint32_t id;
MDefinition* value;
if (!f.iter().readSetLocal(f.locals(), &id, &value)) {
return false;
}
f.assign(id, value);
return true;
}
static bool EmitTeeLocal(FunctionCompiler& f) {
uint32_t id;
MDefinition* value;
if (!f.iter().readTeeLocal(f.locals(), &id, &value)) {
return false;
}
f.assign(id, value);
return true;
}
static bool EmitGetGlobal(FunctionCompiler& f) {
uint32_t id;
if (!f.iter().readGetGlobal(&id)) {
return false;
}
const GlobalDesc& global = f.moduleEnv().globals[id];
if (!global.isConstant()) {
f.iter().setResult(f.loadGlobalVar(global.offset(), !global.isMutable(),
global.isIndirect(),
global.type().toMIRType()));
return true;
}
LitVal value = global.constantValue();
MDefinition* result;
switch (value.type().kind()) {
case ValType::I32:
result = f.constantI32(int32_t(value.i32()));
break;
case ValType::I64:
result = f.constantI64(int64_t(value.i64()));
break;
case ValType::F32:
result = f.constantF32(value.f32());
break;
case ValType::F64:
result = f.constantF64(value.f64());
break;
case ValType::V128:
#ifdef ENABLE_WASM_SIMD
result = f.constantV128(value.v128());
break;
#else
return f.iter().fail("Ion has no SIMD support yet");
#endif
case ValType::Ref:
MOZ_ASSERT(value.ref().isNull());
result = f.constantNullRef();
break;
default:
MOZ_CRASH("unexpected type in EmitGetGlobal");
}
f.iter().setResult(result);
return true;
}
static bool EmitSetGlobal(FunctionCompiler& f) {
uint32_t bytecodeOffset = f.readBytecodeOffset();
uint32_t id;
MDefinition* value;
if (!f.iter().readSetGlobal(&id, &value)) {
return false;
}
const GlobalDesc& global = f.moduleEnv().globals[id];
MOZ_ASSERT(global.isMutable());
return f.storeGlobalVar(bytecodeOffset, global.offset(), global.isIndirect(),
value);
}
static bool EmitTeeGlobal(FunctionCompiler& f) {
uint32_t bytecodeOffset = f.readBytecodeOffset();
uint32_t id;
MDefinition* value;
if (!f.iter().readTeeGlobal(&id, &value)) {
return false;
}
const GlobalDesc& global = f.moduleEnv().globals[id];
MOZ_ASSERT(global.isMutable());
return f.storeGlobalVar(bytecodeOffset, global.offset(), global.isIndirect(),
value);
}
template <typename MIRClass>
static bool EmitUnary(FunctionCompiler& f, ValType operandType) {
MDefinition* input;
if (!f.iter().readUnary(operandType, &input)) {
return false;
}
f.iter().setResult(f.unary<MIRClass>(input));
return true;
}
template <typename MIRClass>
static bool EmitConversion(FunctionCompiler& f, ValType operandType,
ValType resultType) {
MDefinition* input;
if (!f.iter().readConversion(operandType, resultType, &input)) {
return false;
}
f.iter().setResult(f.unary<MIRClass>(input));
return true;
}
template <typename MIRClass>
static bool EmitUnaryWithType(FunctionCompiler& f, ValType operandType,
MIRType mirType) {
MDefinition* input;
if (!f.iter().readUnary(operandType, &input)) {
return false;
}
f.iter().setResult(f.unary<MIRClass>(input, mirType));
return true;
}
template <typename MIRClass>
static bool EmitConversionWithType(FunctionCompiler& f, ValType operandType,
ValType resultType, MIRType mirType) {
MDefinition* input;
if (!f.iter().readConversion(operandType, resultType, &input)) {
return false;
}
f.iter().setResult(f.unary<MIRClass>(input, mirType));
return true;
}
static bool EmitTruncate(FunctionCompiler& f, ValType operandType,
ValType resultType, bool isUnsigned,
bool isSaturating) {
MDefinition* input = nullptr;
if (!f.iter().readConversion(operandType, resultType, &input)) {
return false;
}
TruncFlags flags = 0;
if (isUnsigned) {
flags |= TRUNC_UNSIGNED;
}
if (isSaturating) {
flags |= TRUNC_SATURATING;
}
if (resultType == ValType::I32) {
if (f.moduleEnv().isAsmJS()) {
if (input && (input->type() == MIRType::Double ||
input->type() == MIRType::Float32)) {
f.iter().setResult(f.unary<MWasmBuiltinTruncateToInt32>(input));
} else {
f.iter().setResult(f.unary<MTruncateToInt32>(input));
}
} else {
f.iter().setResult(f.truncate<MWasmTruncateToInt32>(input, flags));
}
} else {
MOZ_ASSERT(resultType == ValType::I64);
MOZ_ASSERT(!f.moduleEnv().isAsmJS());
#if defined(JS_CODEGEN_ARM)
f.iter().setResult(f.truncateWithInstance(input, flags));
#else
f.iter().setResult(f.truncate<MWasmTruncateToInt64>(input, flags));
#endif
}
return true;
}
static bool EmitSignExtend(FunctionCompiler& f, uint32_t srcSize,
uint32_t targetSize) {
MDefinition* input;
ValType type = targetSize == 4 ? ValType::I32 : ValType::I64;
if (!f.iter().readConversion(type, type, &input)) {
return false;
}
f.iter().setResult(f.signExtend(input, srcSize, targetSize));
return true;
}
static bool EmitExtendI32(FunctionCompiler& f, bool isUnsigned) {
MDefinition* input;
if (!f.iter().readConversion(ValType::I32, ValType::I64, &input)) {
return false;
}
f.iter().setResult(f.extendI32(input, isUnsigned));
return true;
}
static bool EmitConvertI64ToFloatingPoint(FunctionCompiler& f,
ValType resultType, MIRType mirType,
bool isUnsigned) {
MDefinition* input;
if (!f.iter().readConversion(ValType::I64, resultType, &input)) {
return false;
}
f.iter().setResult(f.convertI64ToFloatingPoint(input, mirType, isUnsigned));
return true;
}
static bool EmitReinterpret(FunctionCompiler& f, ValType resultType,
ValType operandType, MIRType mirType) {
MDefinition* input;
if (!f.iter().readConversion(operandType, resultType, &input)) {
return false;
}
f.iter().setResult(f.unary<MWasmReinterpret>(input, mirType));
return true;
}
static bool EmitAdd(FunctionCompiler& f, ValType type, MIRType mirType) {
MDefinition* lhs;
MDefinition* rhs;
if (!f.iter().readBinary(type, &lhs, &rhs)) {
return false;
}
f.iter().setResult(f.add(lhs, rhs, mirType));
return true;
}
static bool EmitSub(FunctionCompiler& f, ValType type, MIRType mirType) {
MDefinition* lhs;
MDefinition* rhs;
if (!f.iter().readBinary(type, &lhs, &rhs)) {
return false;
}
f.iter().setResult(f.sub(lhs, rhs, mirType));
return true;
}
static bool EmitRotate(FunctionCompiler& f, ValType type, bool isLeftRotation) {
MDefinition* lhs;
MDefinition* rhs;
if (!f.iter().readBinary(type, &lhs, &rhs)) {
return false;
}
MDefinition* result = f.rotate(lhs, rhs, type.toMIRType(), isLeftRotation);
f.iter().setResult(result);
return true;
}
static bool EmitBitNot(FunctionCompiler& f, ValType operandType) {
MDefinition* input;
if (!f.iter().readUnary(operandType, &input)) {
return false;
}
f.iter().setResult(f.bitnot(input));
return true;
}
static bool EmitBitwiseAndOrXor(FunctionCompiler& f, ValType operandType,
MIRType mirType,
MWasmBinaryBitwise::SubOpcode subOpc) {
MDefinition* lhs;
MDefinition* rhs;
if (!f.iter().readBinary(operandType, &lhs, &rhs)) {
return false;
}
f.iter().setResult(f.binary<MWasmBinaryBitwise>(lhs, rhs, mirType, subOpc));
return true;
}
template <typename MIRClass>
static bool EmitShift(FunctionCompiler& f, ValType operandType,
MIRType mirType) {
MDefinition* lhs;
MDefinition* rhs;
if (!f.iter().readBinary(operandType, &lhs, &rhs)) {
return false;
}
f.iter().setResult(f.binary<MIRClass>(lhs, rhs, mirType));
return true;
}
static bool EmitUrsh(FunctionCompiler& f, ValType operandType,
MIRType mirType) {
MDefinition* lhs;
MDefinition* rhs;
if (!f.iter().readBinary(operandType, &lhs, &rhs)) {
return false;
}
f.iter().setResult(f.ursh(lhs, rhs, mirType));
return true;
}
static bool EmitMul(FunctionCompiler& f, ValType operandType, MIRType mirType) {
MDefinition* lhs;
MDefinition* rhs;
if (!f.iter().readBinary(operandType, &lhs, &rhs)) {
return false;
}
f.iter().setResult(
f.mul(lhs, rhs, mirType,
mirType == MIRType::Int32 ? MMul::Integer : MMul::Normal));
return true;
}
static bool EmitDiv(FunctionCompiler& f, ValType operandType, MIRType mirType,
bool isUnsigned) {
MDefinition* lhs;
MDefinition* rhs;
if (!f.iter().readBinary(operandType, &lhs, &rhs)) {
return false;
}
f.iter().setResult(f.div(lhs, rhs, mirType, isUnsigned));
return true;
}
static bool EmitRem(FunctionCompiler& f, ValType operandType, MIRType mirType,
bool isUnsigned) {
MDefinition* lhs;
MDefinition* rhs;
if (!f.iter().readBinary(operandType, &lhs, &rhs)) {
return false;
}
f.iter().setResult(f.mod(lhs, rhs, mirType, isUnsigned));
return true;
}
static bool EmitMinMax(FunctionCompiler& f, ValType operandType,
MIRType mirType, bool isMax) {
MDefinition* lhs;
MDefinition* rhs;
if (!f.iter().readBinary(operandType, &lhs, &rhs)) {
return false;
}
f.iter().setResult(f.minMax(lhs, rhs, mirType, isMax));
return true;
}
static bool EmitCopySign(FunctionCompiler& f, ValType operandType) {
MDefinition* lhs;
MDefinition* rhs;
if (!f.iter().readBinary(operandType, &lhs, &rhs)) {
return false;
}
f.iter().setResult(f.binary<MCopySign>(lhs, rhs, operandType.toMIRType()));
return true;
}
static bool EmitComparison(FunctionCompiler& f, ValType operandType,
JSOp compareOp, MCompare::CompareType compareType) {
MDefinition* lhs;
MDefinition* rhs;
if (!f.iter().readComparison(operandType, &lhs, &rhs)) {
return false;
}
f.iter().setResult(f.compare(lhs, rhs, compareOp, compareType));
return true;
}
static bool EmitSelect(FunctionCompiler& f, bool typed) {
StackType type;
MDefinition* trueValue;
MDefinition* falseValue;
MDefinition* condition;
if (!f.iter().readSelect(typed, &type, &trueValue, &falseValue, &condition)) {
return false;
}
f.iter().setResult(f.select(trueValue, falseValue, condition));
return true;
}
static bool EmitLoad(FunctionCompiler& f, ValType type, Scalar::Type viewType) {
LinearMemoryAddress<MDefinition*> addr;
if (!f.iter().readLoad(type, Scalar::byteSize(viewType), &addr)) {
return false;
}
MemoryAccessDesc access(addr.memoryIndex, viewType, addr.align, addr.offset,
f.bytecodeIfNotAsmJS(),
f.hugeMemoryEnabled(addr.memoryIndex));
auto* ins = f.load(addr.base, &access, type);
if (!f.inDeadCode() && !ins) {
return false;
}
f.iter().setResult(ins);
return true;
}
static bool EmitStore(FunctionCompiler& f, ValType resultType,
Scalar::Type viewType) {
LinearMemoryAddress<MDefinition*> addr;
MDefinition* value;
if (!f.iter().readStore(resultType, Scalar::byteSize(viewType), &addr,
&value)) {
return false;
}
MemoryAccessDesc access(addr.memoryIndex, viewType, addr.align, addr.offset,
f.bytecodeIfNotAsmJS(),
f.hugeMemoryEnabled(addr.memoryIndex));
f.store(addr.base, &access, value);
return true;
}
static bool EmitTeeStore(FunctionCompiler& f, ValType resultType,
Scalar::Type viewType) {
LinearMemoryAddress<MDefinition*> addr;
MDefinition* value;
if (!f.iter().readTeeStore(resultType, Scalar::byteSize(viewType), &addr,
&value)) {
return false;
}
MOZ_ASSERT(f.isMem32(addr.memoryIndex)); // asm.js opcode
MemoryAccessDesc access(addr.memoryIndex, viewType, addr.align, addr.offset,
f.bytecodeIfNotAsmJS(),
f.hugeMemoryEnabled(addr.memoryIndex));
f.store(addr.base, &access, value);
return true;
}
static bool EmitTeeStoreWithCoercion(FunctionCompiler& f, ValType resultType,
Scalar::Type viewType) {
LinearMemoryAddress<MDefinition*> addr;
MDefinition* value;
if (!f.iter().readTeeStore(resultType, Scalar::byteSize(viewType), &addr,
&value)) {
return false;
}
if (resultType == ValType::F32 && viewType == Scalar::Float64) {
value = f.unary<MToDouble>(value);
} else if (resultType == ValType::F64 && viewType == Scalar::Float32) {
value = f.unary<MToFloat32>(value);
} else {
MOZ_CRASH("unexpected coerced store");
}
MOZ_ASSERT(f.isMem32(addr.memoryIndex)); // asm.js opcode
MemoryAccessDesc access(addr.memoryIndex, viewType, addr.align, addr.offset,
f.bytecodeIfNotAsmJS(),
f.hugeMemoryEnabled(addr.memoryIndex));
f.store(addr.base, &access, value);
return true;
}
static bool TryInlineUnaryBuiltin(FunctionCompiler& f, SymbolicAddress callee,
MDefinition* input) {
if (!input) {
return false;
}
MOZ_ASSERT(IsFloatingPointType(input->type()));
RoundingMode mode;
if (!IsRoundingFunction(callee, &mode)) {
return false;
}
if (!MNearbyInt::HasAssemblerSupport(mode)) {
return false;
}
f.iter().setResult(f.nearbyInt(input, mode));
return true;
}
static bool EmitUnaryMathBuiltinCall(FunctionCompiler& f,
const SymbolicAddressSignature& callee) {
MOZ_ASSERT(callee.numArgs == 1);
uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
MDefinition* input;
if (!f.iter().readUnary(ValType::fromMIRType(callee.argTypes[0]), &input)) {
return false;
}
if (TryInlineUnaryBuiltin(f, callee.identity, input)) {
return true;
}
CallCompileState call;
if (!f.passArg(input, callee.argTypes[0], &call)) {
return false;
}
if (!f.finishCall(&call)) {
return false;
}
MDefinition* def;
if (!f.builtinCall(callee, lineOrBytecode, call, &def)) {
return false;
}
f.iter().setResult(def);
return true;
}
static bool EmitBinaryMathBuiltinCall(FunctionCompiler& f,
const SymbolicAddressSignature& callee) {
MOZ_ASSERT(callee.numArgs == 2);
MOZ_ASSERT(callee.argTypes[0] == callee.argTypes[1]);
uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
CallCompileState call;
MDefinition* lhs;
MDefinition* rhs;
// This call to readBinary assumes both operands have the same type.
if (!f.iter().readBinary(ValType::fromMIRType(callee.argTypes[0]), &lhs,
&rhs)) {
return false;
}
if (!f.passArg(lhs, callee.argTypes[0], &call)) {
return false;
}
if (!f.passArg(rhs, callee.argTypes[1], &call)) {
return false;
}
if (!f.finishCall(&call)) {
return false;
}
MDefinition* def;
if (!f.builtinCall(callee, lineOrBytecode, call, &def)) {
return false;
}
f.iter().setResult(def);
return true;
}
static bool EmitMemoryGrow(FunctionCompiler& f) {
uint32_t bytecodeOffset = f.readBytecodeOffset();
MDefinition* delta;
uint32_t memoryIndex;
if (!f.iter().readMemoryGrow(&memoryIndex, &delta)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
MDefinition* memoryIndexValue = f.constantI32(int32_t(memoryIndex));
if (!memoryIndexValue) {
return false;
}
const SymbolicAddressSignature& callee =
f.isMem32(memoryIndex) ? SASigMemoryGrowM32 : SASigMemoryGrowM64;
MDefinition* ret;
if (!f.emitInstanceCall2(bytecodeOffset, callee, delta, memoryIndexValue,
&ret)) {
return false;
}
f.iter().setResult(ret);
return true;
}
static bool EmitMemorySize(FunctionCompiler& f) {
uint32_t bytecodeOffset = f.readBytecodeOffset();
uint32_t memoryIndex;
if (!f.iter().readMemorySize(&memoryIndex)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
MDefinition* memoryIndexValue = f.constantI32(int32_t(memoryIndex));
if (!memoryIndexValue) {
return false;
}
const SymbolicAddressSignature& callee =
f.isMem32(memoryIndex) ? SASigMemorySizeM32 : SASigMemorySizeM64;
MDefinition* ret;
if (!f.emitInstanceCall1(bytecodeOffset, callee, memoryIndexValue, &ret)) {
return false;
}
f.iter().setResult(ret);
return true;
}
static bool EmitAtomicCmpXchg(FunctionCompiler& f, ValType type,
Scalar::Type viewType) {
LinearMemoryAddress<MDefinition*> addr;
MDefinition* oldValue;
MDefinition* newValue;
if (!f.iter().readAtomicCmpXchg(&addr, type, byteSize(viewType), &oldValue,
&newValue)) {
return false;
}
MemoryAccessDesc access(
addr.memoryIndex, viewType, addr.align, addr.offset, f.bytecodeOffset(),
f.hugeMemoryEnabled(addr.memoryIndex), Synchronization::Full());
auto* ins =
f.atomicCompareExchangeHeap(addr.base, &access, type, oldValue, newValue);
if (!f.inDeadCode() && !ins) {
return false;
}
f.iter().setResult(ins);
return true;
}
static bool EmitAtomicLoad(FunctionCompiler& f, ValType type,
Scalar::Type viewType) {
LinearMemoryAddress<MDefinition*> addr;
if (!f.iter().readAtomicLoad(&addr, type, byteSize(viewType))) {
return false;
}
MemoryAccessDesc access(
addr.memoryIndex, viewType, addr.align, addr.offset, f.bytecodeOffset(),
f.hugeMemoryEnabled(addr.memoryIndex), Synchronization::Load());
auto* ins = f.load(addr.base, &access, type);
if (!f.inDeadCode() && !ins) {
return false;
}
f.iter().setResult(ins);
return true;
}
static bool EmitAtomicRMW(FunctionCompiler& f, ValType type,
Scalar::Type viewType, jit::AtomicOp op) {
LinearMemoryAddress<MDefinition*> addr;
MDefinition* value;
if (!f.iter().readAtomicRMW(&addr, type, byteSize(viewType), &value)) {
return false;
}
MemoryAccessDesc access(
addr.memoryIndex, viewType, addr.align, addr.offset, f.bytecodeOffset(),
f.hugeMemoryEnabled(addr.memoryIndex), Synchronization::Full());
auto* ins = f.atomicBinopHeap(op, addr.base, &access, type, value);
if (!f.inDeadCode() && !ins) {
return false;
}
f.iter().setResult(ins);
return true;
}
static bool EmitAtomicStore(FunctionCompiler& f, ValType type,
Scalar::Type viewType) {
LinearMemoryAddress<MDefinition*> addr;
MDefinition* value;
if (!f.iter().readAtomicStore(&addr, type, byteSize(viewType), &value)) {
return false;
}
MemoryAccessDesc access(
addr.memoryIndex, viewType, addr.align, addr.offset, f.bytecodeOffset(),
f.hugeMemoryEnabled(addr.memoryIndex), Synchronization::Store());
f.store(addr.base, &access, value);
return true;
}
static bool EmitWait(FunctionCompiler& f, ValType type, uint32_t byteSize) {
MOZ_ASSERT(type == ValType::I32 || type == ValType::I64);
MOZ_ASSERT(type.size() == byteSize);
uint32_t bytecodeOffset = f.readBytecodeOffset();
LinearMemoryAddress<MDefinition*> addr;
MDefinition* expected;
MDefinition* timeout;
if (!f.iter().readWait(&addr, type, byteSize, &expected, &timeout)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
MemoryAccessDesc access(addr.memoryIndex,
type == ValType::I32 ? Scalar::Int32 : Scalar::Int64,
addr.align, addr.offset, f.bytecodeOffset(),
f.hugeMemoryEnabled(addr.memoryIndex));
MDefinition* ptr = f.computeEffectiveAddress(addr.base, &access);
if (!ptr) {
return false;
}
MDefinition* memoryIndex = f.constantI32(int32_t(addr.memoryIndex));
if (!memoryIndex) {
return false;
}
const SymbolicAddressSignature& callee =
f.isMem32(addr.memoryIndex)
? (type == ValType::I32 ? SASigWaitI32M32 : SASigWaitI64M32)
: (type == ValType::I32 ? SASigWaitI32M64 : SASigWaitI64M64);
MDefinition* ret;
if (!f.emitInstanceCall4(bytecodeOffset, callee, ptr, expected, timeout,
memoryIndex, &ret)) {
return false;
}
f.iter().setResult(ret);
return true;
}
static bool EmitFence(FunctionCompiler& f) {
if (!f.iter().readFence()) {
return false;
}
f.fence();
return true;
}
static bool EmitWake(FunctionCompiler& f) {
uint32_t bytecodeOffset = f.readBytecodeOffset();
LinearMemoryAddress<MDefinition*> addr;
MDefinition* count;
if (!f.iter().readWake(&addr, &count)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
MemoryAccessDesc access(addr.memoryIndex, Scalar::Int32, addr.align,
addr.offset, f.bytecodeOffset(),
f.hugeMemoryEnabled(addr.memoryIndex));
MDefinition* ptr = f.computeEffectiveAddress(addr.base, &access);
if (!ptr) {
return false;
}
MDefinition* memoryIndex = f.constantI32(int32_t(addr.memoryIndex));
if (!memoryIndex) {
return false;
}
const SymbolicAddressSignature& callee =
f.isMem32(addr.memoryIndex) ? SASigWakeM32 : SASigWakeM64;
MDefinition* ret;
if (!f.emitInstanceCall3(bytecodeOffset, callee, ptr, count, memoryIndex,
&ret)) {
return false;
}
f.iter().setResult(ret);
return true;
}
static bool EmitAtomicXchg(FunctionCompiler& f, ValType type,
Scalar::Type viewType) {
LinearMemoryAddress<MDefinition*> addr;
MDefinition* value;
if (!f.iter().readAtomicRMW(&addr, type, byteSize(viewType), &value)) {
return false;
}
MemoryAccessDesc access(
addr.memoryIndex, viewType, addr.align, addr.offset, f.bytecodeOffset(),
f.hugeMemoryEnabled(addr.memoryIndex), Synchronization::Full());
MDefinition* ins = f.atomicExchangeHeap(addr.base, &access, type, value);
if (!f.inDeadCode() && !ins) {
return false;
}
f.iter().setResult(ins);
return true;
}
static bool EmitMemCopyCall(FunctionCompiler& f, uint32_t dstMemIndex,
uint32_t srcMemIndex, MDefinition* dst,
MDefinition* src, MDefinition* len) {
uint32_t bytecodeOffset = f.readBytecodeOffset();
if (dstMemIndex == srcMemIndex) {
const SymbolicAddressSignature& callee =
(f.moduleEnv().usesSharedMemory(dstMemIndex)
? (f.isMem32(dstMemIndex) ? SASigMemCopySharedM32
: SASigMemCopySharedM64)
: (f.isMem32(dstMemIndex) ? SASigMemCopyM32 : SASigMemCopyM64));
MDefinition* memoryBase = f.memoryBase(dstMemIndex);
if (!memoryBase) {
return false;
}
return f.emitInstanceCall4(bytecodeOffset, callee, dst, src, len,
memoryBase);
}
IndexType dstIndexType = f.moduleEnv().memories[dstMemIndex].indexType();
IndexType srcIndexType = f.moduleEnv().memories[srcMemIndex].indexType();
if (dstIndexType == IndexType::I32) {
dst = f.extendI32(dst, /*isUnsigned=*/true);
if (!dst) {
return false;
}
}
if (srcIndexType == IndexType::I32) {
src = f.extendI32(src, /*isUnsigned=*/true);
if (!src) {
return false;
}
}
if (dstIndexType == IndexType::I32 || srcIndexType == IndexType::I32) {
len = f.extendI32(len, /*isUnsigned=*/true);
if (!len) {
return false;
}
}
MDefinition* dstMemIndexValue = f.constantI32(int32_t(dstMemIndex));
if (!dstMemIndexValue) {
return false;
}
MDefinition* srcMemIndexValue = f.constantI32(int32_t(srcMemIndex));
if (!srcMemIndexValue) {
return false;
}
return f.emitInstanceCall5(bytecodeOffset, SASigMemCopyAny, dst, src, len,
dstMemIndexValue, srcMemIndexValue);
}
static bool EmitMemCopyInline(FunctionCompiler& f, uint32_t memoryIndex,
MDefinition* dst, MDefinition* src,
uint32_t length) {
MOZ_ASSERT(length != 0 && length <= MaxInlineMemoryCopyLength);
// Compute the number of copies of each width we will need to do
size_t remainder = length;
#ifdef ENABLE_WASM_SIMD
size_t numCopies16 = 0;
if (MacroAssembler::SupportsFastUnalignedFPAccesses()) {
numCopies16 = remainder / sizeof(V128);
remainder %= sizeof(V128);
}
#endif
#ifdef JS_64BIT
size_t numCopies8 = remainder / sizeof(uint64_t);
remainder %= sizeof(uint64_t);
#endif
size_t numCopies4 = remainder / sizeof(uint32_t);
remainder %= sizeof(uint32_t);
size_t numCopies2 = remainder / sizeof(uint16_t);
remainder %= sizeof(uint16_t);
size_t numCopies1 = remainder;
// Load all source bytes from low to high using the widest transfer width we
// can for the system. We will trap without writing anything if any source
// byte is out-of-bounds.
size_t offset = 0;
DefVector loadedValues;
#ifdef ENABLE_WASM_SIMD
for (uint32_t i = 0; i < numCopies16; i++) {
MemoryAccessDesc access(memoryIndex, Scalar::Simd128, 1, offset,
f.bytecodeOffset(),
f.hugeMemoryEnabled(memoryIndex));
auto* load = f.load(src, &access, ValType::V128);
if (!load || !loadedValues.append(load)) {
return false;
}
offset += sizeof(V128);
}
#endif
#ifdef JS_64BIT
for (uint32_t i = 0; i < numCopies8; i++) {
MemoryAccessDesc access(memoryIndex, Scalar::Int64, 1, offset,
f.bytecodeOffset(),
f.hugeMemoryEnabled(memoryIndex));
auto* load = f.load(src, &access, ValType::I64);
if (!load || !loadedValues.append(load)) {
return false;
}
offset += sizeof(uint64_t);
}
#endif
for (uint32_t i = 0; i < numCopies4; i++) {
MemoryAccessDesc access(memoryIndex, Scalar::Uint32, 1, offset,
f.bytecodeOffset(),
f.hugeMemoryEnabled(memoryIndex));
auto* load = f.load(src, &access, ValType::I32);
if (!load || !loadedValues.append(load)) {
return false;
}
offset += sizeof(uint32_t);
}
if (numCopies2) {
MemoryAccessDesc access(memoryIndex, Scalar::Uint16, 1, offset,
f.bytecodeOffset(),
f.hugeMemoryEnabled(memoryIndex));
auto* load = f.load(src, &access, ValType::I32);
if (!load || !loadedValues.append(load)) {
return false;
}
offset += sizeof(uint16_t);
}
if (numCopies1) {
MemoryAccessDesc access(memoryIndex, Scalar::Uint8, 1, offset,
f.bytecodeOffset(),
f.hugeMemoryEnabled(memoryIndex));
auto* load = f.load(src, &access, ValType::I32);
if (!load || !loadedValues.append(load)) {
return false;
}
}
// Store all source bytes to the destination from high to low. We will trap
// without writing anything on the first store if any dest byte is
// out-of-bounds.
offset = length;
if (numCopies1) {
offset -= sizeof(uint8_t);
MemoryAccessDesc access(memoryIndex, Scalar::Uint8, 1, offset,
f.bytecodeOffset(),
f.hugeMemoryEnabled(memoryIndex));
auto* value = loadedValues.popCopy();
f.store(dst, &access, value);
}
if (numCopies2) {
offset -= sizeof(uint16_t);
MemoryAccessDesc access(memoryIndex, Scalar::Uint16, 1, offset,
f.bytecodeOffset(),
f.hugeMemoryEnabled(memoryIndex));
auto* value = loadedValues.popCopy();
f.store(dst, &access, value);
}
for (uint32_t i = 0; i < numCopies4; i++) {
offset -= sizeof(uint32_t);
MemoryAccessDesc access(memoryIndex, Scalar::Uint32, 1, offset,
f.bytecodeOffset(),
f.hugeMemoryEnabled(memoryIndex));
auto* value = loadedValues.popCopy();
f.store(dst, &access, value);
}
#ifdef JS_64BIT
for (uint32_t i = 0; i < numCopies8; i++) {
offset -= sizeof(uint64_t);
MemoryAccessDesc access(memoryIndex, Scalar::Int64, 1, offset,
f.bytecodeOffset(),
f.hugeMemoryEnabled(memoryIndex));
auto* value = loadedValues.popCopy();
f.store(dst, &access, value);
}
#endif
#ifdef ENABLE_WASM_SIMD
for (uint32_t i = 0; i < numCopies16; i++) {
offset -= sizeof(V128);
MemoryAccessDesc access(memoryIndex, Scalar::Simd128, 1, offset,
f.bytecodeOffset(),
f.hugeMemoryEnabled(memoryIndex));
auto* value = loadedValues.popCopy();
f.store(dst, &access, value);
}
#endif
return true;
}
static bool EmitMemCopy(FunctionCompiler& f) {
MDefinition *dst, *src, *len;
uint32_t dstMemIndex;
uint32_t srcMemIndex;
if (!f.iter().readMemOrTableCopy(true, &dstMemIndex, &dst, &srcMemIndex, &src,
&len)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
if (dstMemIndex == srcMemIndex && len->isConstant()) {
uint64_t length = f.isMem32(dstMemIndex) ? len->toConstant()->toInt32()
: len->toConstant()->toInt64();
static_assert(MaxInlineMemoryCopyLength <= UINT32_MAX);
if (length != 0 && length <= MaxInlineMemoryCopyLength) {
return EmitMemCopyInline(f, dstMemIndex, dst, src, uint32_t(length));
}
}
return EmitMemCopyCall(f, dstMemIndex, srcMemIndex, dst, src, len);
}
static bool EmitTableCopy(FunctionCompiler& f) {
MDefinition *dst, *src, *len;
uint32_t dstTableIndex;
uint32_t srcTableIndex;
if (!f.iter().readMemOrTableCopy(false, &dstTableIndex, &dst, &srcTableIndex,
&src, &len)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
uint32_t bytecodeOffset = f.readBytecodeOffset();
MDefinition* dti = f.constantI32(int32_t(dstTableIndex));
MDefinition* sti = f.constantI32(int32_t(srcTableIndex));
return f.emitInstanceCall5(bytecodeOffset, SASigTableCopy, dst, src, len, dti,
sti);
}
static bool EmitDataOrElemDrop(FunctionCompiler& f, bool isData) {
uint32_t segIndexVal = 0;
if (!f.iter().readDataOrElemDrop(isData, &segIndexVal)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
uint32_t bytecodeOffset = f.readBytecodeOffset();
MDefinition* segIndex = f.constantI32(int32_t(segIndexVal));
const SymbolicAddressSignature& callee =
isData ? SASigDataDrop : SASigElemDrop;
return f.emitInstanceCall1(bytecodeOffset, callee, segIndex);
}
static bool EmitMemFillCall(FunctionCompiler& f, uint32_t memoryIndex,
MDefinition* start, MDefinition* val,
MDefinition* len) {
MDefinition* memoryBase = f.memoryBase(memoryIndex);
uint32_t bytecodeOffset = f.readBytecodeOffset();
const SymbolicAddressSignature& callee =
(f.moduleEnv().usesSharedMemory(memoryIndex)
? (f.isMem32(memoryIndex) ? SASigMemFillSharedM32
: SASigMemFillSharedM64)
: (f.isMem32(memoryIndex) ? SASigMemFillM32 : SASigMemFillM64));
return f.emitInstanceCall4(bytecodeOffset, callee, start, val, len,
memoryBase);
}
static bool EmitMemFillInline(FunctionCompiler& f, uint32_t memoryIndex,
MDefinition* start, MDefinition* val,
uint32_t length) {
MOZ_ASSERT(length != 0 && length <= MaxInlineMemoryFillLength);
uint32_t value = val->toConstant()->toInt32();
// Compute the number of copies of each width we will need to do
size_t remainder = length;
#ifdef ENABLE_WASM_SIMD
size_t numCopies16 = 0;
if (MacroAssembler::SupportsFastUnalignedFPAccesses()) {
numCopies16 = remainder / sizeof(V128);
remainder %= sizeof(V128);
}
#endif
#ifdef JS_64BIT
size_t numCopies8 = remainder / sizeof(uint64_t);
remainder %= sizeof(uint64_t);
#endif
size_t numCopies4 = remainder / sizeof(uint32_t);
remainder %= sizeof(uint32_t);
size_t numCopies2 = remainder / sizeof(uint16_t);
remainder %= sizeof(uint16_t);
size_t numCopies1 = remainder;
// Generate splatted definitions for wider fills as needed
#ifdef ENABLE_WASM_SIMD
MDefinition* val16 = numCopies16 ? f.constantV128(V128(value)) : nullptr;
#endif
#ifdef JS_64BIT
MDefinition* val8 =
numCopies8 ? f.constantI64(int64_t(SplatByteToUInt<uint64_t>(value, 8)))
: nullptr;
#endif
MDefinition* val4 =
numCopies4 ? f.constantI32(int32_t(SplatByteToUInt<uint32_t>(value, 4)))
: nullptr;
MDefinition* val2 =
numCopies2 ? f.constantI32(int32_t(SplatByteToUInt<uint32_t>(value, 2)))
: nullptr;
// Store the fill value to the destination from high to low. We will trap
// without writing anything on the first store if any dest byte is
// out-of-bounds.
size_t offset = length;
if (numCopies1) {
offset -= sizeof(uint8_t);
MemoryAccessDesc access(memoryIndex, Scalar::Uint8, 1, offset,
f.bytecodeOffset(),
f.hugeMemoryEnabled(memoryIndex));
f.store(start, &access, val);
}
if (numCopies2) {
offset -= sizeof(uint16_t);
MemoryAccessDesc access(memoryIndex, Scalar::Uint16, 1, offset,
f.bytecodeOffset(),
f.hugeMemoryEnabled(memoryIndex));
f.store(start, &access, val2);
}
for (uint32_t i = 0; i < numCopies4; i++) {
offset -= sizeof(uint32_t);
MemoryAccessDesc access(memoryIndex, Scalar::Uint32, 1, offset,
f.bytecodeOffset(),
f.hugeMemoryEnabled(memoryIndex));
f.store(start, &access, val4);
}
#ifdef JS_64BIT
for (uint32_t i = 0; i < numCopies8; i++) {
offset -= sizeof(uint64_t);
MemoryAccessDesc access(memoryIndex, Scalar::Int64, 1, offset,
f.bytecodeOffset(),
f.hugeMemoryEnabled(memoryIndex));
f.store(start, &access, val8);
}
#endif
#ifdef ENABLE_WASM_SIMD
for (uint32_t i = 0; i < numCopies16; i++) {
offset -= sizeof(V128);
MemoryAccessDesc access(memoryIndex, Scalar::Simd128, 1, offset,
f.bytecodeOffset(),
f.hugeMemoryEnabled(memoryIndex));
f.store(start, &access, val16);
}
#endif
return true;
}
static bool EmitMemFill(FunctionCompiler& f) {
uint32_t memoryIndex;
MDefinition *start, *val, *len;
if (!f.iter().readMemFill(&memoryIndex, &start, &val, &len)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
if (len->isConstant() && val->isConstant()) {
uint64_t length = f.isMem32(memoryIndex) ? len->toConstant()->toInt32()
: len->toConstant()->toInt64();
static_assert(MaxInlineMemoryFillLength <= UINT32_MAX);
if (length != 0 && length <= MaxInlineMemoryFillLength) {
return EmitMemFillInline(f, memoryIndex, start, val, uint32_t(length));
}
}
return EmitMemFillCall(f, memoryIndex, start, val, len);
}
static bool EmitMemOrTableInit(FunctionCompiler& f, bool isMem) {
uint32_t segIndexVal = 0, dstMemOrTableIndex = 0;
MDefinition *dstOff, *srcOff, *len;
if (!f.iter().readMemOrTableInit(isMem, &segIndexVal, &dstMemOrTableIndex,
&dstOff, &srcOff, &len)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
uint32_t bytecodeOffset = f.readBytecodeOffset();
const SymbolicAddressSignature& callee =
isMem
? (f.isMem32(dstMemOrTableIndex) ? SASigMemInitM32 : SASigMemInitM64)
: SASigTableInit;
MDefinition* segIndex = f.constantI32(int32_t(segIndexVal));
if (!segIndex) {
return false;
}
MDefinition* dti = f.constantI32(int32_t(dstMemOrTableIndex));
if (!dti) {
return false;
}
return f.emitInstanceCall5(bytecodeOffset, callee, dstOff, srcOff, len,
segIndex, dti);
}
// Note, table.{get,grow,set} on table(funcref) are currently rejected by the
// verifier.
static bool EmitTableFill(FunctionCompiler& f) {
uint32_t tableIndex;
MDefinition *start, *val, *len;
if (!f.iter().readTableFill(&tableIndex, &start, &val, &len)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
uint32_t bytecodeOffset = f.readBytecodeOffset();
MDefinition* tableIndexArg = f.constantI32(int32_t(tableIndex));
if (!tableIndexArg) {
return false;
}
return f.emitInstanceCall4(bytecodeOffset, SASigTableFill, start, val, len,
tableIndexArg);
}
#if ENABLE_WASM_MEMORY_CONTROL
static bool EmitMemDiscard(FunctionCompiler& f) {
uint32_t memoryIndex;
MDefinition *start, *len;
if (!f.iter().readMemDiscard(&memoryIndex, &start, &len)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
uint32_t bytecodeOffset = f.readBytecodeOffset();
MDefinition* memoryBase = f.memoryBase(memoryIndex);
bool isMem32 = f.isMem32(memoryIndex);
const SymbolicAddressSignature& callee =
(f.moduleEnv().usesSharedMemory(memoryIndex)
? (isMem32 ? SASigMemDiscardSharedM32 : SASigMemDiscardSharedM64)
: (isMem32 ? SASigMemDiscardM32 : SASigMemDiscardM64));
return f.emitInstanceCall3(bytecodeOffset, callee, start, len, memoryBase);
}
#endif
static bool EmitTableGet(FunctionCompiler& f) {
uint32_t tableIndex;
MDefinition* index;
if (!f.iter().readTableGet(&tableIndex, &index)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
const TableDesc& table = f.moduleEnv().tables[tableIndex];
if (table.elemType.tableRepr() == TableRepr::Ref) {
MDefinition* ret = f.tableGetAnyRef(tableIndex, index);
if (!ret) {
return false;
}
f.iter().setResult(ret);
return true;
}
uint32_t bytecodeOffset = f.readBytecodeOffset();
MDefinition* tableIndexArg = f.constantI32(int32_t(tableIndex));
if (!tableIndexArg) {
return false;
}
// The return value here is either null, denoting an error, or a short-lived
// pointer to a location containing a possibly-null ref.
MDefinition* ret;
if (!f.emitInstanceCall2(bytecodeOffset, SASigTableGet, index, tableIndexArg,
&ret)) {
return false;
}
f.iter().setResult(ret);
return true;
}
static bool EmitTableGrow(FunctionCompiler& f) {
uint32_t tableIndex;
MDefinition* initValue;
MDefinition* delta;
if (!f.iter().readTableGrow(&tableIndex, &initValue, &delta)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
uint32_t bytecodeOffset = f.readBytecodeOffset();
MDefinition* tableIndexArg = f.constantI32(int32_t(tableIndex));
if (!tableIndexArg) {
return false;
}
MDefinition* ret;
if (!f.emitInstanceCall3(bytecodeOffset, SASigTableGrow, initValue, delta,
tableIndexArg, &ret)) {
return false;
}
f.iter().setResult(ret);
return true;
}
static bool EmitTableSet(FunctionCompiler& f) {
uint32_t tableIndex;
MDefinition* index;
MDefinition* value;
if (!f.iter().readTableSet(&tableIndex, &index, &value)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
uint32_t bytecodeOffset = f.readBytecodeOffset();
const TableDesc& table = f.moduleEnv().tables[tableIndex];
if (table.elemType.tableRepr() == TableRepr::Ref) {
return f.tableSetAnyRef(tableIndex, index, value, bytecodeOffset);
}
MDefinition* tableIndexArg = f.constantI32(int32_t(tableIndex));
if (!tableIndexArg) {
return false;
}
return f.emitInstanceCall3(bytecodeOffset, SASigTableSet, index, value,
tableIndexArg);
}
static bool EmitTableSize(FunctionCompiler& f) {
uint32_t tableIndex;
if (!f.iter().readTableSize(&tableIndex)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
MDefinition* length = f.loadTableLength(tableIndex);
if (!length) {
return false;
}
f.iter().setResult(length);
return true;
}
static bool EmitRefFunc(FunctionCompiler& f) {
uint32_t funcIndex;
if (!f.iter().readRefFunc(&funcIndex)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
uint32_t bytecodeOffset = f.readBytecodeOffset();
MDefinition* funcIndexArg = f.constantI32(int32_t(funcIndex));
if (!funcIndexArg) {
return false;
}
// The return value here is either null, denoting an error, or a short-lived
// pointer to a location containing a possibly-null ref.
MDefinition* ret;
if (!f.emitInstanceCall1(bytecodeOffset, SASigRefFunc, funcIndexArg, &ret)) {
return false;
}
f.iter().setResult(ret);
return true;
}
static bool EmitRefNull(FunctionCompiler& f) {
RefType type;
if (!f.iter().readRefNull(&type)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
MDefinition* nullVal = f.constantNullRef();
if (!nullVal) {
return false;
}
f.iter().setResult(nullVal);
return true;
}
static bool EmitRefIsNull(FunctionCompiler& f) {
MDefinition* input;
if (!f.iter().readRefIsNull(&input)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
MDefinition* nullVal = f.constantNullRef();
if (!nullVal) {
return false;
}
f.iter().setResult(
f.compare(input, nullVal, JSOp::Eq, MCompare::Compare_WasmAnyRef));
return true;
}
#ifdef ENABLE_WASM_SIMD
static bool EmitConstSimd128(FunctionCompiler& f) {
V128 v128;
if (!f.iter().readV128Const(&v128)) {
return false;
}
f.iter().setResult(f.constantV128(v128));
return true;
}
static bool EmitBinarySimd128(FunctionCompiler& f, bool commutative,
SimdOp op) {
MDefinition* lhs;
MDefinition* rhs;
if (!f.iter().readBinary(ValType::V128, &lhs, &rhs)) {
return false;
}
f.iter().setResult(f.binarySimd128(lhs, rhs, commutative, op));
return true;
}
static bool EmitTernarySimd128(FunctionCompiler& f, wasm::SimdOp op) {
MDefinition* v0;
MDefinition* v1;
MDefinition* v2;
if (!f.iter().readTernary(ValType::V128, &v0, &v1, &v2)) {
return false;
}
f.iter().setResult(f.ternarySimd128(v0, v1, v2, op));
return true;
}
static bool EmitShiftSimd128(FunctionCompiler& f, SimdOp op) {
MDefinition* lhs;
MDefinition* rhs;
if (!f.iter().readVectorShift(&lhs, &rhs)) {
return false;
}
f.iter().setResult(f.shiftSimd128(lhs, rhs, op));
return true;
}
static bool EmitSplatSimd128(FunctionCompiler& f, ValType inType, SimdOp op) {
MDefinition* src;
if (!f.iter().readConversion(inType, ValType::V128, &src)) {
return false;
}
f.iter().setResult(f.scalarToSimd128(src, op));
return true;
}
static bool EmitUnarySimd128(FunctionCompiler& f, SimdOp op) {
MDefinition* src;
if (!f.iter().readUnary(ValType::V128, &src)) {
return false;
}
f.iter().setResult(f.unarySimd128(src, op));
return true;
}
static bool EmitReduceSimd128(FunctionCompiler& f, SimdOp op) {
MDefinition* src;
if (!f.iter().readConversion(ValType::V128, ValType::I32, &src)) {
return false;
}
f.iter().setResult(f.reduceSimd128(src, op, ValType::I32));
return true;
}
static bool EmitExtractLaneSimd128(FunctionCompiler& f, ValType outType,
uint32_t laneLimit, SimdOp op) {
uint32_t laneIndex;
MDefinition* src;
if (!f.iter().readExtractLane(outType, laneLimit, &laneIndex, &src)) {
return false;
}
f.iter().setResult(f.reduceSimd128(src, op, outType, laneIndex));
return true;
}
static bool EmitReplaceLaneSimd128(FunctionCompiler& f, ValType laneType,
uint32_t laneLimit, SimdOp op) {
uint32_t laneIndex;
MDefinition* lhs;
MDefinition* rhs;
if (!f.iter().readReplaceLane(laneType, laneLimit, &laneIndex, &lhs, &rhs)) {
return false;
}
f.iter().setResult(f.replaceLaneSimd128(lhs, rhs, laneIndex, op));
return true;
}
static bool EmitShuffleSimd128(FunctionCompiler& f) {
MDefinition* v1;
MDefinition* v2;
V128 control;
if (!f.iter().readVectorShuffle(&v1, &v2, &control)) {
return false;
}
f.iter().setResult(f.shuffleSimd128(v1, v2, control));
return true;
}
static bool EmitLoadSplatSimd128(FunctionCompiler& f, Scalar::Type viewType,
wasm::SimdOp splatOp) {
LinearMemoryAddress<MDefinition*> addr;
if (!f.iter().readLoadSplat(Scalar::byteSize(viewType), &addr)) {
return false;
}
auto* ins = f.loadSplatSimd128(viewType, addr, splatOp);
if (!f.inDeadCode() && !ins) {
return false;
}
f.iter().setResult(ins);
return true;
}
static bool EmitLoadExtendSimd128(FunctionCompiler& f, wasm::SimdOp op) {
LinearMemoryAddress<MDefinition*> addr;
if (!f.iter().readLoadExtend(&addr)) {
return false;
}
auto* ins = f.loadExtendSimd128(addr, op);
if (!f.inDeadCode() && !ins) {
return false;
}
f.iter().setResult(ins);
return true;
}
static bool EmitLoadZeroSimd128(FunctionCompiler& f, Scalar::Type viewType,
size_t numBytes) {
LinearMemoryAddress<MDefinition*> addr;
if (!f.iter().readLoadSplat(numBytes, &addr)) {
return false;
}
auto* ins = f.loadZeroSimd128(viewType, numBytes, addr);
if (!f.inDeadCode() && !ins) {
return false;
}
f.iter().setResult(ins);
return true;
}
static bool EmitLoadLaneSimd128(FunctionCompiler& f, uint32_t laneSize) {
uint32_t laneIndex;
MDefinition* src;
LinearMemoryAddress<MDefinition*> addr;
if (!f.iter().readLoadLane(laneSize, &addr, &laneIndex, &src)) {
return false;
}
auto* ins = f.loadLaneSimd128(laneSize, addr, laneIndex, src);
if (!f.inDeadCode() && !ins) {
return false;
}
f.iter().setResult(ins);
return true;
}
static bool EmitStoreLaneSimd128(FunctionCompiler& f, uint32_t laneSize) {
uint32_t laneIndex;
MDefinition* src;
LinearMemoryAddress<MDefinition*> addr;
if (!f.iter().readStoreLane(laneSize, &addr, &laneIndex, &src)) {
return false;
}
f.storeLaneSimd128(laneSize, addr, laneIndex, src);
return true;
}
#endif // ENABLE_WASM_SIMD
#ifdef ENABLE_WASM_GC
static bool EmitRefAsNonNull(FunctionCompiler& f) {
MDefinition* ref;
if (!f.iter().readRefAsNonNull(&ref)) {
return false;
}
return f.refAsNonNull(ref);
}
static bool EmitBrOnNull(FunctionCompiler& f) {
uint32_t relativeDepth;
ResultType type;
DefVector values;
MDefinition* condition;
if (!f.iter().readBrOnNull(&relativeDepth, &type, &values, &condition)) {
return false;
}
return f.brOnNull(relativeDepth, values, type, condition);
}
static bool EmitBrOnNonNull(FunctionCompiler& f) {
uint32_t relativeDepth;
ResultType type;
DefVector values;
MDefinition* condition;
if (!f.iter().readBrOnNonNull(&relativeDepth, &type, &values, &condition)) {
return false;
}
return f.brOnNonNull(relativeDepth, values, type, condition);
}
static bool EmitCallRef(FunctionCompiler& f) {
uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
const FuncType* funcType;
MDefinition* callee;
DefVector args;
if (!f.iter().readCallRef(&funcType, &callee, &args)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
CallCompileState call;
if (!EmitCallArgs(f, *funcType, args, &call)) {
return false;
}
DefVector results;
if (!f.callRef(*funcType, callee, lineOrBytecode, call, &results)) {
return false;
}
f.iter().setResults(results.length(), results);
return true;
}
#endif // ENABLE_WASM_GC
#ifdef ENABLE_WASM_GC
static bool EmitStructNew(FunctionCompiler& f) {
uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
uint32_t typeIndex;
DefVector args;
if (!f.iter().readStructNew(&typeIndex, &args)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
const TypeDef& typeDef = (*f.moduleEnv().types)[typeIndex];
const StructType& structType = typeDef.structType();
MOZ_ASSERT(args.length() == structType.fields_.length());
MDefinition* structObject = f.createStructObject(typeIndex, false);
if (!structObject) {
return false;
}
// And fill in the fields.
for (uint32_t fieldIndex = 0; fieldIndex < structType.fields_.length();
fieldIndex++) {
if (!f.mirGen().ensureBallast()) {
return false;
}
const StructField& field = structType.fields_[fieldIndex];
if (!f.writeValueToStructField(lineOrBytecode, field, structObject,
args[fieldIndex],
WasmPreBarrierKind::None)) {
return false;
}
}
f.iter().setResult(structObject);
return true;
}
static bool EmitStructNewDefault(FunctionCompiler& f) {
uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
uint32_t typeIndex;
if (!f.iter().readStructNewDefault(&typeIndex)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
const StructType& structType = (*f.moduleEnv().types)[typeIndex].structType();
// Allocate a default initialized struct. This requires the type definition
// for the struct.
MDefinition* typeDefData = f.loadTypeDefInstanceData(typeIndex);
if (!typeDefData) {
return false;
}
// Figure out whether we need an OOL storage area, and hence which routine
// to call.
SymbolicAddressSignature calleeSASig =
WasmStructObject::requiresOutlineBytes(structType.size_)
? SASigStructNewOOL_true
: SASigStructNewIL_true;
// Create call: structObject = Instance::structNew{IL,OOL}<true>(typeDefData)
MDefinition* structObject;
if (!f.emitInstanceCall1(lineOrBytecode, calleeSASig, typeDefData,
&structObject)) {
return false;
}
f.iter().setResult(structObject);
return true;
}
static bool EmitStructSet(FunctionCompiler& f) {
uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
uint32_t typeIndex;
uint32_t fieldIndex;
MDefinition* structObject;
MDefinition* value;
if (!f.iter().readStructSet(&typeIndex, &fieldIndex, &structObject, &value)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
// Check for null is done at writeValueToStructField.
// And fill in the field.
const StructType& structType = (*f.moduleEnv().types)[typeIndex].structType();
const StructField& field = structType.fields_[fieldIndex];
return f.writeValueToStructField(lineOrBytecode, field, structObject, value,
WasmPreBarrierKind::Normal);
}
static bool EmitStructGet(FunctionCompiler& f, FieldWideningOp wideningOp) {
uint32_t typeIndex;
uint32_t fieldIndex;
MDefinition* structObject;
if (!f.iter().readStructGet(&typeIndex, &fieldIndex, wideningOp,
&structObject)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
// Check for null is done at readValueFromStructField.
// And fetch the data.
const StructType& structType = (*f.moduleEnv().types)[typeIndex].structType();
const StructField& field = structType.fields_[fieldIndex];
MDefinition* load =
f.readValueFromStructField(field, wideningOp, structObject);
if (!load) {
return false;
}
f.iter().setResult(load);
return true;
}
static bool EmitArrayNew(FunctionCompiler& f) {
uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
uint32_t typeIndex;
MDefinition* numElements;
MDefinition* fillValue;
if (!f.iter().readArrayNew(&typeIndex, &numElements, &fillValue)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
// If the requested size exceeds MaxArrayPayloadBytes, the MIR generated by
// this helper will trap.
MDefinition* arrayObject = f.createArrayNewCallAndLoop(
lineOrBytecode, typeIndex, numElements, fillValue);
if (!arrayObject) {
return false;
}
f.iter().setResult(arrayObject);
return true;
}
static bool EmitArrayNewDefault(FunctionCompiler& f) {
// This is almost identical to EmitArrayNew, except we skip the
// initialisation loop.
uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
uint32_t typeIndex;
MDefinition* numElements;
if (!f.iter().readArrayNewDefault(&typeIndex, &numElements)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
// Create the array object, default-initialized.
const ArrayType& arrayType = (*f.moduleEnv().types)[typeIndex].arrayType();
MDefinition* arrayObject =
f.createArrayObject(lineOrBytecode, typeIndex, numElements,
arrayType.elementType_.size(), /*zeroFields=*/true);
if (!arrayObject) {
return false;
}
f.iter().setResult(arrayObject);
return true;
}
static bool EmitArrayNewFixed(FunctionCompiler& f) {
uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
uint32_t typeIndex, numElements;
DefVector values;
if (!f.iter().readArrayNewFixed(&typeIndex, &numElements, &values)) {
return false;
}
MOZ_ASSERT(values.length() == numElements);
if (f.inDeadCode()) {
return true;
}
MDefinition* numElementsDef = f.constantI32(int32_t(numElements));
if (!numElementsDef) {
return false;
}
// Create the array object, uninitialized.
const ArrayType& arrayType = (*f.moduleEnv().types)[typeIndex].arrayType();
StorageType elemType = arrayType.elementType_;
uint32_t elemSize = elemType.size();
MDefinition* arrayObject =
f.createArrayObject(lineOrBytecode, typeIndex, numElementsDef, elemSize,
/*zeroFields=*/false);
if (!arrayObject) {
return false;
}
// Make `base` point at the first byte of the (OOL) data area.
MDefinition* base = f.getWasmArrayObjectData(arrayObject);
if (!base) {
return false;
}
// Write each element in turn.
// How do we know that the offset expression `i * elemSize` below remains
// within 2^31 (signed-i32) range? In the worst case we will have 16-byte
// values, and there can be at most MaxFunctionBytes expressions, if it were
// theoretically possible to generate one expression per instruction byte.
// Hence the max offset we can be expected to generate is
// `16 * MaxFunctionBytes`.
static_assert(16 /* sizeof v128 */ * MaxFunctionBytes <=
MaxArrayPayloadBytes);
MOZ_RELEASE_ASSERT(numElements <= MaxFunctionBytes);
for (uint32_t i = 0; i < numElements; i++) {
if (!f.mirGen().ensureBallast()) {
return false;
}
// `i * elemSize` is made safe by the assertions above.
if (!f.writeGcValueAtBasePlusOffset(
lineOrBytecode, elemType, arrayObject, AliasSet::WasmArrayDataArea,
values[numElements - 1 - i], base, i * elemSize, false,
WasmPreBarrierKind::None)) {
return false;
}
}
f.iter().setResult(arrayObject);
return true;
}
static bool EmitArrayNewData(FunctionCompiler& f) {
uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
uint32_t typeIndex, segIndex;
MDefinition* segByteOffset;
MDefinition* numElements;
if (!f.iter().readArrayNewData(&typeIndex, &segIndex, &segByteOffset,
&numElements)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
// Get the type definition data for the array as a whole.
MDefinition* typeDefData = f.loadTypeDefInstanceData(typeIndex);
if (!typeDefData) {
return false;
}
// Other values we need to pass to the instance call:
MDefinition* segIndexM = f.constantI32(int32_t(segIndex));
if (!segIndexM) {
return false;
}
// Create call:
// arrayObject = Instance::arrayNewData(segByteOffset:u32, numElements:u32,
// typeDefData:word, segIndex:u32)
// If the requested size exceeds MaxArrayPayloadBytes, the MIR generated by
// this call will trap.
MDefinition* arrayObject;
if (!f.emitInstanceCall4(lineOrBytecode, SASigArrayNewData, segByteOffset,
numElements, typeDefData, segIndexM, &arrayObject)) {
return false;
}
f.iter().setResult(arrayObject);
return true;
}
static bool EmitArrayNewElem(FunctionCompiler& f) {
uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
uint32_t typeIndex, segIndex;
MDefinition* segElemIndex;
MDefinition* numElements;
if (!f.iter().readArrayNewElem(&typeIndex, &segIndex, &segElemIndex,
&numElements)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
// Get the type definition for the array as a whole.
// Get the type definition data for the array as a whole.
MDefinition* typeDefData = f.loadTypeDefInstanceData(typeIndex);
if (!typeDefData) {
return false;
}
// Other values we need to pass to the instance call:
MDefinition* segIndexM = f.constantI32(int32_t(segIndex));
if (!segIndexM) {
return false;
}
// Create call:
// arrayObject = Instance::arrayNewElem(segElemIndex:u32, numElements:u32,
// typeDefData:word, segIndex:u32)
// If the requested size exceeds MaxArrayPayloadBytes, the MIR generated by
// this call will trap.
MDefinition* arrayObject;
if (!f.emitInstanceCall4(lineOrBytecode, SASigArrayNewElem, segElemIndex,
numElements, typeDefData, segIndexM, &arrayObject)) {
return false;
}
f.iter().setResult(arrayObject);
return true;
}
static bool EmitArrayInitData(FunctionCompiler& f) {
uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
uint32_t typeIndex, segIndex;
MDefinition* array;
MDefinition* arrayIndex;
MDefinition* segOffset;
MDefinition* length;
if (!f.iter().readArrayInitData(&typeIndex, &segIndex, &array, &arrayIndex,
&segOffset, &length)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
// Get the type definition data for the array as a whole.
MDefinition* typeDefData = f.loadTypeDefInstanceData(typeIndex);
if (!typeDefData) {
return false;
}
// Other values we need to pass to the instance call:
MDefinition* segIndexM = f.constantI32(int32_t(segIndex));
if (!segIndexM) {
return false;
}
// Create call:
// Instance::arrayInitData(array:word, index:u32, segByteOffset:u32,
// numElements:u32, typeDefData:word, segIndex:u32) If the requested size
// exceeds MaxArrayPayloadBytes, the MIR generated by this call will trap.
return f.emitInstanceCall6(lineOrBytecode, SASigArrayInitData, array,
arrayIndex, segOffset, length, typeDefData,
segIndexM);
}
static bool EmitArrayInitElem(FunctionCompiler& f) {
uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
uint32_t typeIndex, segIndex;
MDefinition* array;
MDefinition* arrayIndex;
MDefinition* segOffset;
MDefinition* length;
if (!f.iter().readArrayInitElem(&typeIndex, &segIndex, &array, &arrayIndex,
&segOffset, &length)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
// Get the type definition data for the array as a whole.
MDefinition* typeDefData = f.loadTypeDefInstanceData(typeIndex);
if (!typeDefData) {
return false;
}
// Other values we need to pass to the instance call:
MDefinition* segIndexM = f.constantI32(int32_t(segIndex));
if (!segIndexM) {
return false;
}
// Create call:
// Instance::arrayInitElem(array:word, index:u32, segByteOffset:u32,
// numElements:u32, typeDefData:word, segIndex:u32) If the requested size
// exceeds MaxArrayPayloadBytes, the MIR generated by this call will trap.
return f.emitInstanceCall6(lineOrBytecode, SASigArrayInitElem, array,
arrayIndex, segOffset, length, typeDefData,
segIndexM);
}
static bool EmitArraySet(FunctionCompiler& f) {
uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
uint32_t typeIndex;
MDefinition* value;
MDefinition* index;
MDefinition* arrayObject;
if (!f.iter().readArraySet(&typeIndex, &value, &index, &arrayObject)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
// Check for null is done at setupForArrayAccess.
// Create the object null check and the array bounds check and get the OOL
// data pointer.
MDefinition* base = f.setupForArrayAccess(arrayObject, index);
if (!base) {
return false;
}
// And do the store.
const ArrayType& arrayType = (*f.moduleEnv().types)[typeIndex].arrayType();
StorageType elemType = arrayType.elementType_;
uint32_t elemSize = elemType.size();
MOZ_ASSERT(elemSize >= 1 && elemSize <= 16);
return f.writeGcValueAtBasePlusScaledIndex(
lineOrBytecode, elemType, arrayObject, AliasSet::WasmArrayDataArea, value,
base, elemSize, index, WasmPreBarrierKind::Normal);
}
static bool EmitArrayGet(FunctionCompiler& f, FieldWideningOp wideningOp) {
uint32_t typeIndex;
MDefinition* index;
MDefinition* arrayObject;
if (!f.iter().readArrayGet(&typeIndex, wideningOp, &index, &arrayObject)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
// Check for null is done at setupForArrayAccess.
// Create the object null check and the array bounds check and get the data
// pointer.
MDefinition* base = f.setupForArrayAccess(arrayObject, index);
if (!base) {
return false;
}
// And do the load.
const ArrayType& arrayType = (*f.moduleEnv().types)[typeIndex].arrayType();
StorageType elemType = arrayType.elementType_;
MDefinition* load =
f.readGcArrayValueAtIndex(elemType, wideningOp, arrayObject,
AliasSet::WasmArrayDataArea, base, index);
if (!load) {
return false;
}
f.iter().setResult(load);
return true;
}
static bool EmitArrayLen(FunctionCompiler& f) {
MDefinition* arrayObject;
if (!f.iter().readArrayLen(&arrayObject)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
// Check for null is done at getWasmArrayObjectNumElements.
// Get the size value for the array
MDefinition* numElements = f.getWasmArrayObjectNumElements(arrayObject);
if (!numElements) {
return false;
}
f.iter().setResult(numElements);
return true;
}
static bool EmitArrayCopy(FunctionCompiler& f) {
uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
int32_t elemSize;
bool elemsAreRefTyped;
MDefinition* dstArrayObject;
MDefinition* dstArrayIndex;
MDefinition* srcArrayObject;
MDefinition* srcArrayIndex;
MDefinition* numElements;
if (!f.iter().readArrayCopy(&elemSize, &elemsAreRefTyped, &dstArrayObject,
&dstArrayIndex, &srcArrayObject, &srcArrayIndex,
&numElements)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
MOZ_ASSERT_IF(elemsAreRefTyped,
size_t(elemSize) == MIRTypeToSize(TargetWordMIRType()));
MOZ_ASSERT_IF(!elemsAreRefTyped, elemSize == 1 || elemSize == 2 ||
elemSize == 4 || elemSize == 8 ||
elemSize == 16);
// A negative element size is used to inform Instance::arrayCopy that the
// values are reftyped. This avoids having to pass it an extra boolean
// argument.
MDefinition* elemSizeDef =
f.constantI32(elemsAreRefTyped ? -elemSize : elemSize);
if (!elemSizeDef) {
return false;
}
// Create call:
// Instance::arrayCopy(dstArrayObject:word, dstArrayIndex:u32,
// srcArrayObject:word, srcArrayIndex:u32,
// numElements:u32,
// (elemsAreRefTyped ? -elemSize : elemSize):u32))
return f.emitInstanceCall6(lineOrBytecode, SASigArrayCopy, dstArrayObject,
dstArrayIndex, srcArrayObject, srcArrayIndex,
numElements, elemSizeDef);
}
static bool EmitArrayFill(FunctionCompiler& f) {
uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode();
uint32_t typeIndex;
MDefinition* array;
MDefinition* index;
MDefinition* val;
MDefinition* numElements;
if (!f.iter().readArrayFill(&typeIndex, &array, &index, &val, &numElements)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
return f.createArrayFill(lineOrBytecode, typeIndex, array, index, val,
numElements);
}
static bool EmitRefI31(FunctionCompiler& f) {
MDefinition* input;
if (!f.iter().readConversion(
ValType::I32, ValType(RefType::i31().asNonNullable()), &input)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
MDefinition* output = f.refI31(input);
if (!output) {
return false;
}
f.iter().setResult(output);
return true;
}
static bool EmitI31Get(FunctionCompiler& f, FieldWideningOp wideningOp) {
MOZ_ASSERT(wideningOp != FieldWideningOp::None);
MDefinition* input;
if (!f.iter().readConversion(ValType(RefType::i31()), ValType::I32, &input)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
if (!f.refAsNonNull(input)) {
return false;
}
MDefinition* output = f.i31Get(input, wideningOp);
if (!output) {
return false;
}
f.iter().setResult(output);
return true;
}
static bool EmitRefTest(FunctionCompiler& f, bool nullable) {
MDefinition* ref;
RefType sourceType;
RefType destType;
if (!f.iter().readRefTest(nullable, &sourceType, &destType, &ref)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
MDefinition* success = f.refTest(ref, sourceType, destType);
if (!success) {
return false;
}
f.iter().setResult(success);
return true;
}
static bool EmitRefCast(FunctionCompiler& f, bool nullable) {
MDefinition* ref;
RefType sourceType;
RefType destType;
if (!f.iter().readRefCast(nullable, &sourceType, &destType, &ref)) {
return false;
}
if (f.inDeadCode()) {
return true;
}
if (!f.refCast(ref, sourceType, destType)) {
return false;
}
f.iter().setResult(ref);
return true;
}
static bool EmitBrOnCast(FunctionCompiler& f, bool onSuccess) {
uint32_t labelRelativeDepth;
RefType sourceType;
RefType destType;
ResultType labelType;
DefVector values;
if (!f.iter().readBrOnCast(onSuccess, &labelRelativeDepth, &sourceType,
&destType, &labelType, &values)) {
return false;
}
return f.brOnCastCommon(onSuccess, labelRelativeDepth, sourceType, destType,
labelType, values);
}
static bool EmitAnyConvertExtern(FunctionCompiler& f) {
// any.convert_extern is a no-op because anyref and extern share the same
// representation
MDefinition* ref;
if (!f.iter().readRefConversion(RefType::extern_(), RefType::any(), &ref)) {
return false;
}
f.iter().setResult(ref);
return true;
}
static bool EmitExternConvertAny(FunctionCompiler& f) {
// extern.convert_any is a no-op because anyref and extern share the same
// representation
MDefinition* ref;
if (!f.iter().readRefConversion(RefType::any(), RefType::extern_(), &ref)) {
return false;
}
f.iter().setResult(ref);
return true;
}
#endif // ENABLE_WASM_GC
static bool EmitCallBuiltinModuleFunc(FunctionCompiler& f) {
// It's almost possible to use FunctionCompiler::emitInstanceCallN here.
// Unfortunately not currently possible though, since ::emitInstanceCallN
// expects an array of arguments along with a size, and that's not what is
// available here. It would be possible if we were prepared to copy
// `builtinModuleFunc->params` into a fixed-sized (16 element?) array, add
// `memoryBase`, and make the call.
const BuiltinModuleFunc* builtinModuleFunc;
DefVector params;
if (!f.iter().readCallBuiltinModuleFunc(&builtinModuleFunc, &params)) {
return false;
}
uint32_t bytecodeOffset = f.readBytecodeOffset();
const SymbolicAddressSignature& callee = *builtinModuleFunc->sig();
CallCompileState args;
if (!f.passInstance(callee.argTypes[0], &args)) {
return false;
}
if (!f.passArgs(params, builtinModuleFunc->funcType()->args(), &args)) {
return false;
}
if (builtinModuleFunc->usesMemory()) {
MDefinition* memoryBase = f.memoryBase(0);
if (!f.passArg(memoryBase, MIRType::Pointer, &args)) {
return false;
}
}
if (!f.finishCall(&args)) {
return false;
}
bool hasResult = !builtinModuleFunc->funcType()->results().empty();
MDefinition* result = nullptr;
MDefinition** resultOutParam = hasResult ? &result : nullptr;
if (!f.builtinInstanceMethodCall(callee, bytecodeOffset, args,
resultOutParam)) {
return false;
}
if (hasResult) {
f.iter().setResult(result);
}
return true;
}
static bool EmitBodyExprs(FunctionCompiler& f) {
if (!f.iter().startFunction(f.funcIndex(), f.locals())) {
return false;
}
#define CHECK(c) \
if (!(c)) return false; \
break
while (true) {
if (!f.mirGen().ensureBallast()) {
return false;
}
OpBytes op;
if (!f.iter().readOp(&op)) {
return false;
}
switch (op.b0) {
case uint16_t(Op::End):
if (!EmitEnd(f)) {
return false;
}
if (f.iter().controlStackEmpty()) {
return true;
}
break;
// Control opcodes
case uint16_t(Op::Unreachable):
CHECK(EmitUnreachable(f));
case uint16_t(Op::Nop):
CHECK(f.iter().readNop());
case uint16_t(Op::Block):
CHECK(EmitBlock(f));
case uint16_t(Op::Loop):
CHECK(EmitLoop(f));
case uint16_t(Op::If):
CHECK(EmitIf(f));
case uint16_t(Op::Else):
CHECK(EmitElse(f));
case uint16_t(Op::Try):
CHECK(EmitTry(f));
case uint16_t(Op::Catch):
CHECK(EmitCatch(f));
case uint16_t(Op::CatchAll):
CHECK(EmitCatchAll(f));
case uint16_t(Op::Delegate):
if (!EmitDelegate(f)) {
return false;
}
break;
case uint16_t(Op::Throw):
CHECK(EmitThrow(f));
case uint16_t(Op::Rethrow):
CHECK(EmitRethrow(f));
case uint16_t(Op::ThrowRef):
if (!f.moduleEnv().exnrefEnabled()) {
return f.iter().unrecognizedOpcode(&op);
}
CHECK(EmitThrowRef(f));
case uint16_t(Op::TryTable):
if (!f.moduleEnv().exnrefEnabled()) {
return f.iter().unrecognizedOpcode(&op);
}
CHECK(EmitTryTable(f));
case uint16_t(Op::Br):
CHECK(EmitBr(f));
case uint16_t(Op::BrIf):
CHECK(EmitBrIf(f));
case uint16_t(Op::BrTable):
CHECK(EmitBrTable(f));
case uint16_t(Op::Return):
CHECK(EmitReturn(f));
// Calls
case uint16_t(Op::Call):
CHECK(EmitCall(f, /* asmJSFuncDef = */ false));
case uint16_t(Op::CallIndirect):
CHECK(EmitCallIndirect(f, /* oldStyle = */ false));
// Parametric operators
case uint16_t(Op::Drop):
CHECK(f.iter().readDrop());
case uint16_t(Op::SelectNumeric):
CHECK(EmitSelect(f, /*typed*/ false));
case uint16_t(Op::SelectTyped):
CHECK(EmitSelect(f, /*typed*/ true));
// Locals and globals
case uint16_t(Op::LocalGet):
CHECK(EmitGetLocal(f));
case uint16_t(Op::LocalSet):
CHECK(EmitSetLocal(f));
case uint16_t(Op::LocalTee):
CHECK(EmitTeeLocal(f));
case uint16_t(Op::GlobalGet):
CHECK(EmitGetGlobal(f));
case uint16_t(Op::GlobalSet):
CHECK(EmitSetGlobal(f));
case uint16_t(Op::TableGet):
CHECK(EmitTableGet(f));
case uint16_t(Op::TableSet):
CHECK(EmitTableSet(f));
// Memory-related operators
case uint16_t(Op::I32Load):
CHECK(EmitLoad(f, ValType::I32, Scalar::Int32));
case uint16_t(Op::I64Load):
CHECK(EmitLoad(f, ValType::I64, Scalar::Int64));
case uint16_t(Op::F32Load):
CHECK(EmitLoad(f, ValType::F32, Scalar::Float32));
case uint16_t(Op::F64Load):
CHECK(EmitLoad(f, ValType::F64, Scalar::Float64));
case uint16_t(Op::I32Load8S):
CHECK(EmitLoad(f, ValType::I32, Scalar::Int8));
case uint16_t(Op::I32Load8U):
CHECK(EmitLoad(f, ValType::I32, Scalar::Uint8));
case uint16_t(Op::I32Load16S):
CHECK(EmitLoad(f, ValType::I32, Scalar::Int16));
case uint16_t(Op::I32Load16U):
CHECK(EmitLoad(f, ValType::I32, Scalar::Uint16));
case uint16_t(Op::I64Load8S):
CHECK(EmitLoad(f, ValType::I64, Scalar::Int8));
case uint16_t(Op::I64Load8U):
CHECK(EmitLoad(f, ValType::I64, Scalar::Uint8));
case uint16_t(Op::I64Load16S):
CHECK(EmitLoad(f, ValType::I64, Scalar::Int16));
case uint16_t(Op::I64Load16U):
CHECK(EmitLoad(f, ValType::I64, Scalar::Uint16));
case uint16_t(Op::I64Load32S):
CHECK(EmitLoad(f, ValType::I64, Scalar::Int32));
case uint16_t(Op::I64Load32U):
CHECK(EmitLoad(f, ValType::I64, Scalar::Uint32));
case uint16_t(Op::I32Store):
CHECK(EmitStore(f, ValType::I32, Scalar::Int32));
case uint16_t(Op::I64Store):
CHECK(EmitStore(f, ValType::I64, Scalar::Int64));
case uint16_t(Op::F32Store):
CHECK(EmitStore(f, ValType::F32, Scalar::Float32));
case uint16_t(Op::F64Store):
CHECK(EmitStore(f, ValType::F64, Scalar::Float64));
case uint16_t(Op::I32Store8):
CHECK(EmitStore(f, ValType::I32, Scalar::Int8));
case uint16_t(Op::I32Store16):
CHECK(EmitStore(f, ValType::I32, Scalar::Int16));
case uint16_t(Op::I64Store8):
CHECK(EmitStore(f, ValType::I64, Scalar::Int8));
case uint16_t(Op::I64Store16):
CHECK(EmitStore(f, ValType::I64, Scalar::Int16));
case uint16_t(Op::I64Store32):
CHECK(EmitStore(f, ValType::I64, Scalar::Int32));
case uint16_t(Op::MemorySize):
CHECK(EmitMemorySize(f));
case uint16_t(Op::MemoryGrow):
CHECK(EmitMemoryGrow(f));
// Constants
case uint16_t(Op::I32Const):
CHECK(EmitI32Const(f));
case uint16_t(Op::I64Const):
CHECK(EmitI64Const(f));
case uint16_t(Op::F32Const):
CHECK(EmitF32Const(f));
case uint16_t(Op::F64Const):
CHECK(EmitF64Const(f));
// Comparison operators
case uint16_t(Op::I32Eqz):
CHECK(EmitConversion<MNot>(f, ValType::I32, ValType::I32));
case uint16_t(Op::I32Eq):
CHECK(
EmitComparison(f, ValType::I32, JSOp::Eq, MCompare::Compare_Int32));
case uint16_t(Op::I32Ne):
CHECK(
EmitComparison(f, ValType::I32, JSOp::Ne, MCompare::Compare_Int32));
case uint16_t(Op::I32LtS):
CHECK(
EmitComparison(f, ValType::I32, JSOp::Lt, MCompare::Compare_Int32));
case uint16_t(Op::I32LtU):
CHECK(EmitComparison(f, ValType::I32, JSOp::Lt,
MCompare::Compare_UInt32));
case uint16_t(Op::I32GtS):
CHECK(
EmitComparison(f, ValType::I32, JSOp::Gt, MCompare::Compare_Int32));
case uint16_t(Op::I32GtU):
CHECK(EmitComparison(f, ValType::I32, JSOp::Gt,
MCompare::Compare_UInt32));
case uint16_t(Op::I32LeS):
CHECK(
EmitComparison(f, ValType::I32, JSOp::Le, MCompare::Compare_Int32));
case uint16_t(Op::I32LeU):
CHECK(EmitComparison(f, ValType::I32, JSOp::Le,
MCompare::Compare_UInt32));
case uint16_t(Op::I32GeS):
CHECK(
EmitComparison(f, ValType::I32, JSOp::Ge, MCompare::Compare_Int32));
case uint16_t(Op::I32GeU):
CHECK(EmitComparison(f, ValType::I32, JSOp::Ge,
MCompare::Compare_UInt32));
case uint16_t(Op::I64Eqz):
CHECK(EmitConversion<MNot>(f, ValType::I64, ValType::I32));
case uint16_t(Op::I64Eq):
CHECK(
EmitComparison(f, ValType::I64, JSOp::Eq, MCompare::Compare_Int64));
case uint16_t(Op::I64Ne):
CHECK(
EmitComparison(f, ValType::I64, JSOp::Ne, MCompare::Compare_Int64));
case uint16_t(Op::I64LtS):
CHECK(
EmitComparison(f, ValType::I64, JSOp::Lt, MCompare::Compare_Int64));
case uint16_t(Op::I64LtU):
CHECK(EmitComparison(f, ValType::I64, JSOp::Lt,
MCompare::Compare_UInt64));
case uint16_t(Op::I64GtS):
CHECK(
EmitComparison(f, ValType::I64, JSOp::Gt, MCompare::Compare_Int64));
case uint16_t(Op::I64GtU):
CHECK(EmitComparison(f, ValType::I64, JSOp::Gt,
MCompare::Compare_UInt64));
case uint16_t(Op::I64LeS):
CHECK(
EmitComparison(f, ValType::I64, JSOp::Le, MCompare::Compare_Int64));
case uint16_t(Op::I64LeU):
CHECK(EmitComparison(f, ValType::I64, JSOp::Le,
MCompare::Compare_UInt64));
case uint16_t(Op::I64GeS):
CHECK(
EmitComparison(f, ValType::I64, JSOp::Ge, MCompare::Compare_Int64));
case uint16_t(Op::I64GeU):
CHECK(EmitComparison(f, ValType::I64, JSOp::Ge,
MCompare::Compare_UInt64));
case uint16_t(Op::F32Eq):
CHECK(EmitComparison(f, ValType::F32, JSOp::Eq,
MCompare::Compare_Float32));
case uint16_t(Op::F32Ne):
CHECK(EmitComparison(f, ValType::F32, JSOp::Ne,
MCompare::Compare_Float32));
case uint16_t(Op::F32Lt):
CHECK(EmitComparison(f, ValType::F32, JSOp::Lt,
MCompare::Compare_Float32));
case uint16_t(Op::F32Gt):
CHECK(EmitComparison(f, ValType::F32, JSOp::Gt,
MCompare::Compare_Float32));
case uint16_t(Op::F32Le):
CHECK(EmitComparison(f, ValType::F32, JSOp::Le,
MCompare::Compare_Float32));
case uint16_t(Op::F32Ge):
CHECK(EmitComparison(f, ValType::F32, JSOp::Ge,
MCompare::Compare_Float32));
case uint16_t(Op::F64Eq):
CHECK(EmitComparison(f, ValType::F64, JSOp::Eq,
MCompare::Compare_Double));
case uint16_t(Op::F64Ne):
CHECK(EmitComparison(f, ValType::F64, JSOp::Ne,
MCompare::Compare_Double));
case uint16_t(Op::F64Lt):
CHECK(EmitComparison(f, ValType::F64, JSOp::Lt,
MCompare::Compare_Double));
case uint16_t(Op::F64Gt):
CHECK(EmitComparison(f, ValType::F64, JSOp::Gt,
MCompare::Compare_Double));
case uint16_t(Op::F64Le):
CHECK(EmitComparison(f, ValType::F64, JSOp::Le,
MCompare::Compare_Double));
case uint16_t(Op::F64Ge):
CHECK(EmitComparison(f, ValType::F64, JSOp::Ge,
MCompare::Compare_Double));
// Numeric operators
case uint16_t(Op::I32Clz):
CHECK(EmitUnaryWithType<MClz>(f, ValType::I32, MIRType::Int32));
case uint16_t(Op::I32Ctz):
CHECK(EmitUnaryWithType<MCtz>(f, ValType::I32, MIRType::Int32));
case uint16_t(Op::I32Popcnt):
CHECK(EmitUnaryWithType<MPopcnt>(f, ValType::I32, MIRType::Int32));
case uint16_t(Op::I32Add):
CHECK(EmitAdd(f, ValType::I32, MIRType::Int32));
case uint16_t(Op::I32Sub):
CHECK(EmitSub(f, ValType::I32, MIRType::Int32));
case uint16_t(Op::I32Mul):
CHECK(EmitMul(f, ValType::I32, MIRType::Int32));
case uint16_t(Op::I32DivS):
case uint16_t(Op::I32DivU):
CHECK(
EmitDiv(f, ValType::I32, MIRType::Int32, Op(op.b0) == Op::I32DivU));
case uint16_t(Op::I32RemS):
case uint16_t(Op::I32RemU):
CHECK(
EmitRem(f, ValType::I32, MIRType::Int32, Op(op.b0) == Op::I32RemU));
case uint16_t(Op::I32And):
CHECK(EmitBitwiseAndOrXor(f, ValType::I32, MIRType::Int32,
MWasmBinaryBitwise::SubOpcode::And));
case uint16_t(Op::I32Or):
CHECK(EmitBitwiseAndOrXor(f, ValType::I32, MIRType::Int32,
MWasmBinaryBitwise::SubOpcode::Or));
case uint16_t(Op::I32Xor):
CHECK(EmitBitwiseAndOrXor(f, ValType::I32, MIRType::Int32,
MWasmBinaryBitwise::SubOpcode::Xor));
case uint16_t(Op::I32Shl):
CHECK(EmitShift<MLsh>(f, ValType::I32, MIRType::Int32));
case uint16_t(Op::I32ShrS):
CHECK(EmitShift<MRsh>(f, ValType::I32, MIRType::Int32));
case uint16_t(Op::I32ShrU):
CHECK(EmitUrsh(f, ValType::I32, MIRType::Int32));
case uint16_t(Op::I32Rotl):
case uint16_t(Op::I32Rotr):
CHECK(EmitRotate(f, ValType::I32, Op(op.b0) == Op::I32Rotl));
case uint16_t(Op::I64Clz):
CHECK(EmitUnaryWithType<MClz>(f, ValType::I64, MIRType::Int64));
case uint16_t(Op::I64Ctz):
CHECK(EmitUnaryWithType<MCtz>(f, ValType::I64, MIRType::Int64));
case uint16_t(Op::I64Popcnt):
CHECK(EmitUnaryWithType<MPopcnt>(f, ValType::I64, MIRType::Int64));
case uint16_t(Op::I64Add):
CHECK(EmitAdd(f, ValType::I64, MIRType::Int64));
case uint16_t(Op::I64Sub):
CHECK(EmitSub(f, ValType::I64, MIRType::Int64));
case uint16_t(Op::I64Mul):
CHECK(EmitMul(f, ValType::I64, MIRType::Int64));
case uint16_t(Op::I64DivS):
case uint16_t(Op::I64DivU):
CHECK(
EmitDiv(f, ValType::I64, MIRType::Int64, Op(op.b0) == Op::I64DivU));
case uint16_t(Op::I64RemS):
case uint16_t(Op::I64RemU):
CHECK(
EmitRem(f, ValType::I64, MIRType::Int64, Op(op.b0) == Op::I64RemU));
case uint16_t(Op::I64And):
CHECK(EmitBitwiseAndOrXor(f, ValType::I64, MIRType::Int64,
MWasmBinaryBitwise::SubOpcode::And));
case uint16_t(Op::I64Or):
CHECK(EmitBitwiseAndOrXor(f, ValType::I64, MIRType::Int64,
MWasmBinaryBitwise::SubOpcode::Or));
case uint16_t(Op::I64Xor):
CHECK(EmitBitwiseAndOrXor(f, ValType::I64, MIRType::Int64,
MWasmBinaryBitwise::SubOpcode::Xor));
case uint16_t(Op::I64Shl):
CHECK(EmitShift<MLsh>(f, ValType::I64, MIRType::Int64));
case uint16_t(Op::I64ShrS):
CHECK(EmitShift<MRsh>(f, ValType::I64, MIRType::Int64));
case uint16_t(Op::I64ShrU):
CHECK(EmitUrsh(f, ValType::I64, MIRType::Int64));
case uint16_t(Op::I64Rotl):
case uint16_t(Op::I64Rotr):
CHECK(EmitRotate(f, ValType::I64, Op(op.b0) == Op::I64Rotl));
case uint16_t(Op::F32Abs):
CHECK(EmitUnaryWithType<MAbs>(f, ValType::F32, MIRType::Float32));
case uint16_t(Op::F32Neg):
CHECK(EmitUnaryWithType<MWasmNeg>(f, ValType::F32, MIRType::Float32));
case uint16_t(Op::F32Ceil):
CHECK(EmitUnaryMathBuiltinCall(f, SASigCeilF));
case uint16_t(Op::F32Floor):
CHECK(EmitUnaryMathBuiltinCall(f, SASigFloorF));
case uint16_t(Op::F32Trunc):
CHECK(EmitUnaryMathBuiltinCall(f, SASigTruncF));
case uint16_t(Op::F32Nearest):
CHECK(EmitUnaryMathBuiltinCall(f, SASigNearbyIntF));
case uint16_t(Op::F32Sqrt):
CHECK(EmitUnaryWithType<MSqrt>(f, ValType::F32, MIRType::Float32));
case uint16_t(Op::F32Add):
CHECK(EmitAdd(f, ValType::F32, MIRType::Float32));
case uint16_t(Op::F32Sub):
CHECK(EmitSub(f, ValType::F32, MIRType::Float32));
case uint16_t(Op::F32Mul):
CHECK(EmitMul(f, ValType::F32, MIRType::Float32));
case uint16_t(Op::F32Div):
CHECK(EmitDiv(f, ValType::F32, MIRType::Float32,
/* isUnsigned = */ false));
case uint16_t(Op::F32Min):
case uint16_t(Op::F32Max):
CHECK(EmitMinMax(f, ValType::F32, MIRType::Float32,
Op(op.b0) == Op::F32Max));
case uint16_t(Op::F32CopySign):
CHECK(EmitCopySign(f, ValType::F32));
case uint16_t(Op::F64Abs):
CHECK(EmitUnaryWithType<MAbs>(f, ValType::F64, MIRType::Double));
case uint16_t(Op::F64Neg):
CHECK(EmitUnaryWithType<MWasmNeg>(f, ValType::F64, MIRType::Double));
case uint16_t(Op::F64Ceil):
CHECK(EmitUnaryMathBuiltinCall(f, SASigCeilD));
case uint16_t(Op::F64Floor):
CHECK(EmitUnaryMathBuiltinCall(f, SASigFloorD));
case uint16_t(Op::F64Trunc):
CHECK(EmitUnaryMathBuiltinCall(f, SASigTruncD));
case uint16_t(Op::F64Nearest):
CHECK(EmitUnaryMathBuiltinCall(f, SASigNearbyIntD));
case uint16_t(Op::F64Sqrt):
CHECK(EmitUnaryWithType<MSqrt>(f, ValType::F64, MIRType::Double));
case uint16_t(Op::F64Add):
CHECK(EmitAdd(f, ValType::F64, MIRType::Double));
case uint16_t(Op::F64Sub):
CHECK(EmitSub(f, ValType::F64, MIRType::Double));
case uint16_t(Op::F64Mul):
CHECK(EmitMul(f, ValType::F64, MIRType::Double));
case uint16_t(Op::F64Div):
CHECK(EmitDiv(f, ValType::F64, MIRType::Double,
/* isUnsigned = */ false));
case uint16_t(Op::F64Min):
case uint16_t(Op::F64Max):
CHECK(EmitMinMax(f, ValType::F64, MIRType::Double,
Op(op.b0) == Op::F64Max));
case uint16_t(Op::F64CopySign):
CHECK(EmitCopySign(f, ValType::F64));
// Conversions
case uint16_t(Op::I32WrapI64):
CHECK(EmitConversion<MWrapInt64ToInt32>(f, ValType::I64, ValType::I32));
case uint16_t(Op::I32TruncF32S):
case uint16_t(Op::I32TruncF32U):
CHECK(EmitTruncate(f, ValType::F32, ValType::I32,
Op(op.b0) == Op::I32TruncF32U, false));
case uint16_t(Op::I32TruncF64S):
case uint16_t(Op::I32TruncF64U):
CHECK(EmitTruncate(f, ValType::F64, ValType::I32,
Op(op.b0) == Op::I32TruncF64U, false));
case uint16_t(Op::I64ExtendI32S):
case uint16_t(Op::I64ExtendI32U):
CHECK(EmitExtendI32(f, Op(op.b0) == Op::I64ExtendI32U));
case uint16_t(Op::I64TruncF32S):
case uint16_t(Op::I64TruncF32U):
CHECK(EmitTruncate(f, ValType::F32, ValType::I64,
Op(op.b0) == Op::I64TruncF32U, false));
case uint16_t(Op::I64TruncF64S):
case uint16_t(Op::I64TruncF64U):
CHECK(EmitTruncate(f, ValType::F64, ValType::I64,
Op(op.b0) == Op::I64TruncF64U, false));
case uint16_t(Op::F32ConvertI32S):
CHECK(EmitConversion<MToFloat32>(f, ValType::I32, ValType::F32));
case uint16_t(Op::F32ConvertI32U):
CHECK(EmitConversion<MWasmUnsignedToFloat32>(f, ValType::I32,
ValType::F32));
case uint16_t(Op::F32ConvertI64S):
case uint16_t(Op::F32ConvertI64U):
CHECK(EmitConvertI64ToFloatingPoint(f, ValType::F32, MIRType::Float32,
Op(op.b0) == Op::F32ConvertI64U));
case uint16_t(Op::F32DemoteF64):
CHECK(EmitConversion<MToFloat32>(f, ValType::F64, ValType::F32));
case uint16_t(Op::F64ConvertI32S):
CHECK(EmitConversion<MToDouble>(f, ValType::I32, ValType::F64));
case uint16_t(Op::F64ConvertI32U):
CHECK(EmitConversion<MWasmUnsignedToDouble>(f, ValType::I32,
ValType::F64));
case uint16_t(Op::F64ConvertI64S):
case uint16_t(Op::F64ConvertI64U):
CHECK(EmitConvertI64ToFloatingPoint(f, ValType::F64, MIRType::Double,
Op(op.b0) == Op::F64ConvertI64U));
case uint16_t(Op::F64PromoteF32):
CHECK(EmitConversion<MToDouble>(f, ValType::F32, ValType::F64));
// Reinterpretations
case uint16_t(Op::I32ReinterpretF32):
CHECK(EmitReinterpret(f, ValType::I32, ValType::F32, MIRType::Int32));
case uint16_t(Op::I64ReinterpretF64):
CHECK(EmitReinterpret(f, ValType::I64, ValType::F64, MIRType::Int64));
case uint16_t(Op::F32ReinterpretI32):
CHECK(EmitReinterpret(f, ValType::F32, ValType::I32, MIRType::Float32));
case uint16_t(Op::F64ReinterpretI64):
CHECK(EmitReinterpret(f, ValType::F64, ValType::I64, MIRType::Double));
#ifdef ENABLE_WASM_GC
case uint16_t(Op::RefEq):
if (!f.moduleEnv().gcEnabled()) {
return f.iter().unrecognizedOpcode(&op);
}
CHECK(EmitComparison(f, RefType::eq(), JSOp::Eq,
MCompare::Compare_WasmAnyRef));
#endif
case uint16_t(Op::RefFunc):
CHECK(EmitRefFunc(f));
case uint16_t(Op::RefNull):
CHECK(EmitRefNull(f));
case uint16_t(Op::RefIsNull):
CHECK(EmitRefIsNull(f));
// Sign extensions
case uint16_t(Op::I32Extend8S):
CHECK(EmitSignExtend(f, 1, 4));
case uint16_t(Op::I32Extend16S):
CHECK(EmitSignExtend(f, 2, 4));
case uint16_t(Op::I64Extend8S):
CHECK(EmitSignExtend(f, 1, 8));
case uint16_t(Op::I64Extend16S):
CHECK(EmitSignExtend(f, 2, 8));
case uint16_t(Op::I64Extend32S):
CHECK(EmitSignExtend(f, 4, 8));
#ifdef ENABLE_WASM_TAIL_CALLS
case uint16_t(Op::ReturnCall): {
if (!f.moduleEnv().tailCallsEnabled()) {
return f.iter().unrecognizedOpcode(&op);
}
CHECK(EmitReturnCall(f));
}
case uint16_t(Op::ReturnCallIndirect): {
if (!f.moduleEnv().tailCallsEnabled()) {
return f.iter().unrecognizedOpcode(&op);
}
CHECK(EmitReturnCallIndirect(f));
}
#endif
#ifdef ENABLE_WASM_GC
case uint16_t(Op::RefAsNonNull):
if (!f.moduleEnv().gcEnabled()) {
return f.iter().unrecognizedOpcode(&op);
}
CHECK(EmitRefAsNonNull(f));
case uint16_t(Op::BrOnNull): {
if (!f.moduleEnv().gcEnabled()) {
return f.iter().unrecognizedOpcode(&op);
}
CHECK(EmitBrOnNull(f));
}
case uint16_t(Op::BrOnNonNull): {
if (!f.moduleEnv().gcEnabled()) {
return f.iter().unrecognizedOpcode(&op);
}
CHECK(EmitBrOnNonNull(f));
}
case uint16_t(Op::CallRef): {
if (!f.moduleEnv().gcEnabled()) {
return f.iter().unrecognizedOpcode(&op);
}
CHECK(EmitCallRef(f));
}
#endif
#if defined(ENABLE_WASM_TAIL_CALLS) && defined(ENABLE_WASM_GC)
case uint16_t(Op::ReturnCallRef): {
if (!f.moduleEnv().gcEnabled() || !f.moduleEnv().tailCallsEnabled()) {
return f.iter().unrecognizedOpcode(&op);
}
CHECK(EmitReturnCallRef(f));
}
#endif
// Gc operations
#ifdef ENABLE_WASM_GC
case uint16_t(Op::GcPrefix): {
if (!f.moduleEnv().gcEnabled()) {
return f.iter().unrecognizedOpcode(&op);
}
switch (op.b1) {
case uint32_t(GcOp::StructNew):
CHECK(EmitStructNew(f));
case uint32_t(GcOp::StructNewDefault):
CHECK(EmitStructNewDefault(f));
case uint32_t(GcOp::StructSet):
CHECK(EmitStructSet(f));
case uint32_t(GcOp::StructGet):
CHECK(EmitStructGet(f, FieldWideningOp::None));
case uint32_t(GcOp::StructGetS):
CHECK(EmitStructGet(f, FieldWideningOp::Signed));
case uint32_t(GcOp::StructGetU):
CHECK(EmitStructGet(f, FieldWideningOp::Unsigned));
case uint32_t(GcOp::ArrayNew):
CHECK(EmitArrayNew(f));
case uint32_t(GcOp::ArrayNewDefault):
CHECK(EmitArrayNewDefault(f));
case uint32_t(GcOp::ArrayNewFixed):
CHECK(EmitArrayNewFixed(f));
case uint32_t(GcOp::ArrayNewData):
CHECK(EmitArrayNewData(f));
case uint32_t(GcOp::ArrayNewElem):
CHECK(EmitArrayNewElem(f));
case uint32_t(GcOp::ArrayInitData):
CHECK(EmitArrayInitData(f));
case uint32_t(GcOp::ArrayInitElem):
CHECK(EmitArrayInitElem(f));
case uint32_t(GcOp::ArraySet):
CHECK(EmitArraySet(f));
case uint32_t(GcOp::ArrayGet):
CHECK(EmitArrayGet(f, FieldWideningOp::None));
case uint32_t(GcOp::ArrayGetS):
CHECK(EmitArrayGet(f, FieldWideningOp::Signed));
case uint32_t(GcOp::ArrayGetU):
CHECK(EmitArrayGet(f, FieldWideningOp::Unsigned));
case uint32_t(GcOp::ArrayLen):
CHECK(EmitArrayLen(f));
case uint32_t(GcOp::ArrayCopy):
CHECK(EmitArrayCopy(f));
case uint32_t(GcOp::ArrayFill):
CHECK(EmitArrayFill(f));
case uint32_t(GcOp::RefI31):
CHECK(EmitRefI31(f));
case uint32_t(GcOp::I31GetS):
CHECK(EmitI31Get(f, FieldWideningOp::Signed));
case uint32_t(GcOp::I31GetU):
CHECK(EmitI31Get(f, FieldWideningOp::Unsigned));
case uint32_t(GcOp::BrOnCast):
CHECK(EmitBrOnCast(f, /*onSuccess=*/true));
case uint32_t(GcOp::BrOnCastFail):
CHECK(EmitBrOnCast(f, /*onSuccess=*/false));
case uint32_t(GcOp::RefTest):
CHECK(EmitRefTest(f, /*nullable=*/false));
case uint32_t(GcOp::RefTestNull):
CHECK(EmitRefTest(f, /*nullable=*/true));
case uint32_t(GcOp::RefCast):
CHECK(EmitRefCast(f, /*nullable=*/false));
case uint32_t(GcOp::RefCastNull):
CHECK(EmitRefCast(f, /*nullable=*/true));
case uint16_t(GcOp::AnyConvertExtern):
CHECK(EmitAnyConvertExtern(f));
case uint16_t(GcOp::ExternConvertAny):
CHECK(EmitExternConvertAny(f));
default:
return f.iter().unrecognizedOpcode(&op);
} // switch (op.b1)
break;
}
#endif
// SIMD operations
#ifdef ENABLE_WASM_SIMD
case uint16_t(Op::SimdPrefix): {
if (!f.moduleEnv().simdAvailable()) {
return f.iter().unrecognizedOpcode(&op);
}
switch (op.b1) {
case uint32_t(SimdOp::V128Const):
CHECK(EmitConstSimd128(f));
case uint32_t(SimdOp::V128Load):
CHECK(EmitLoad(f, ValType::V128, Scalar::Simd128));
case uint32_t(SimdOp::V128Store):
CHECK(EmitStore(f, ValType::V128, Scalar::Simd128));
case uint32_t(SimdOp::V128And):
case uint32_t(SimdOp::V128Or):
case uint32_t(SimdOp::V128Xor):
case uint32_t(SimdOp::I8x16AvgrU):
case uint32_t(SimdOp::I16x8AvgrU):
case uint32_t(SimdOp::I8x16Add):
case uint32_t(SimdOp::I8x16AddSatS):
case uint32_t(SimdOp::I8x16AddSatU):
case uint32_t(SimdOp::I8x16MinS):
case uint32_t(SimdOp::I8x16MinU):
case uint32_t(SimdOp::I8x16MaxS):
case uint32_t(SimdOp::I8x16MaxU):
case uint32_t(SimdOp::I16x8Add):
case uint32_t(SimdOp::I16x8AddSatS):
case uint32_t(SimdOp::I16x8AddSatU):
case uint32_t(SimdOp::I16x8Mul):
case uint32_t(SimdOp::I16x8MinS):
case uint32_t(SimdOp::I16x8MinU):
case uint32_t(SimdOp::I16x8MaxS):
case uint32_t(SimdOp::I16x8MaxU):
case uint32_t(SimdOp::I32x4Add):
case uint32_t(SimdOp::I32x4Mul):
case uint32_t(SimdOp::I32x4MinS):
case uint32_t(SimdOp::I32x4MinU):
case uint32_t(SimdOp::I32x4MaxS):
case uint32_t(SimdOp::I32x4MaxU):
case uint32_t(SimdOp::I64x2Add):
case uint32_t(SimdOp::I64x2Mul):
case uint32_t(SimdOp::F32x4Add):
case uint32_t(SimdOp::F32x4Mul):
case uint32_t(SimdOp::F32x4Min):
case uint32_t(SimdOp::F32x4Max):
case uint32_t(SimdOp::F64x2Add):
case uint32_t(SimdOp::F64x2Mul):
case uint32_t(SimdOp::F64x2Min):
case uint32_t(SimdOp::F64x2Max):
case uint32_t(SimdOp::I8x16Eq):
case uint32_t(SimdOp::I8x16Ne):
case uint32_t(SimdOp::I16x8Eq):
case uint32_t(SimdOp::I16x8Ne):
case uint32_t(SimdOp::I32x4Eq):
case uint32_t(SimdOp::I32x4Ne):
case uint32_t(SimdOp::I64x2Eq):
case uint32_t(SimdOp::I64x2Ne):
case uint32_t(SimdOp::F32x4Eq):
case uint32_t(SimdOp::F32x4Ne):
case uint32_t(SimdOp::F64x2Eq):
case uint32_t(SimdOp::F64x2Ne):
case uint32_t(SimdOp::I32x4DotI16x8S):
case uint32_t(SimdOp::I16x8ExtmulLowI8x16S):
case uint32_t(SimdOp::I16x8ExtmulHighI8x16S):
case uint32_t(SimdOp::I16x8ExtmulLowI8x16U):
case uint32_t(SimdOp::I16x8ExtmulHighI8x16U):
case uint32_t(SimdOp::I32x4ExtmulLowI16x8S):
case uint32_t(SimdOp::I32x4ExtmulHighI16x8S):
case uint32_t(SimdOp::I32x4ExtmulLowI16x8U):
case uint32_t(SimdOp::I32x4ExtmulHighI16x8U):
case uint32_t(SimdOp::I64x2ExtmulLowI32x4S):
case uint32_t(SimdOp::I64x2ExtmulHighI32x4S):
case uint32_t(SimdOp::I64x2ExtmulLowI32x4U):
case uint32_t(SimdOp::I64x2ExtmulHighI32x4U):
case uint32_t(SimdOp::I16x8Q15MulrSatS):
CHECK(EmitBinarySimd128(f, /* commutative= */ true, SimdOp(op.b1)));
case uint32_t(SimdOp::V128AndNot):
case uint32_t(SimdOp::I8x16Sub):
case uint32_t(SimdOp::I8x16SubSatS):
case uint32_t(SimdOp::I8x16SubSatU):
case uint32_t(SimdOp::I16x8Sub):
case uint32_t(SimdOp::I16x8SubSatS):
case uint32_t(SimdOp::I16x8SubSatU):
case uint32_t(SimdOp::I32x4Sub):
case uint32_t(SimdOp::I64x2Sub):
case uint32_t(SimdOp::F32x4Sub):
case uint32_t(SimdOp::F32x4Div):
case uint32_t(SimdOp::F64x2Sub):
case uint32_t(SimdOp::F64x2Div):
case uint32_t(SimdOp::I8x16NarrowI16x8S):
case uint32_t(SimdOp::I8x16NarrowI16x8U):
case uint32_t(SimdOp::I16x8NarrowI32x4S):
case uint32_t(SimdOp::I16x8NarrowI32x4U):
case uint32_t(SimdOp::I8x16LtS):
case uint32_t(SimdOp::I8x16LtU):
case uint32_t(SimdOp::I8x16GtS):
case uint32_t(SimdOp::I8x16GtU):
case uint32_t(SimdOp::I8x16LeS):
case uint32_t(SimdOp::I8x16LeU):
case uint32_t(SimdOp::I8x16GeS):
case uint32_t(SimdOp::I8x16GeU):
case uint32_t(SimdOp::I16x8LtS):
case uint32_t(SimdOp::I16x8LtU):
case uint32_t(SimdOp::I16x8GtS):
case uint32_t(SimdOp::I16x8GtU):
case uint32_t(SimdOp::I16x8LeS):
case uint32_t(SimdOp::I16x8LeU):
case uint32_t(SimdOp::I16x8GeS):
case uint32_t(SimdOp::I16x8GeU):
case uint32_t(SimdOp::I32x4LtS):
case uint32_t(SimdOp::I32x4LtU):
case uint32_t(SimdOp::I32x4GtS):
case uint32_t(SimdOp::I32x4GtU):
case uint32_t(SimdOp::I32x4LeS):
case uint32_t(SimdOp::I32x4LeU):
case uint32_t(SimdOp::I32x4GeS):
case uint32_t(SimdOp::I32x4GeU):
case uint32_t(SimdOp::I64x2LtS):
case uint32_t(SimdOp::I64x2GtS):
case uint32_t(SimdOp::I64x2LeS):
case uint32_t(SimdOp::I64x2GeS):
case uint32_t(SimdOp::F32x4Lt):
case uint32_t(SimdOp::F32x4Gt):
case uint32_t(SimdOp::F32x4Le):
case uint32_t(SimdOp::F32x4Ge):
case uint32_t(SimdOp::F64x2Lt):
case uint32_t(SimdOp::F64x2Gt):
case uint32_t(SimdOp::F64x2Le):
case uint32_t(SimdOp::F64x2Ge):
case uint32_t(SimdOp::I8x16Swizzle):
case uint32_t(SimdOp::F32x4PMax):
case uint32_t(SimdOp::F32x4PMin):
case uint32_t(SimdOp::F64x2PMax):
case uint32_t(SimdOp::F64x2PMin):
CHECK(
EmitBinarySimd128(f, /* commutative= */ false, SimdOp(op.b1)));
case uint32_t(SimdOp::I8x16Splat):
case uint32_t(SimdOp::I16x8Splat):
case uint32_t(SimdOp::I32x4Splat):
CHECK(EmitSplatSimd128(f, ValType::I32, SimdOp(op.b1)));
case uint32_t(SimdOp::I64x2Splat):
CHECK(EmitSplatSimd128(f, ValType::I64, SimdOp(op.b1)));
case uint32_t(SimdOp::F32x4Splat):
CHECK(EmitSplatSimd128(f, ValType::F32, SimdOp(op.b1)));
case uint32_t(SimdOp::F64x2Splat):
CHECK(EmitSplatSimd128(f, ValType::F64, SimdOp(op.b1)));
case uint32_t(SimdOp::I8x16Neg):
case uint32_t(SimdOp::I16x8Neg):
case uint32_t(SimdOp::I16x8ExtendLowI8x16S):
case uint32_t(SimdOp::I16x8ExtendHighI8x16S):
case uint32_t(SimdOp::I16x8ExtendLowI8x16U):
case uint32_t(SimdOp::I16x8ExtendHighI8x16U):
case uint32_t(SimdOp::I32x4Neg):
case uint32_t(SimdOp::I32x4ExtendLowI16x8S):
case uint32_t(SimdOp::I32x4ExtendHighI16x8S):
case uint32_t(SimdOp::I32x4ExtendLowI16x8U):
case uint32_t(SimdOp::I32x4ExtendHighI16x8U):
case uint32_t(SimdOp::I32x4TruncSatF32x4S):
case uint32_t(SimdOp::I32x4TruncSatF32x4U):
case uint32_t(SimdOp::I64x2Neg):
case uint32_t(SimdOp::I64x2ExtendLowI32x4S):
case uint32_t(SimdOp::I64x2ExtendHighI32x4S):
case uint32_t(SimdOp::I64x2ExtendLowI32x4U):
case uint32_t(SimdOp::I64x2ExtendHighI32x4U):
case uint32_t(SimdOp::F32x4Abs):
case uint32_t(SimdOp::F32x4Neg):
case uint32_t(SimdOp::F32x4Sqrt):
case uint32_t(SimdOp::F32x4ConvertI32x4S):
case uint32_t(SimdOp::F32x4ConvertI32x4U):
case uint32_t(SimdOp::F64x2Abs):
case uint32_t(SimdOp::F64x2Neg):
case uint32_t(SimdOp::F64x2Sqrt):
case uint32_t(SimdOp::V128Not):
case uint32_t(SimdOp::I8x16Popcnt):
case uint32_t(SimdOp::I8x16Abs):
case uint32_t(SimdOp::I16x8Abs):
case uint32_t(SimdOp::I32x4Abs):
case uint32_t(SimdOp::I64x2Abs):
case uint32_t(SimdOp::F32x4Ceil):
case uint32_t(SimdOp::F32x4Floor):
case uint32_t(SimdOp::F32x4Trunc):
case uint32_t(SimdOp::F32x4Nearest):
case uint32_t(SimdOp::F64x2Ceil):
case uint32_t(SimdOp::F64x2Floor):
case uint32_t(SimdOp::F64x2Trunc):
case uint32_t(SimdOp::F64x2Nearest):
case uint32_t(SimdOp::F32x4DemoteF64x2Zero):
case uint32_t(SimdOp::F64x2PromoteLowF32x4):
case uint32_t(SimdOp::F64x2ConvertLowI32x4S):
case uint32_t(SimdOp::F64x2ConvertLowI32x4U):
case uint32_t(SimdOp::I32x4TruncSatF64x2SZero):
case uint32_t(SimdOp::I32x4TruncSatF64x2UZero):
case uint32_t(SimdOp::I16x8ExtaddPairwiseI8x16S):
case uint32_t(SimdOp::I16x8ExtaddPairwiseI8x16U):
case uint32_t(SimdOp::I32x4ExtaddPairwiseI16x8S):
case uint32_t(SimdOp::I32x4ExtaddPairwiseI16x8U):
CHECK(EmitUnarySimd128(f, SimdOp(op.b1)));
case uint32_t(SimdOp::V128AnyTrue):
case uint32_t(SimdOp::I8x16AllTrue):
case uint32_t(SimdOp::I16x8AllTrue):
case uint32_t(SimdOp::I32x4AllTrue):
case uint32_t(SimdOp::I64x2AllTrue):
case uint32_t(SimdOp::I8x16Bitmask):
case uint32_t(SimdOp::I16x8Bitmask):
case uint32_t(SimdOp::I32x4Bitmask):
case uint32_t(SimdOp::I64x2Bitmask):
CHECK(EmitReduceSimd128(f, SimdOp(op.b1)));
case uint32_t(SimdOp::I8x16Shl):
case uint32_t(SimdOp::I8x16ShrS):
case uint32_t(SimdOp::I8x16ShrU):
case uint32_t(SimdOp::I16x8Shl):
case uint32_t(SimdOp::I16x8ShrS):
case uint32_t(SimdOp::I16x8ShrU):
case uint32_t(SimdOp::I32x4Shl):
case uint32_t(SimdOp::I32x4ShrS):
case uint32_t(SimdOp::I32x4ShrU):
case uint32_t(SimdOp::I64x2Shl):
case uint32_t(SimdOp::I64x2ShrS):
case uint32_t(SimdOp::I64x2ShrU):
CHECK(EmitShiftSimd128(f, SimdOp(op.b1)));
case uint32_t(SimdOp::I8x16ExtractLaneS):
case uint32_t(SimdOp::I8x16ExtractLaneU):
CHECK(EmitExtractLaneSimd128(f, ValType::I32, 16, SimdOp(op.b1)));
case uint32_t(SimdOp::I16x8ExtractLaneS):
case uint32_t(SimdOp::I16x8ExtractLaneU):
CHECK(EmitExtractLaneSimd128(f, ValType::I32, 8, SimdOp(op.b1)));
case uint32_t(SimdOp::I32x4ExtractLane):
CHECK(EmitExtractLaneSimd128(f, ValType::I32, 4, SimdOp(op.b1)));
case uint32_t(SimdOp::I64x2ExtractLane):
CHECK(EmitExtractLaneSimd128(f, ValType::I64, 2, SimdOp(op.b1)));
case uint32_t(SimdOp::F32x4ExtractLane):
CHECK(EmitExtractLaneSimd128(f, ValType::F32, 4, SimdOp(op.b1)));
case uint32_t(SimdOp::F64x2ExtractLane):
CHECK(EmitExtractLaneSimd128(f, ValType::F64, 2, SimdOp(op.b1)));
case uint32_t(SimdOp::I8x16ReplaceLane):
CHECK(EmitReplaceLaneSimd128(f, ValType::I32, 16, SimdOp(op.b1)));
case uint32_t(SimdOp::I16x8ReplaceLane):
CHECK(EmitReplaceLaneSimd128(f, ValType::I32, 8, SimdOp(op.b1)));
case uint32_t(SimdOp::I32x4ReplaceLane):
CHECK(EmitReplaceLaneSimd128(f, ValType::I32, 4, SimdOp(op.b1)));
case uint32_t(SimdOp::I64x2ReplaceLane):
CHECK(EmitReplaceLaneSimd128(f, ValType::I64, 2, SimdOp(op.b1)));
case uint32_t(SimdOp::F32x4ReplaceLane):
CHECK(EmitReplaceLaneSimd128(f, ValType::F32, 4, SimdOp(op.b1)));
case uint32_t(SimdOp::F64x2ReplaceLane):
CHECK(EmitReplaceLaneSimd128(f, ValType::F64, 2, SimdOp(op.b1)));
case uint32_t(SimdOp::V128Bitselect):
CHECK(EmitTernarySimd128(f, SimdOp(op.b1)));
case uint32_t(SimdOp::I8x16Shuffle):
CHECK(EmitShuffleSimd128(f));
case uint32_t(SimdOp::V128Load8Splat):
CHECK(EmitLoadSplatSimd128(f, Scalar::Uint8, SimdOp::I8x16Splat));
case uint32_t(SimdOp::V128Load16Splat):
CHECK(EmitLoadSplatSimd128(f, Scalar::Uint16, SimdOp::I16x8Splat));
case uint32_t(SimdOp::V128Load32Splat):
CHECK(EmitLoadSplatSimd128(f, Scalar::Float32, SimdOp::I32x4Splat));
case uint32_t(SimdOp::V128Load64Splat):
CHECK(EmitLoadSplatSimd128(f, Scalar::Float64, SimdOp::I64x2Splat));
case uint32_t(SimdOp::V128Load8x8S):
case uint32_t(SimdOp::V128Load8x8U):
case uint32_t(SimdOp::V128Load16x4S):
case uint32_t(SimdOp::V128Load16x4U):
case uint32_t(SimdOp::V128Load32x2S):
case uint32_t(SimdOp::V128Load32x2U):
CHECK(EmitLoadExtendSimd128(f, SimdOp(op.b1)));
case uint32_t(SimdOp::V128Load32Zero):
CHECK(EmitLoadZeroSimd128(f, Scalar::Float32, 4));
case uint32_t(SimdOp::V128Load64Zero):
CHECK(EmitLoadZeroSimd128(f, Scalar::Float64, 8));
case uint32_t(SimdOp::V128Load8Lane):
CHECK(EmitLoadLaneSimd128(f, 1));
case uint32_t(SimdOp::V128Load16Lane):
CHECK(EmitLoadLaneSimd128(f, 2));
case uint32_t(SimdOp::V128Load32Lane):
CHECK(EmitLoadLaneSimd128(f, 4));
case uint32_t(SimdOp::V128Load64Lane):
CHECK(EmitLoadLaneSimd128(f, 8));
case uint32_t(SimdOp::V128Store8Lane):
CHECK(EmitStoreLaneSimd128(f, 1));
case uint32_t(SimdOp::V128Store16Lane):
CHECK(EmitStoreLaneSimd128(f, 2));
case uint32_t(SimdOp::V128Store32Lane):
CHECK(EmitStoreLaneSimd128(f, 4));
case uint32_t(SimdOp::V128Store64Lane):
CHECK(EmitStoreLaneSimd128(f, 8));
# ifdef ENABLE_WASM_RELAXED_SIMD
case uint32_t(SimdOp::F32x4RelaxedMadd):
case uint32_t(SimdOp::F32x4RelaxedNmadd):
case uint32_t(SimdOp::F64x2RelaxedMadd):
case uint32_t(SimdOp::F64x2RelaxedNmadd):
case uint32_t(SimdOp::I8x16RelaxedLaneSelect):
case uint32_t(SimdOp::I16x8RelaxedLaneSelect):
case uint32_t(SimdOp::I32x4RelaxedLaneSelect):
case uint32_t(SimdOp::I64x2RelaxedLaneSelect):
case uint32_t(SimdOp::I32x4DotI8x16I7x16AddS): {
if (!f.moduleEnv().v128RelaxedEnabled()) {
return f.iter().unrecognizedOpcode(&op);
}
CHECK(EmitTernarySimd128(f, SimdOp(op.b1)));
}
case uint32_t(SimdOp::F32x4RelaxedMin):
case uint32_t(SimdOp::F32x4RelaxedMax):
case uint32_t(SimdOp::F64x2RelaxedMin):
case uint32_t(SimdOp::F64x2RelaxedMax):
case uint32_t(SimdOp::I16x8RelaxedQ15MulrS): {
if (!f.moduleEnv().v128RelaxedEnabled()) {
return f.iter().unrecognizedOpcode(&op);
}
CHECK(EmitBinarySimd128(f, /* commutative= */ true, SimdOp(op.b1)));
}
case uint32_t(SimdOp::I32x4RelaxedTruncF32x4S):
case uint32_t(SimdOp::I32x4RelaxedTruncF32x4U):
case uint32_t(SimdOp::I32x4RelaxedTruncF64x2SZero):
case uint32_t(SimdOp::I32x4RelaxedTruncF64x2UZero): {
if (!f.moduleEnv().v128RelaxedEnabled()) {
return f.iter().unrecognizedOpcode(&op);
}
CHECK(EmitUnarySimd128(f, SimdOp(op.b1)));
}
case uint32_t(SimdOp::I8x16RelaxedSwizzle):
case uint32_t(SimdOp::I16x8DotI8x16I7x16S): {
if (!f.moduleEnv().v128RelaxedEnabled()) {
return f.iter().unrecognizedOpcode(&op);
}
CHECK(
EmitBinarySimd128(f, /* commutative= */ false, SimdOp(op.b1)));
}
# endif
default:
return f.iter().unrecognizedOpcode(&op);
} // switch (op.b1)
break;
}
#endif
// Miscellaneous operations
case uint16_t(Op::MiscPrefix): {
switch (op.b1) {
case uint32_t(MiscOp::I32TruncSatF32S):
case uint32_t(MiscOp::I32TruncSatF32U):
CHECK(EmitTruncate(f, ValType::F32, ValType::I32,
MiscOp(op.b1) == MiscOp::I32TruncSatF32U, true));
case uint32_t(MiscOp::I32TruncSatF64S):
case uint32_t(MiscOp::I32TruncSatF64U):
CHECK(EmitTruncate(f, ValType::F64, ValType::I32,
MiscOp(op.b1) == MiscOp::I32TruncSatF64U, true));
case uint32_t(MiscOp::I64TruncSatF32S):
case uint32_t(MiscOp::I64TruncSatF32U):
CHECK(EmitTruncate(f, ValType::F32, ValType::I64,
MiscOp(op.b1) == MiscOp::I64TruncSatF32U, true));
case uint32_t(MiscOp::I64TruncSatF64S):
case uint32_t(MiscOp::I64TruncSatF64U):
CHECK(EmitTruncate(f, ValType::F64, ValType::I64,
MiscOp(op.b1) == MiscOp::I64TruncSatF64U, true));
case uint32_t(MiscOp::MemoryCopy):
CHECK(EmitMemCopy(f));
case uint32_t(MiscOp::DataDrop):
CHECK(EmitDataOrElemDrop(f, /*isData=*/true));
case uint32_t(MiscOp::MemoryFill):
CHECK(EmitMemFill(f));
case uint32_t(MiscOp::MemoryInit):
CHECK(EmitMemOrTableInit(f, /*isMem=*/true));
case uint32_t(MiscOp::TableCopy):
CHECK(EmitTableCopy(f));
case uint32_t(MiscOp::ElemDrop):
CHECK(EmitDataOrElemDrop(f, /*isData=*/false));
case uint32_t(MiscOp::TableInit):
CHECK(EmitMemOrTableInit(f, /*isMem=*/false));
case uint32_t(MiscOp::TableFill):
CHECK(EmitTableFill(f));
#if ENABLE_WASM_MEMORY_CONTROL
case uint32_t(MiscOp::MemoryDiscard): {
if (!f.moduleEnv().memoryControlEnabled()) {
return f.iter().unrecognizedOpcode(&op);
}
CHECK(EmitMemDiscard(f));
}
#endif
case uint32_t(MiscOp::TableGrow):
CHECK(EmitTableGrow(f));
case uint32_t(MiscOp::TableSize):
CHECK(EmitTableSize(f));
default:
return f.iter().unrecognizedOpcode(&op);
}
break;
}
// Thread operations
case uint16_t(Op::ThreadPrefix): {
// Though thread ops can be used on nonshared memories, we make them
// unavailable if shared memory has been disabled in the prefs, for
// maximum predictability and safety and consistency with JS.
if (f.moduleEnv().sharedMemoryEnabled() == Shareable::False) {
return f.iter().unrecognizedOpcode(&op);
}
switch (op.b1) {
case uint32_t(ThreadOp::Wake):
CHECK(EmitWake(f));
case uint32_t(ThreadOp::I32Wait):
CHECK(EmitWait(f, ValType::I32, 4));
case uint32_t(ThreadOp::I64Wait):
CHECK(EmitWait(f, ValType::I64, 8));
case uint32_t(ThreadOp::Fence):
CHECK(EmitFence(f));
case uint32_t(ThreadOp::I32AtomicLoad):
CHECK(EmitAtomicLoad(f, ValType::I32, Scalar::Int32));
case uint32_t(ThreadOp::I64AtomicLoad):
CHECK(EmitAtomicLoad(f, ValType::I64, Scalar::Int64));
case uint32_t(ThreadOp::I32AtomicLoad8U):
CHECK(EmitAtomicLoad(f, ValType::I32, Scalar::Uint8));
case uint32_t(ThreadOp::I32AtomicLoad16U):
CHECK(EmitAtomicLoad(f, ValType::I32, Scalar::Uint16));
case uint32_t(ThreadOp::I64AtomicLoad8U):
CHECK(EmitAtomicLoad(f, ValType::I64, Scalar::Uint8));
case uint32_t(ThreadOp::I64AtomicLoad16U):
CHECK(EmitAtomicLoad(f, ValType::I64, Scalar::Uint16));
case uint32_t(ThreadOp::I64AtomicLoad32U):
CHECK(EmitAtomicLoad(f, ValType::I64, Scalar::Uint32));
case uint32_t(ThreadOp::I32AtomicStore):
CHECK(EmitAtomicStore(f, ValType::I32, Scalar::Int32));
case uint32_t(ThreadOp::I64AtomicStore):
CHECK(EmitAtomicStore(f, ValType::I64, Scalar::Int64));
case uint32_t(ThreadOp::I32AtomicStore8U):
CHECK(EmitAtomicStore(f, ValType::I32, Scalar::Uint8));
case uint32_t(ThreadOp::I32AtomicStore16U):
CHECK(EmitAtomicStore(f, ValType::I32, Scalar::Uint16));
case uint32_t(ThreadOp::I64AtomicStore8U):
CHECK(EmitAtomicStore(f, ValType::I64, Scalar::Uint8));
case uint32_t(ThreadOp::I64AtomicStore16U):
CHECK(EmitAtomicStore(f, ValType::I64, Scalar::Uint16));
case uint32_t(ThreadOp::I64AtomicStore32U):
CHECK(EmitAtomicStore(f, ValType::I64, Scalar::Uint32));
case uint32_t(ThreadOp::I32AtomicAdd):
CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Int32, AtomicOp::Add));
case uint32_t(ThreadOp::I64AtomicAdd):
CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Int64, AtomicOp::Add));
case uint32_t(ThreadOp::I32AtomicAdd8U):
CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint8, AtomicOp::Add));
case uint32_t(ThreadOp::I32AtomicAdd16U):
CHECK(
EmitAtomicRMW(f, ValType::I32, Scalar::Uint16, AtomicOp::Add));
case uint32_t(ThreadOp::I64AtomicAdd8U):
CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint8, AtomicOp::Add));
case uint32_t(ThreadOp::I64AtomicAdd16U):
CHECK(
EmitAtomicRMW(f, ValType::I64, Scalar::Uint16, AtomicOp::Add));
case uint32_t(ThreadOp::I64AtomicAdd32U):
CHECK(
EmitAtomicRMW(f, ValType::I64, Scalar::Uint32, AtomicOp::Add));
case uint32_t(ThreadOp::I32AtomicSub):
CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Int32, AtomicOp::Sub));
case uint32_t(ThreadOp::I64AtomicSub):
CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Int64, AtomicOp::Sub));
case uint32_t(ThreadOp::I32AtomicSub8U):
CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint8, AtomicOp::Sub));
case uint32_t(ThreadOp::I32AtomicSub16U):
CHECK(
EmitAtomicRMW(f, ValType::I32, Scalar::Uint16, AtomicOp::Sub));
case uint32_t(ThreadOp::I64AtomicSub8U):
CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint8, AtomicOp::Sub));
case uint32_t(ThreadOp::I64AtomicSub16U):
CHECK(
EmitAtomicRMW(f, ValType::I64, Scalar::Uint16, AtomicOp::Sub));
case uint32_t(ThreadOp::I64AtomicSub32U):
CHECK(
EmitAtomicRMW(f, ValType::I64, Scalar::Uint32, AtomicOp::Sub));
case uint32_t(ThreadOp::I32AtomicAnd):
CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Int32, AtomicOp::And));
case uint32_t(ThreadOp::I64AtomicAnd):
CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Int64, AtomicOp::And));
case uint32_t(ThreadOp::I32AtomicAnd8U):
CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint8, AtomicOp::And));
case uint32_t(ThreadOp::I32AtomicAnd16U):
CHECK(
EmitAtomicRMW(f, ValType::I32, Scalar::Uint16, AtomicOp::And));
case uint32_t(ThreadOp::I64AtomicAnd8U):
CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint8, AtomicOp::And));
case uint32_t(ThreadOp::I64AtomicAnd16U):
CHECK(
EmitAtomicRMW(f, ValType::I64, Scalar::Uint16, AtomicOp::And));
case uint32_t(ThreadOp::I64AtomicAnd32U):
CHECK(
EmitAtomicRMW(f, ValType::I64, Scalar::Uint32, AtomicOp::And));
case uint32_t(ThreadOp::I32AtomicOr):
CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Int32, AtomicOp::Or));
case uint32_t(ThreadOp::I64AtomicOr):
CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Int64, AtomicOp::Or));
case uint32_t(ThreadOp::I32AtomicOr8U):
CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint8, AtomicOp::Or));
case uint32_t(ThreadOp::I32AtomicOr16U):
CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint16, AtomicOp::Or));
case uint32_t(ThreadOp::I64AtomicOr8U):
CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint8, AtomicOp::Or));
case uint32_t(ThreadOp::I64AtomicOr16U):
CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint16, AtomicOp::Or));
case uint32_t(ThreadOp::I64AtomicOr32U):
CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint32, AtomicOp::Or));
case uint32_t(ThreadOp::I32AtomicXor):
CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Int32, AtomicOp::Xor));
case uint32_t(ThreadOp::I64AtomicXor):
CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Int64, AtomicOp::Xor));
case uint32_t(ThreadOp::I32AtomicXor8U):
CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint8, AtomicOp::Xor));
case uint32_t(ThreadOp::I32AtomicXor16U):
CHECK(
EmitAtomicRMW(f, ValType::I32, Scalar::Uint16, AtomicOp::Xor));
case uint32_t(ThreadOp::I64AtomicXor8U):
CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint8, AtomicOp::Xor));
case uint32_t(ThreadOp::I64AtomicXor16U):
CHECK(
EmitAtomicRMW(f, ValType::I64, Scalar::Uint16, AtomicOp::Xor));
case uint32_t(ThreadOp::I64AtomicXor32U):
CHECK(
EmitAtomicRMW(f, ValType::I64, Scalar::Uint32, AtomicOp::Xor));
case uint32_t(ThreadOp::I32AtomicXchg):
CHECK(EmitAtomicXchg(f, ValType::I32, Scalar::Int32));
case uint32_t(ThreadOp::I64AtomicXchg):
CHECK(EmitAtomicXchg(f, ValType::I64, Scalar::Int64));
case uint32_t(ThreadOp::I32AtomicXchg8U):
CHECK(EmitAtomicXchg(f, ValType::I32, Scalar::Uint8));
case uint32_t(ThreadOp::I32AtomicXchg16U):
CHECK(EmitAtomicXchg(f, ValType::I32, Scalar::Uint16));
case uint32_t(ThreadOp::I64AtomicXchg8U):
CHECK(EmitAtomicXchg(f, ValType::I64, Scalar::Uint8));
case uint32_t(ThreadOp::I64AtomicXchg16U):
CHECK(EmitAtomicXchg(f, ValType::I64, Scalar::Uint16));
case uint32_t(ThreadOp::I64AtomicXchg32U):
CHECK(EmitAtomicXchg(f, ValType::I64, Scalar::Uint32));
case uint32_t(ThreadOp::I32AtomicCmpXchg):
CHECK(EmitAtomicCmpXchg(f, ValType::I32, Scalar::Int32));
case uint32_t(ThreadOp::I64AtomicCmpXchg):
CHECK(EmitAtomicCmpXchg(f, ValType::I64, Scalar::Int64));
case uint32_t(ThreadOp::I32AtomicCmpXchg8U):
CHECK(EmitAtomicCmpXchg(f, ValType::I32, Scalar::Uint8));
case uint32_t(ThreadOp::I32AtomicCmpXchg16U):
CHECK(EmitAtomicCmpXchg(f, ValType::I32, Scalar::Uint16));
case uint32_t(ThreadOp::I64AtomicCmpXchg8U):
CHECK(EmitAtomicCmpXchg(f, ValType::I64, Scalar::Uint8));
case uint32_t(ThreadOp::I64AtomicCmpXchg16U):
CHECK(EmitAtomicCmpXchg(f, ValType::I64, Scalar::Uint16));
case uint32_t(ThreadOp::I64AtomicCmpXchg32U):
CHECK(EmitAtomicCmpXchg(f, ValType::I64, Scalar::Uint32));
default:
return f.iter().unrecognizedOpcode(&op);
}
break;
}
// asm.js-specific operators
case uint16_t(Op::MozPrefix): {
if (op.b1 == uint32_t(MozOp::CallBuiltinModuleFunc)) {
if (!f.moduleEnv().isBuiltinModule()) {
return f.iter().unrecognizedOpcode(&op);
}
CHECK(EmitCallBuiltinModuleFunc(f));
}
if (!f.moduleEnv().isAsmJS()) {
return f.iter().unrecognizedOpcode(&op);
}
switch (op.b1) {
case uint32_t(MozOp::TeeGlobal):
CHECK(EmitTeeGlobal(f));
case uint32_t(MozOp::I32Min):
case uint32_t(MozOp::I32Max):
CHECK(EmitMinMax(f, ValType::I32, MIRType::Int32,
MozOp(op.b1) == MozOp::I32Max));
case uint32_t(MozOp::I32Neg):
CHECK(EmitUnaryWithType<MWasmNeg>(f, ValType::I32, MIRType::Int32));
case uint32_t(MozOp::I32BitNot):
CHECK(EmitBitNot(f, ValType::I32));
case uint32_t(MozOp::I32Abs):
CHECK(EmitUnaryWithType<MAbs>(f, ValType::I32, MIRType::Int32));
case uint32_t(MozOp::F32TeeStoreF64):
CHECK(EmitTeeStoreWithCoercion(f, ValType::F32, Scalar::Float64));
case uint32_t(MozOp::F64TeeStoreF32):
CHECK(EmitTeeStoreWithCoercion(f, ValType::F64, Scalar::Float32));
case uint32_t(MozOp::I32TeeStore8):
CHECK(EmitTeeStore(f, ValType::I32, Scalar::Int8));
case uint32_t(MozOp::I32TeeStore16):
CHECK(EmitTeeStore(f, ValType::I32, Scalar::Int16));
case uint32_t(MozOp::I64TeeStore8):
CHECK(EmitTeeStore(f, ValType::I64, Scalar::Int8));
case uint32_t(MozOp::I64TeeStore16):
CHECK(EmitTeeStore(f, ValType::I64, Scalar::Int16));
case uint32_t(MozOp::I64TeeStore32):
CHECK(EmitTeeStore(f, ValType::I64, Scalar::Int32));
case uint32_t(MozOp::I32TeeStore):
CHECK(EmitTeeStore(f, ValType::I32, Scalar::Int32));
case uint32_t(MozOp::I64TeeStore):
CHECK(EmitTeeStore(f, ValType::I64, Scalar::Int64));
case uint32_t(MozOp::F32TeeStore):
CHECK(EmitTeeStore(f, ValType::F32, Scalar::Float32));
case uint32_t(MozOp::F64TeeStore):
CHECK(EmitTeeStore(f, ValType::F64, Scalar::Float64));
case uint32_t(MozOp::F64Mod):
CHECK(EmitRem(f, ValType::F64, MIRType::Double,
/* isUnsigned = */ false));
case uint32_t(MozOp::F64SinNative):
CHECK(EmitUnaryMathBuiltinCall(f, SASigSinNativeD));
case uint32_t(MozOp::F64SinFdlibm):
CHECK(EmitUnaryMathBuiltinCall(f, SASigSinFdlibmD));
case uint32_t(MozOp::F64CosNative):
CHECK(EmitUnaryMathBuiltinCall(f, SASigCosNativeD));
case uint32_t(MozOp::F64CosFdlibm):
CHECK(EmitUnaryMathBuiltinCall(f, SASigCosFdlibmD));
case uint32_t(MozOp::F64TanNative):
CHECK(EmitUnaryMathBuiltinCall(f, SASigTanNativeD));
case uint32_t(MozOp::F64TanFdlibm):
CHECK(EmitUnaryMathBuiltinCall(f, SASigTanFdlibmD));
case uint32_t(MozOp::F64Asin):
CHECK(EmitUnaryMathBuiltinCall(f, SASigASinD));
case uint32_t(MozOp::F64Acos):
CHECK(EmitUnaryMathBuiltinCall(f, SASigACosD));
case uint32_t(MozOp::F64Atan):
CHECK(EmitUnaryMathBuiltinCall(f, SASigATanD));
case uint32_t(MozOp::F64Exp):
CHECK(EmitUnaryMathBuiltinCall(f, SASigExpD));
case uint32_t(MozOp::F64Log):
CHECK(EmitUnaryMathBuiltinCall(f, SASigLogD));
case uint32_t(MozOp::F64Pow):
CHECK(EmitBinaryMathBuiltinCall(f, SASigPowD));
case uint32_t(MozOp::F64Atan2):
CHECK(EmitBinaryMathBuiltinCall(f, SASigATan2D));
case uint32_t(MozOp::OldCallDirect):
CHECK(EmitCall(f, /* asmJSFuncDef = */ true));
case uint32_t(MozOp::OldCallIndirect):
CHECK(EmitCallIndirect(f, /* oldStyle = */ true));
default:
return f.iter().unrecognizedOpcode(&op);
}
break;
}
default:
return f.iter().unrecognizedOpcode(&op);
}
}
MOZ_CRASH("unreachable");
#undef CHECK
}
static bool IonBuildMIR(Decoder& d, const ModuleEnvironment& moduleEnv,
const FuncCompileInput& func,
const ValTypeVector& locals, MIRGenerator& mir,
TryNoteVector& tryNotes, FeatureUsage* observedFeatures,
UniqueChars* error) {
// Initialize MIR global information used for optimization
if (moduleEnv.numMemories() > 0) {
if (moduleEnv.memories[0].indexType() == IndexType::I32) {
mir.initMinWasmMemory0Length(moduleEnv.memories[0].initialLength32());
} else {
mir.initMinWasmMemory0Length(moduleEnv.memories[0].initialLength64());
}
}
// Build MIR graph
FunctionCompiler f(moduleEnv, d, func, locals, mir, tryNotes);
if (!f.init()) {
return false;
}
if (!f.startBlock()) {
return false;
}
if (!EmitBodyExprs(f)) {
return false;
}
f.finish();
*observedFeatures = f.featureUsage();
return true;
}
bool wasm::IonCompileFunctions(const ModuleEnvironment& moduleEnv,
const CompilerEnvironment& compilerEnv,
LifoAlloc& lifo,
const FuncCompileInputVector& inputs,
CompiledCode* code, UniqueChars* error) {
MOZ_ASSERT(compilerEnv.tier() == Tier::Optimized);
MOZ_ASSERT(compilerEnv.debug() == DebugEnabled::False);
TempAllocator alloc(&lifo);
JitContext jitContext;
MOZ_ASSERT(IsCompilingWasm());
WasmMacroAssembler masm(alloc, moduleEnv);
#if defined(JS_CODEGEN_ARM64)
masm.SetStackPointer64(PseudoStackPointer64);
#endif
// Swap in already-allocated empty vectors to avoid malloc/free.
MOZ_ASSERT(code->empty());
if (!code->swap(masm)) {
return false;
}
// Create a description of the stack layout created by GenerateTrapExit().
RegisterOffsets trapExitLayout;
size_t trapExitLayoutNumWords;
GenerateTrapExitRegisterOffsets(&trapExitLayout, &trapExitLayoutNumWords);
for (const FuncCompileInput& func : inputs) {
JitSpewCont(JitSpew_Codegen, "\n");
JitSpew(JitSpew_Codegen,
"# ================================"
"==================================");
JitSpew(JitSpew_Codegen, "# ==");
JitSpew(JitSpew_Codegen,
"# wasm::IonCompileFunctions: starting on function index %d",
(int)func.index);
Decoder d(func.begin, func.end, func.lineOrBytecode, error);
// Build the local types vector.
ValTypeVector locals;
if (!DecodeLocalEntriesWithParams(d, moduleEnv, func.index, &locals)) {
return false;
}
// Set up for Ion compilation.
const JitCompileOptions options;
MIRGraph graph(&alloc);
CompileInfo compileInfo(locals.length());
// Only activate branch hinting if the option is enabled and some hints were
// parsed.
if (moduleEnv.branchHintingEnabled() && !moduleEnv.branchHints.isEmpty()) {
compileInfo.setBranchHinting(true);
}
MIRGenerator mir(nullptr, options, &alloc, &graph, &compileInfo,
IonOptimizations.get(OptimizationLevel::Wasm));
// Build MIR graph
FeatureUsage observedFeatures;
if (!IonBuildMIR(d, moduleEnv, func, locals, mir, masm.tryNotes(),
&observedFeatures, error)) {
return false;
}
// Record observed feature usage
code->featureUsage |= observedFeatures;
// Compile MIR graph
{
jit::SpewBeginWasmFunction(&mir, func.index);
jit::AutoSpewEndFunction spewEndFunction(&mir);
if (!OptimizeMIR(&mir)) {
return false;
}
LIRGraph* lir = GenerateLIR(&mir);
if (!lir) {
return false;
}
size_t unwindInfoBefore = masm.codeRangeUnwindInfos().length();
CodeGenerator codegen(&mir, lir, &masm);
BytecodeOffset prologueTrapOffset(func.lineOrBytecode);
FuncOffsets offsets;
ArgTypeVector args(*moduleEnv.funcs[func.index].type);
if (!codegen.generateWasm(CallIndirectId::forFunc(moduleEnv, func.index),
prologueTrapOffset, args, trapExitLayout,
trapExitLayoutNumWords, &offsets,
&code->stackMaps, &d)) {
return false;
}
bool hasUnwindInfo =
unwindInfoBefore != masm.codeRangeUnwindInfos().length();
if (!code->codeRanges.emplaceBack(func.index, func.lineOrBytecode,
offsets, hasUnwindInfo)) {
return false;
}
}
JitSpew(JitSpew_Codegen,
"# wasm::IonCompileFunctions: completed function index %d",
(int)func.index);
JitSpew(JitSpew_Codegen, "# ==");
JitSpew(JitSpew_Codegen,
"# ================================"
"==================================");
JitSpewCont(JitSpew_Codegen, "\n");
}
masm.finish();
if (masm.oom()) {
return false;
}
return code->swap(masm);
}
bool wasm::IonDumpFunction(const ModuleEnvironment& moduleEnv,
const FuncCompileInput& func,
IonDumpContents contents, GenericPrinter& out,
UniqueChars* error) {
LifoAlloc lifo(TempAllocator::PreferredLifoChunkSize);
TempAllocator alloc(&lifo);
JitContext jitContext;
Decoder d(func.begin, func.end, func.lineOrBytecode, error);
// Decode the locals.
ValTypeVector locals;
if (!DecodeLocalEntriesWithParams(d, moduleEnv, func.index, &locals)) {
return false;
}
// Set up for Ion compilation.
const JitCompileOptions options;
MIRGraph graph(&alloc);
CompileInfo compileInfo(locals.length());
MIRGenerator mir(nullptr, options, &alloc, &graph, &compileInfo,
IonOptimizations.get(OptimizationLevel::Wasm));
// Build MIR graph
TryNoteVector tryNotes;
FeatureUsage observedFeatures;
if (!IonBuildMIR(d, moduleEnv, func, locals, mir, tryNotes, &observedFeatures,
error)) {
return false;
}
if (contents == IonDumpContents::UnoptimizedMIR) {
graph.dump(out);
return true;
}
// Optimize the MIR graph
if (!OptimizeMIR(&mir)) {
return false;
}
if (contents == IonDumpContents::OptimizedMIR) {
graph.dump(out);
return true;
}
#ifdef JS_JITSPEW
// Generate the LIR graph
LIRGraph* lir = GenerateLIR(&mir);
if (!lir) {
return false;
}
MOZ_ASSERT(contents == IonDumpContents::LIR);
lir->dump(out);
#else
out.printf("cannot dump LIR without --enable-jitspew");
#endif
return true;
}
bool js::wasm::IonPlatformSupport() {
#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || \
defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS64) || \
defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_LOONG64) || \
defined(JS_CODEGEN_RISCV64)
return true;
#else
return false;
#endif
}