// Copyright 2019 the V8 project authors. All rights reserved.

// Use of this source code is governed by a BSD-style license that can be

// found in the LICENSE file.

#include "src/objects/backing-store.h"

#include <cstring>

#include <optional>

#include "src/base/bits.h"

#include "src/execution/isolate.h"

#include "src/handles/global-handles.h"

#include "src/logging/counters.h"

#include "src/sandbox/sandbox.h"

#if V8_ENABLE_WEBASSEMBLY

#include "src/trap-handler/trap-handler.h"

#include "src/wasm/wasm-constants.h"

#include "src/wasm/wasm-engine.h"

#include "src/wasm/wasm-limits.h"

#include "src/wasm/wasm-module.h"

#include "src/wasm/wasm-objects-inl.h"

#endif // V8_ENABLE_WEBASSEMBLY

#define TRACE_BS(...) \

do { \

if (v8_flags.trace_backing_store) PrintF(__VA_ARGS__); \

} while (false)

namespace v8::internal {

namespace {

#if V8_ENABLE_WEBASSEMBLY && V8_TARGET_ARCH_64_BIT

constexpr size_t kFullGuardSize32 = uint64_t{8} * GB;

#endif

std::atomic<uint32_t> next_backing_store_id_{1};

// Allocation results are reported to UMA

//

// See wasm_memory_allocation_result in counters-definitions.h

enum class AllocationStatus {

kSuccess, // Succeeded on the first try

kSuccessAfterRetry, // Succeeded after garbage collection

kAddressSpaceLimitReachedFailure, // Failed because Wasm is at its address

// space limit

kOtherFailure // Failed for an unknown reason

};

size_t GetReservationSize(bool has_guard_regions, size_t byte_capacity,

bool is_wasm_memory64) {

#if V8_TARGET_ARCH_64_BIT && V8_ENABLE_WEBASSEMBLY

DCHECK_IMPLIES(is_wasm_memory64 && has_guard_regions,

v8_flags.wasm_memory64_trap_handling);

if (has_guard_regions) {

if (is_wasm_memory64) {

DCHECK_LE(byte_capacity, wasm::kMaxMemory64Size);

return wasm::kMaxMemory64Size;

} else {

static_assert(kFullGuardSize32 >= size_t{4} * GB);

DCHECK_LE(byte_capacity, size_t{4} * GB);

return kFullGuardSize32;

}

}

#else

DCHECK(!has_guard_regions);

#endif

return byte_capacity;

}

base::AddressRegion GetReservedRegion(bool has_guard_regions,

bool is_wasm_memory64, void* buffer_start,

size_t byte_capacity) {

return base::AddressRegion(

reinterpret_cast<Address>(buffer_start),

GetReservationSize(has_guard_regions, byte_capacity, is_wasm_memory64));

}

void RecordStatus(Isolate* isolate, AllocationStatus status) {

isolate->counters()->wasm_memory_allocation_result()->AddSample(

static_cast<int>(status));

}

} // namespace

// The backing store for a Wasm shared memory remembers all the isolates

// with which it has been shared.

struct SharedWasmMemoryData {

std::vector<Isolate*> isolates_;

};

BackingStore::BackingStore(PageAllocator* page_allocator, void* buffer_start,

size_t byte_length, size_t max_byte_length,

size_t byte_capacity, SharedFlag shared,

ResizableFlag resizable, bool is_wasm_memory,

bool is_wasm_memory64, bool has_guard_regions,

bool custom_deleter, bool empty_deleter)

: buffer_start_(buffer_start),

byte_length_(byte_length),

max_byte_length_(max_byte_length),

byte_capacity_(byte_capacity),

id_(next_backing_store_id_.fetch_add(1)),

page_allocator_(page_allocator) {

// TODO(v8:11111): RAB / GSAB - Wasm integration.

DCHECK_IMPLIES(is_wasm_memory64, is_wasm_memory);

DCHECK_IMPLIES(has_guard_regions, is_wasm_memory);

DCHECK_IMPLIES(is_wasm_memory, resizable == ResizableFlag::kNotResizable);

DCHECK_IMPLIES(resizable == ResizableFlag::kResizable, !custom_deleter);

DCHECK_IMPLIES(!is_wasm_memory && resizable == ResizableFlag::kNotResizable,

byte_length_ == max_byte_length_);

DCHECK_GE(max_byte_length, byte_length);

DCHECK_GE(byte_capacity, max_byte_length);

// TODO(1445003): Demote to a DCHECK once we found the issue.

// Wasm memory should never be empty (== zero capacity). Otherwise

// {JSArrayBuffer::Attach} would replace it by the {EmptyBackingStore} and we

// loose information.

// This is particularly important for shared Wasm memory.

CHECK_IMPLIES(is_wasm_memory, byte_capacity != 0);

base::EnumSet<Flag, uint16_t> flags;

if (shared == SharedFlag::kShared) flags.Add(kIsShared);

if (resizable == ResizableFlag::kResizable) flags.Add(kIsResizableByJs);

if (is_wasm_memory) flags.Add(kIsWasmMemory);

if (is_wasm_memory64) flags.Add(kIsWasmMemory64);

if (has_guard_regions) flags.Add(kHasGuardRegions);

if (custom_deleter) flags.Add(kCustomDeleter);

if (empty_deleter) flags.Add(kEmptyDeleter);

flags_.store(flags, std::memory_order_relaxed);

}

BackingStore::~BackingStore() {

GlobalBackingStoreRegistry::Unregister(this);

struct ClearSharedAllocator {

BackingStore* const bs;

~ClearSharedAllocator() {

if (!bs->holds_shared_ptr_to_allocator()) return;

bs->type_specific_data_.v8_api_array_buffer_allocator_shared

.std::shared_ptr<v8::ArrayBuffer::Allocator>::~shared_ptr();

}

} clear_shared_allocator{this};

if (buffer_start_ == nullptr) return;

auto FreeResizableMemory = [this] {

DCHECK(!custom_deleter());

DCHECK(is_resizable_by_js() || is_wasm_memory());

auto region = GetReservedRegion(has_guard_regions(), is_wasm_memory64(),

buffer_start_, byte_capacity_);

if (!region.is_empty()) {

FreePages(page_allocator_, reinterpret_cast<void*>(region.begin()),

region.size());

}

};

#if V8_ENABLE_WEBASSEMBLY

if (is_wasm_memory()) {

size_t reservation_size = GetReservationSize(

has_guard_regions(), byte_capacity_, is_wasm_memory64());

TRACE_BS(

"BSw:free bs=%p mem=%p (length=%zu, capacity=%zu, reservation=%zu)\n",

this, buffer_start_, byte_length(), byte_capacity_, reservation_size);

if (is_shared()) {

// Deallocate the list of attached memory objects.

SharedWasmMemoryData* shared_data = get_shared_wasm_memory_data();

delete shared_data;

}

// Wasm memories are always allocated through the page allocator.

FreeResizableMemory();

return;

}

#endif // V8_ENABLE_WEBASSEMBLY

if (is_resizable_by_js()) {

FreeResizableMemory();

return;

}

if (custom_deleter()) {

TRACE_BS("BS:custom deleter bs=%p mem=%p (length=%zu, capacity=%zu)\n",

this, buffer_start_, byte_length(), byte_capacity_);

type_specific_data_.deleter.callback(buffer_start_, byte_length_,

type_specific_data_.deleter.data);

return;

}

// JSArrayBuffer backing store. Deallocate through the embedder's allocator.

auto allocator = get_v8_api_array_buffer_allocator();

TRACE_BS("BS:free bs=%p mem=%p (length=%zu, capacity=%zu)\n", this,

buffer_start_, byte_length(), byte_capacity_);

allocator->Free(buffer_start_, byte_length_);

}

// Allocate a backing store using the array buffer allocator from the embedder.

std::unique_ptr<BackingStore> BackingStore::Allocate(

Isolate* isolate, size_t byte_length, SharedFlag shared,

InitializedFlag initialized) {

void* buffer_start = nullptr;

auto allocator = isolate->array_buffer_allocator();

CHECK_NOT_NULL(allocator);

if (byte_length > allocator->MaxAllocationSize()) return {};

if (byte_length != 0) {

auto counters = isolate->counters();

int mb_length = static_cast<int>(byte_length / MB);

if (mb_length > 0) {

counters->array_buffer_big_allocations()->AddSample(mb_length);

}

if (shared == SharedFlag::kShared) {

counters->shared_array_allocations()->AddSample(mb_length);

}

auto allocate_buffer = [allocator, initialized](size_t byte_length) {

if (initialized == InitializedFlag::kUninitialized) {

return allocator->AllocateUninitialized(byte_length);

}

return allocator->Allocate(byte_length);

};

buffer_start = isolate->heap()->AllocateExternalBackingStore(

allocate_buffer, byte_length);

if (buffer_start == nullptr) {

// Allocation failed.

counters->array_buffer_new_size_failures()->AddSample(mb_length);

return {};

}

#ifdef V8_ENABLE_SANDBOX

// Check to catch use of a non-sandbox-compatible ArrayBufferAllocator.

CHECK_WITH_MSG(isolate->isolate_group()->sandbox()->Contains(buffer_start),

"When the V8 Sandbox is enabled, ArrayBuffer backing stores "

"must be allocated inside the sandbox address space. Please "

"use an appropriate ArrayBuffer::Allocator to allocate "

"these buffers, or disable the sandbox.");

#endif

}

PageAllocator* page_allocator =

isolate->isolate_group()->GetBackingStorePageAllocator();

auto result = new BackingStore(page_allocator,

buffer_start, // start

byte_length, // length

byte_length, // max length

byte_length, // capacity

shared, // shared

ResizableFlag::kNotResizable, // resizable

false, // is_wasm_memory

false, // is_wasm_memory64

false, // has_guard_regions

false, // custom_deleter

false); // empty_deleter

TRACE_BS("BS:alloc bs=%p mem=%p (length=%zu)\n", result,

result->buffer_start(), byte_length);

result->SetAllocatorFromIsolate(isolate);

return std::unique_ptr<BackingStore>(result);

}

void BackingStore::SetAllocatorFromIsolate(Isolate* isolate) {

if (auto allocator_shared = isolate->array_buffer_allocator_shared()) {

set_flag(kHoldsSharedPtrToAllocater);

new (&type_specific_data_.v8_api_array_buffer_allocator_shared)

std::shared_ptr<v8::ArrayBuffer::Allocator>(

std::move(allocator_shared));

} else {

type_specific_data_.v8_api_array_buffer_allocator =

isolate->array_buffer_allocator();

}

}

std::unique_ptr<BackingStore> BackingStore::TryAllocateAndPartiallyCommitMemory(

Isolate* isolate, size_t byte_length, size_t max_byte_length,

size_t page_size, size_t initial_pages, size_t maximum_pages,

WasmMemoryFlag wasm_memory, SharedFlag shared, bool has_guard_regions) {

// Enforce engine limitation on the maximum number of pages.

if (maximum_pages > std::numeric_limits<size_t>::max() / page_size) {

return nullptr;

}

// Cannot reserve 0 pages on some OSes.

if (maximum_pages == 0) maximum_pages = 1;

TRACE_BS("BSw:try %zu pages, %zu max\n", initial_pages, maximum_pages);

#if V8_ENABLE_WEBASSEMBLY

bool is_wasm_memory = wasm_memory != WasmMemoryFlag::kNotWasm;

bool is_wasm_memory64 = wasm_memory == WasmMemoryFlag::kWasmMemory64;

#else

CHECK_EQ(WasmMemoryFlag::kNotWasm, wasm_memory);

constexpr bool is_wasm_memory = false;

constexpr bool is_wasm_memory64 = false;

#endif // V8_ENABLE_WEBASSEMBLY

DCHECK_IMPLIES(has_guard_regions, is_wasm_memory);

// For accounting purposes, whether a GC was necessary.

bool did_retry = false;

// A helper to try running a function up to 3 times, executing a GC

// if the first and second attempts failed.

auto gc_retry = [&](const std::function<bool()>& fn) {

for (int i = 0; i < 3; i++) {

if (fn()) return true;

// Collect garbage and retry.

did_retry = true;

if (isolate != nullptr) {

isolate->heap()->MemoryPressureNotification(

MemoryPressureLevel::kCritical, true);

}

}

return false;

};

size_t byte_capacity = maximum_pages * page_size;

size_t reservation_size =

GetReservationSize(has_guard_regions, byte_capacity, is_wasm_memory64);

//--------------------------------------------------------------------------

// Allocate pages (inaccessible by default).

//--------------------------------------------------------------------------

void* allocation_base = nullptr;

#ifdef V8_COMPRESS_POINTERS_IN_MULTIPLE_CAGES

#ifdef V8_ENABLE_SANDBOX

CHECK_WITH_MSG(isolate || Sandbox::current(),

"One must enter an v8::Isolate before allocating resizable "

"array backing stores");

#endif

#endif

PageAllocator* page_allocator =

isolate ? isolate->isolate_group()->GetBackingStorePageAllocator()

: GetArrayBufferPageAllocator();

auto allocate_pages = [&] {

allocation_base = AllocatePages(page_allocator, nullptr, reservation_size,

page_size, PageAllocator::kNoAccess);

return allocation_base != nullptr;

};

if (!gc_retry(allocate_pages)) {

// Page allocator could not reserve enough pages.

if (isolate != nullptr) {

RecordStatus(isolate, AllocationStatus::kOtherFailure);

}

TRACE_BS("BSw:try failed to allocate pages\n");

return {};

}

uint8_t* buffer_start = reinterpret_cast<uint8_t*>(allocation_base);

//--------------------------------------------------------------------------

// Commit the initial pages (allow read/write).

//--------------------------------------------------------------------------

size_t committed_byte_length = initial_pages * page_size;

auto commit_memory = [&] {

return committed_byte_length == 0 ||

SetPermissions(page_allocator, buffer_start, committed_byte_length,

PageAllocator::kReadWrite);

};

if (!gc_retry(commit_memory)) {

TRACE_BS("BSw:try failed to set permissions (%p, %zu)\n", buffer_start,

committed_byte_length);

FreePages(page_allocator, allocation_base, reservation_size);

// SetPermissions put us over the process memory limit.

// We return an empty result so that the caller can throw an exception.

return {};

}

if (isolate != nullptr) {

RecordStatus(isolate, did_retry ? AllocationStatus::kSuccessAfterRetry

: AllocationStatus::kSuccess);

}

ResizableFlag resizable =

is_wasm_memory ? ResizableFlag::kNotResizable : ResizableFlag::kResizable;

auto result = new BackingStore(page_allocator,

buffer_start, // start

byte_length, // length

max_byte_length, // max_byte_length

byte_capacity, // capacity

shared, // shared

resizable, // resizable

is_wasm_memory, // is_wasm_memory

is_wasm_memory64, // is_wasm_memory64

has_guard_regions, // has_guard_regions

false, // custom_deleter

false); // empty_deleter

TRACE_BS(

"BSw:alloc bs=%p mem=%p (length=%zu, capacity=%zu, reservation=%zu)\n",

result, result->buffer_start(), byte_length, byte_capacity,

reservation_size);

return std::unique_ptr<BackingStore>(result);

}

#if V8_ENABLE_WEBASSEMBLY

// Allocate a backing store for a Wasm memory. Always use the page allocator

// and add guard regions.

std::unique_ptr<BackingStore> BackingStore::AllocateWasmMemory(

Isolate* isolate, size_t initial_pages, size_t maximum_pages,

WasmMemoryFlag wasm_memory, SharedFlag shared) {

// Wasm pages must be a multiple of the allocation page size.

DCHECK_EQ(0, wasm::kWasmPageSize % AllocatePageSize());

DCHECK_LE(initial_pages, maximum_pages);

DCHECK_LE(maximum_pages, wasm_memory == WasmMemoryFlag::kWasmMemory32

? wasm::kV8MaxWasmMemory32Pages

: wasm::kV8MaxWasmMemory64Pages);

DCHECK(wasm_memory == WasmMemoryFlag::kWasmMemory32 ||

wasm_memory == WasmMemoryFlag::kWasmMemory64);

bool is_wasm_memory64 = wasm_memory == WasmMemoryFlag::kWasmMemory64;

bool has_guard_regions =

trap_handler::IsTrapHandlerEnabled() &&

(wasm_memory == WasmMemoryFlag::kWasmMemory32 ||

(is_wasm_memory64 && v8_flags.wasm_memory64_trap_handling));

auto TryAllocate = [isolate, initial_pages, wasm_memory, shared,

has_guard_regions](size_t maximum_pages) {

auto result = TryAllocateAndPartiallyCommitMemory(

isolate, initial_pages * wasm::kWasmPageSize,

maximum_pages * wasm::kWasmPageSize, wasm::kWasmPageSize, initial_pages,

maximum_pages, wasm_memory, shared, has_guard_regions);

if (result && shared == SharedFlag::kShared) {

result->type_specific_data_.shared_wasm_memory_data =

new SharedWasmMemoryData();

}

return result;

};

auto backing_store = TryAllocate(maximum_pages);

if (!backing_store &&

!has_guard_regions && // With guard regions we always reserved a fixed

// number of pages.

maximum_pages - initial_pages >= 4) {

// Retry with smaller maximum pages at each retry.

auto delta = (maximum_pages - initial_pages) / 4;

size_t sizes[] = {maximum_pages - delta, maximum_pages - 2 * delta,

maximum_pages - 3 * delta, initial_pages};

for (size_t reduced_maximum_pages : sizes) {

backing_store = TryAllocate(reduced_maximum_pages);

if (backing_store) break;

}

}

return backing_store;

}

std::unique_ptr<BackingStore> BackingStore::CopyWasmMemory(

Isolate* isolate, size_t new_pages, size_t max_pages,

WasmMemoryFlag wasm_memory) {

// Note that we could allocate uninitialized to save initialization cost here,

// but since Wasm memories are allocated by the page allocator, the zeroing

// cost is already built-in.

auto new_backing_store = BackingStore::AllocateWasmMemory(

isolate, new_pages, max_pages, wasm_memory,

is_shared() ? SharedFlag::kShared : SharedFlag::kNotShared);

if (!new_backing_store ||

new_backing_store->has_guard_regions() != has_guard_regions()) {

return {};

}

if (byte_length_ > 0) {

// If the allocation was successful, then the new buffer must be at least

// as big as the old one.

DCHECK_GE(new_pages * wasm::kWasmPageSize, byte_length_);

memcpy(new_backing_store->buffer_start(), buffer_start_, byte_length_);

}

return new_backing_store;

}

// Try to grow the size of a wasm memory in place, without realloc + copy.

std::optional<size_t> BackingStore::GrowWasmMemoryInPlace(Isolate* isolate,

size_t delta_pages,

size_t max_pages) {

// This function grows wasm memory by

// * changing the permissions of additional {delta_pages} pages to kReadWrite;

// * increment {byte_length_};

//

// As this code is executed concurrently, the following steps are executed:

// 1) Read the current value of {byte_length_};

// 2) Change the permission of all pages from {buffer_start_} to

// {byte_length_} + {delta_pages} * {page_size} to kReadWrite;

// * This operation may be executed racefully. The OS takes care of

// synchronization.

// 3) Try to update {byte_length_} with a compare_exchange;

// 4) Repeat 1) to 3) until the compare_exchange in 3) succeeds;

//

// The result of this function is the {byte_length_} before growing in pages.

// The result of this function appears like the result of an RMW-update on

// {byte_length_}, i.e. two concurrent calls to this function will result in

// different return values if {delta_pages} != 0.

//

// Invariants:

// * Permissions are always set incrementally, i.e. for any page {b} with

// kReadWrite permission, all pages between the first page {a} and page {b}

// also have kReadWrite permission.

// * {byte_length_} is always lower or equal than the amount of memory with

// permissions set to kReadWrite;

// * This is guaranteed by incrementing {byte_length_} with a

// compare_exchange after changing the permissions.

// * This invariant is the reason why we cannot use a fetch_add.

DCHECK(is_wasm_memory());

max_pages = std::min(max_pages, byte_capacity_ / wasm::kWasmPageSize);

// Do a compare-exchange loop, because we also need to adjust page

// permissions. Note that multiple racing grows both try to set page

// permissions for the entire range (to be RW), so the operating system

// should deal with that raciness. We know we succeeded when we can

// compare/swap the old length with the new length.

size_t old_length = byte_length_.load(std::memory_order_relaxed);

if (delta_pages == 0)

return {old_length / wasm::kWasmPageSize}; // degenerate grow.

if (delta_pages > max_pages) return {}; // would never work.

size_t new_length = 0;

while (true) {

size_t current_pages = old_length / wasm::kWasmPageSize;

// Check if we have exceed the supplied maximum.

if (current_pages > (max_pages - delta_pages)) return {};

new_length = (current_pages + delta_pages) * wasm::kWasmPageSize;

// Try to adjust the permissions on the memory.

if (!i::SetPermissions(GetPlatformPageAllocator(), buffer_start_,

new_length, PageAllocator::kReadWrite)) {

// This is a nondeterministic failure; mark as such in the WasmEngine (for

// differential fuzzing).

wasm::WasmEngine::set_had_nondeterminism();

return {};

}

if (byte_length_.compare_exchange_weak(old_length, new_length,

std::memory_order_acq_rel)) {

// Successfully updated both the length and permissions.

break;

}

}

return {old_length / wasm::kWasmPageSize};

}

void BackingStore::AttachSharedWasmMemoryObject(

Isolate* isolate, DirectHandle<WasmMemoryObject> memory_object) {

DCHECK(is_wasm_memory());

DCHECK(is_shared());

// We need to take the global registry lock for this operation.

GlobalBackingStoreRegistry::AddSharedWasmMemoryObject(isolate, this,

memory_object);

}

void BackingStore::BroadcastSharedWasmMemoryGrow(Isolate* isolate) const {

GlobalBackingStoreRegistry::BroadcastSharedWasmMemoryGrow(isolate, this);

}

void BackingStore::RemoveSharedWasmMemoryObjects(Isolate* isolate) {

GlobalBackingStoreRegistry::Purge(isolate);

}

void BackingStore::UpdateSharedWasmMemoryObjects(Isolate* isolate) {

GlobalBackingStoreRegistry::UpdateSharedWasmMemoryObjects(isolate);

}

void BackingStore::MakeWasmMemoryResizableByJS(bool resizable) {

DCHECK(is_wasm_memory());

DCHECK(!is_shared());

if (resizable) {

set_flag(kIsResizableByJs);

} else {

clear_flag(kIsResizableByJs);

}

}

#endif // V8_ENABLE_WEBASSEMBLY

// Commit already reserved memory (for RAB backing stores (not shared)).

BackingStore::ResizeOrGrowResult BackingStore::ResizeInPlace(

Isolate* isolate, size_t new_byte_length) {

size_t page_size = AllocatePageSize();

size_t new_committed_pages;

bool round_return_value =

RoundUpToPageSize(new_byte_length, page_size,

JSArrayBuffer::kMaxByteLength, &new_committed_pages);

CHECK(round_return_value);

size_t new_committed_length = new_committed_pages * page_size;

DCHECK_LE(new_byte_length, new_committed_length);

DCHECK(!is_shared());

if (new_byte_length < byte_length_) {

// Zero the memory so that in case the buffer is grown later, we have

// zeroed the contents already. This is especially needed for the portion of

// the memory we're not going to decommit below (since it belongs to a

// committed page). In addition, we don't rely on all platforms always

// zeroing decommitted-then-recommitted memory, but zero the memory

// explicitly here.

memset(reinterpret_cast<uint8_t*>(buffer_start_) + new_byte_length, 0,

byte_length_ - new_byte_length);

// Check if we can un-commit some pages.

size_t old_committed_pages;

round_return_value =

RoundUpToPageSize(byte_length_, page_size,

JSArrayBuffer::kMaxByteLength, &old_committed_pages);

CHECK(round_return_value);

DCHECK_LE(new_committed_pages, old_committed_pages);

if (new_committed_pages < old_committed_pages) {

size_t old_committed_length = old_committed_pages * page_size;

if (!i::SetPermissions(

GetPlatformPageAllocator(),

reinterpret_cast<uint8_t*>(buffer_start_) + new_committed_length,

old_committed_length - new_committed_length,

PageAllocator::kNoAccess)) {

return kFailure;

}

}

// Changing the byte length wouldn't strictly speaking be needed, since

// the JSArrayBuffer already stores the updated length. This is to keep

// the BackingStore and JSArrayBuffer in sync.

byte_length_ = new_byte_length;

return kSuccess;

}

if (new_byte_length == byte_length_) {

// i::SetPermissions with size 0 fails on some platforms, so special

// handling for the case byte_length_ == new_byte_length == 0 is required.

return kSuccess;

}

// Try to adjust the permissions on the memory.

if (!i::SetPermissions(GetPlatformPageAllocator(), buffer_start_,

new_committed_length, PageAllocator::kReadWrite)) {

return kFailure;

}

byte_length_ = new_byte_length;

return kSuccess;

}

// Commit already reserved memory (for GSAB backing stores (shared)).

BackingStore::ResizeOrGrowResult BackingStore::GrowInPlace(

Isolate* isolate, size_t new_byte_length) {

size_t page_size = AllocatePageSize();

size_t new_committed_pages;

bool round_return_value =

RoundUpToPageSize(new_byte_length, page_size,

JSArrayBuffer::kMaxByteLength, &new_committed_pages);

CHECK(round_return_value);

size_t new_committed_length = new_committed_pages * page_size;

DCHECK_LE(new_byte_length, new_committed_length);

DCHECK(is_shared());

// See comment in GrowWasmMemoryInPlace.

// GrowableSharedArrayBuffer.prototype.grow can be called from several

// threads. If two threads try to grow() in a racy way, the spec allows the

// larger grow to throw also if the smaller grow succeeds first. The

// implementation below doesn't throw in that case - instead, it retries and

// succeeds. If the larger grow finishes first though, the smaller grow must

// throw.

size_t old_byte_length = byte_length_.load(std::memory_order_seq_cst);

while (true) {

if (new_byte_length < old_byte_length) {

// The caller checks for the new_byte_length < old_byte_length_ case. This

// can only happen if another thread grew the memory after that.

return kRace;

}

if (new_byte_length == old_byte_length) {

// i::SetPermissions with size 0 fails on some platforms, so special

// handling for the case old_byte_length == new_byte_length == 0 is

// required.

return kSuccess;

}

// Try to adjust the permissions on the memory.

if (!i::SetPermissions(GetPlatformPageAllocator(), buffer_start_,

new_committed_length, PageAllocator::kReadWrite)) {

return kFailure;

}

// compare_exchange_weak updates old_byte_length.

if (byte_length_.compare_exchange_weak(old_byte_length, new_byte_length,

std::memory_order_seq_cst)) {

// Successfully updated both the length and permissions.

break;

}

}

return kSuccess;

}

std::unique_ptr<BackingStore> BackingStore::WrapAllocation(

void* allocation_base, size_t allocation_length,

v8::BackingStore::DeleterCallback deleter, void* deleter_data,

SharedFlag shared) {

bool is_empty_deleter = (deleter == v8::BackingStore::EmptyDeleter);

auto result = new BackingStore(nullptr,

allocation_base, // start

allocation_length, // length

allocation_length, // max length

allocation_length, // capacity

shared, // shared

ResizableFlag::kNotResizable, // resizable

false, // is_wasm_memory

false, // is_wasm_memory64

false, // has_guard_regions

true, // custom_deleter

is_empty_deleter); // empty_deleter

result->type_specific_data_.deleter = {deleter, deleter_data};

TRACE_BS("BS:wrap bs=%p mem=%p (length=%zu)\n", result,

result->buffer_start(), result->byte_length());

return std::unique_ptr<BackingStore>(result);

}

std::unique_ptr<BackingStore> BackingStore::EmptyBackingStore(

SharedFlag shared) {

auto result = new BackingStore(nullptr,

nullptr, // start

0, // length

0, // max length

0, // capacity

shared, // shared

ResizableFlag::kNotResizable, // resizable

false, // is_wasm_memory

false, // is_wasm_memory64

false, // has_guard_regions

false, // custom_deleter

false); // empty_deleter

return std::unique_ptr<BackingStore>(result);

}

v8::ArrayBuffer::Allocator* BackingStore::get_v8_api_array_buffer_allocator() {

CHECK(!is_wasm_memory());

auto array_buffer_allocator =

holds_shared_ptr_to_allocator()

? type_specific_data_.v8_api_array_buffer_allocator_shared.get()

: type_specific_data_.v8_api_array_buffer_allocator;

CHECK_NOT_NULL(array_buffer_allocator);

return array_buffer_allocator;

}

SharedWasmMemoryData* BackingStore::get_shared_wasm_memory_data() const {

CHECK(is_wasm_memory() && is_shared());

auto shared_wasm_memory_data = type_specific_data_.shared_wasm_memory_data;

CHECK(shared_wasm_memory_data);

return shared_wasm_memory_data;

}

namespace {

// Implementation details of GlobalBackingStoreRegistry.

struct GlobalBackingStoreRegistryImpl {

GlobalBackingStoreRegistryImpl() = default;

base::Mutex mutex_;

std::unordered_map<const void*, std::weak_ptr<BackingStore>> map_;

};

DEFINE_LAZY_LEAKY_OBJECT_GETTER(GlobalBackingStoreRegistryImpl,

GetGlobalBackingStoreRegistryImpl)

} // namespace

void GlobalBackingStoreRegistry::Register(

std::shared_ptr<BackingStore> backing_store) {

if (!backing_store || !backing_store->buffer_start()) return;

// Only wasm memory backing stores need to be registered globally.

CHECK(backing_store->is_wasm_memory());

GlobalBackingStoreRegistryImpl* impl = GetGlobalBackingStoreRegistryImpl();

base::MutexGuard scope_lock(&impl->mutex_);

if (backing_store->globally_registered()) return;

TRACE_BS("BS:reg bs=%p mem=%p (length=%zu, capacity=%zu)\n",

backing_store.get(), backing_store->buffer_start(),

backing_store->byte_length(), backing_store->byte_capacity());

std::weak_ptr<BackingStore> weak = backing_store;

auto result = impl->map_.insert({backing_store->buffer_start(), weak});

CHECK(result.second);

backing_store->set_flag(BackingStore::kGloballyRegistered);

}

void GlobalBackingStoreRegistry::Unregister(BackingStore* backing_store) {

if (!backing_store->globally_registered()) return;

CHECK(backing_store->is_wasm_memory());

DCHECK_NOT_NULL(backing_store->buffer_start());

GlobalBackingStoreRegistryImpl* impl = GetGlobalBackingStoreRegistryImpl();

base::MutexGuard scope_lock(&impl->mutex_);

const auto& result = impl->map_.find(backing_store->buffer_start());

if (result != impl->map_.end()) {

DCHECK(!result->second.lock());

impl->map_.erase(result);

}

backing_store->clear_flag(BackingStore::kGloballyRegistered);

}

void GlobalBackingStoreRegistry::Purge(Isolate* isolate) {

// We need to keep a reference to all backing stores that are inspected

// in the purging loop below. Otherwise, we might get a deadlock

// if the temporary backing store reference created in the loop is

// the last reference. In that case the destructor of the backing store

// may try to take the &impl->mutex_ in order to unregister itself.

std::vector<std::shared_ptr<BackingStore>> prevent_destruction_under_lock;

GlobalBackingStoreRegistryImpl* impl = GetGlobalBackingStoreRegistryImpl();

base::MutexGuard scope_lock(&impl->mutex_);

// Purge all entries in the map that refer to the given isolate.

for (auto& entry : impl->map_) {

auto backing_store = entry.second.lock();

prevent_destruction_under_lock.emplace_back(backing_store);

if (!backing_store) continue; // skip entries where weak ptr is null

CHECK(backing_store->is_wasm_memory());

if (!backing_store->is_shared()) continue; // skip non-shared memory

SharedWasmMemoryData* shared_data =

backing_store->get_shared_wasm_memory_data();

// Remove this isolate from the isolates list.

std::vector<Isolate*>& isolates = shared_data->isolates_;

auto isolates_it = std::find(isolates.begin(), isolates.end(), isolate);

if (isolates_it != isolates.end()) {

*isolates_it = isolates.back();

isolates.pop_back();

}

DCHECK_EQ(isolates.end(),

std::find(isolates.begin(), isolates.end(), isolate));

}

}

#if V8_ENABLE_WEBASSEMBLY

void GlobalBackingStoreRegistry::AddSharedWasmMemoryObject(

Isolate* isolate, BackingStore* backing_store,

DirectHandle<WasmMemoryObject> memory_object) {

// Add to the weak array list of shared memory objects in the isolate.

isolate->AddSharedWasmMemory(memory_object);

// Add the isolate to the list of isolates sharing this backing store.

GlobalBackingStoreRegistryImpl* impl = GetGlobalBackingStoreRegistryImpl();

base::MutexGuard scope_lock(&impl->mutex_);

SharedWasmMemoryData* shared_data =

backing_store->get_shared_wasm_memory_data();

auto& isolates = shared_data->isolates_;

int free_entry = -1;

for (size_t i = 0; i < isolates.size(); i++) {

if (isolates[i] == isolate) return;

if (isolates[i] == nullptr) free_entry = static_cast<int>(i);

}

if (free_entry >= 0)

isolates[free_entry] = isolate;

else

isolates.push_back(isolate);

}

void GlobalBackingStoreRegistry::BroadcastSharedWasmMemoryGrow(

Isolate* isolate, const BackingStore* backing_store) {

{

GlobalBackingStoreRegistryImpl* impl = GetGlobalBackingStoreRegistryImpl();

// The global lock protects the list of isolates per backing store.

base::MutexGuard scope_lock(&impl->mutex_);

SharedWasmMemoryData* shared_data =

backing_store->get_shared_wasm_memory_data();

for (Isolate* other : shared_data->isolates_) {

if (other == isolate) continue;

other->stack_guard()->RequestGrowSharedMemory();

}

}

// Update memory objects in this isolate.

UpdateSharedWasmMemoryObjects(isolate);

}

void GlobalBackingStoreRegistry::UpdateSharedWasmMemoryObjects(

Isolate* isolate) {

HandleScope scope(isolate);

DirectHandle<WeakArrayList> shared_wasm_memories =

isolate->factory()->shared_wasm_memories();

for (int i = 0, e = shared_wasm_memories->length(); i < e; ++i) {

Tagged<HeapObject> obj;

if (!shared_wasm_memories->Get(i).GetHeapObject(&obj)) continue;

DirectHandle<WasmMemoryObject> memory_object(Cast<WasmMemoryObject>(obj),

isolate);

if (memory_object->array_buffer()->is_resizable_by_js()) {

// If the SharedArrayBuffer is exposed as growable already, there's no

// need to refresh it, but instances still need to be updated with the new

// length.

memory_object->UpdateInstances(isolate);

} else {

WasmMemoryObject::RefreshSharedBuffer(isolate, memory_object,

ResizableFlag::kNotResizable);

}

}

}

#endif // V8_ENABLE_WEBASSEMBLY

} // namespace v8::internal

#undef TRACE_BS