// Copyright 2012 the V8 project authors. All rights reserved.

// Use of this source code is governed by a BSD-style license that can be

// found in the LICENSE file.

#include "src/heap/heap.h"

#include <algorithm>

#include <atomic>

#include <cinttypes>

#include <iomanip>

#include <memory>

#include <optional>

#include <unordered_map>

#include <unordered_set>

#include "include/v8-locker.h"

#include "src/api/api-inl.h"

#include "src/base/bits.h"

#include "src/base/flags.h"

#include "src/base/logging.h"

#include "src/base/macros.h"

#include "src/base/once.h"

#include "src/base/platform/memory.h"

#include "src/base/platform/mutex.h"

#include "src/base/platform/time.h"

#include "src/base/utils/random-number-generator.h"

#include "src/builtins/accessors.h"

#include "src/codegen/assembler-inl.h"

#include "src/codegen/compilation-cache.h"

#include "src/common/assert-scope.h"

#include "src/common/globals.h"

#include "src/compiler-dispatcher/optimizing-compile-dispatcher.h"

#include "src/debug/debug.h"

#include "src/deoptimizer/deoptimizer.h"

#include "src/execution/embedder-state.h"

#include "src/execution/isolate-utils-inl.h"

#include "src/execution/microtask-queue.h"

#include "src/execution/v8threads.h"

#include "src/execution/vm-state-inl.h"

#include "src/flags/flags.h"

#include "src/handles/global-handles-inl.h"

#include "src/handles/traced-handles.h"

#include "src/heap/allocation-observer.h"

#include "src/heap/array-buffer-sweeper.h"

#include "src/heap/base/stack.h"

#include "src/heap/base/worklist.h"

#include "src/heap/code-range.h"

#include "src/heap/code-stats.h"

#include "src/heap/collection-barrier.h"

#include "src/heap/combined-heap.h"

#include "src/heap/concurrent-marking.h"

#include "src/heap/conservative-stack-visitor-inl.h"

#include "src/heap/cppgc-js/cpp-heap.h"

#include "src/heap/ephemeron-remembered-set.h"

#include "src/heap/evacuation-verifier-inl.h"

#include "src/heap/finalization-registry-cleanup-task.h"

#include "src/heap/gc-callbacks.h"

#include "src/heap/gc-tracer-inl.h"

#include "src/heap/gc-tracer.h"

#include "src/heap/heap-allocator.h"

#include "src/heap/heap-controller.h"

#include "src/heap/heap-layout-inl.h"

#include "src/heap/heap-layout-tracer.h"

#include "src/heap/heap-utils-inl.h"

#include "src/heap/heap-visitor-inl.h"

#include "src/heap/heap-visitor.h"

#include "src/heap/heap-write-barrier-inl.h"

#include "src/heap/incremental-marking.h"

#include "src/heap/large-spaces.h"

#include "src/heap/local-heap-inl.h"

#include "src/heap/mark-compact-inl.h"

#include "src/heap/mark-compact.h"

#include "src/heap/marking-barrier-inl.h"

#include "src/heap/marking-barrier.h"

#include "src/heap/marking-state-inl.h"

#include "src/heap/marking-state.h"

#include "src/heap/memory-balancer.h"

#include "src/heap/memory-chunk-layout.h"

#include "src/heap/memory-chunk-metadata.h"

#include "src/heap/memory-measurement.h"

#include "src/heap/memory-reducer.h"

#include "src/heap/minor-gc-job.h"

#include "src/heap/minor-mark-sweep.h"

#include "src/heap/mutable-page-metadata-inl.h"

#include "src/heap/new-spaces.h"

#include "src/heap/object-lock.h"

#include "src/heap/object-stats.h"

#include "src/heap/page-pool.h"

#include "src/heap/paged-spaces-inl.h"

#include "src/heap/parked-scope.h"

#include "src/heap/pretenuring-handler.h"

#include "src/heap/read-only-heap.h"

#include "src/heap/remembered-set.h"

#include "src/heap/safepoint.h"

#include "src/heap/scavenger-inl.h"

#include "src/heap/stress-scavenge-observer.h"

#include "src/heap/sweeper.h"

#include "src/heap/trusted-range.h"

#include "src/heap/visit-object.h"

#include "src/heap/zapping.h"

#include "src/init/bootstrapper.h"

#include "src/init/v8.h"

#include "src/interpreter/interpreter.h"

#include "src/logging/log.h"

#include "src/logging/runtime-call-stats-scope.h"

#include "src/numbers/conversions.h"

#include "src/objects/allocation-site.h"

#include "src/objects/data-handler.h"

#include "src/objects/free-space-inl.h"

#include "src/objects/hash-table-inl.h"

#include "src/objects/hash-table.h"

#include "src/objects/instance-type.h"

#include "src/objects/maybe-object.h"

#include "src/objects/objects.h"

#include "src/objects/slots-atomic-inl.h"

#include "src/objects/slots-inl.h"

#include "src/objects/slots.h"

#include "src/objects/visitors.h"

#include "src/profiler/heap-profiler.h"

#include "src/regexp/regexp.h"

#include "src/snapshot/embedded/embedded-data.h"

#include "src/snapshot/serializer-deserializer.h"

#include "src/snapshot/snapshot.h"

#include "src/strings/string-hasher.h"

#include "src/strings/string-stream.h"

#include "src/strings/unicode-inl.h"

#include "src/tasks/cancelable-task.h"

#include "src/tracing/trace-event.h"

#include "src/utils/utils-inl.h"

#include "src/utils/utils.h"

#if V8_ENABLE_WEBASSEMBLY

#include "src/wasm/wasm-engine.h"

#endif // V8_ENABLE_WEBASSEMBLY

// Has to be the last include (doesn't have include guards):

#include "src/objects/object-macros.h"

namespace v8 {

namespace internal {

void Heap::SetConstructStubCreateDeoptPCOffset(int pc_offset) {

DCHECK_EQ(Smi::zero(), construct_stub_create_deopt_pc_offset());

set_construct_stub_create_deopt_pc_offset(Smi::FromInt(pc_offset));

}

void Heap::SetConstructStubInvokeDeoptPCOffset(int pc_offset) {

DCHECK_EQ(Smi::zero(), construct_stub_invoke_deopt_pc_offset());

set_construct_stub_invoke_deopt_pc_offset(Smi::FromInt(pc_offset));

}

void Heap::SetDeoptPCOffsetAfterAdaptShadowStack(int pc_offset) {

DCHECK((Smi::zero() == deopt_pc_offset_after_adapt_shadow_stack()) ||

(pc_offset == deopt_pc_offset_after_adapt_shadow_stack().value()));

set_deopt_pc_offset_after_adapt_shadow_stack(Smi::FromInt(pc_offset));

}

void Heap::SetInterpreterEntryReturnPCOffset(int pc_offset) {

DCHECK_EQ(Smi::zero(), interpreter_entry_return_pc_offset());

set_interpreter_entry_return_pc_offset(Smi::FromInt(pc_offset));

}

void Heap::SetSerializedObjects(Tagged<HeapObject> objects) {

DCHECK(isolate()->serializer_enabled());

set_serialized_objects(objects);

}

void Heap::SetSerializedGlobalProxySizes(Tagged<FixedArray> sizes) {

DCHECK(isolate()->serializer_enabled());

set_serialized_global_proxy_sizes(sizes);

}

void Heap::SetBasicBlockProfilingData(DirectHandle<ArrayList> list) {

set_basic_block_profiling_data(*list);

}

Heap::Heap()

: isolate_(isolate()),

memory_pressure_level_(MemoryPressureLevel::kNone),

safepoint_(std::make_unique<IsolateSafepoint>(this)),

external_string_table_(this),

allocation_type_for_in_place_internalizable_strings_(

isolate()->OwnsStringTables() ? AllocationType::kOld

: AllocationType::kSharedOld),

marking_state_(isolate_),

non_atomic_marking_state_(isolate_),

pretenuring_handler_(this) {

// Ensure old_generation_size_ is a multiple of kPageSize.

DCHECK_EQ(0, max_old_generation_size() & (PageMetadata::kPageSize - 1));

max_regular_code_object_size_ = MemoryChunkLayout::MaxRegularCodeObjectSize();

set_native_contexts_list(Smi::zero());

// Put a dummy entry in the remembered pages so we can find the list the

// minidump even if there are no real unmapped pages.

RememberUnmappedPage(kNullAddress, false);

}

Heap::~Heap() = default;

size_t Heap::MaxReserved() const {

const size_t kMaxNewLargeObjectSpaceSize = max_semi_space_size_;

return static_cast<size_t>(

(v8_flags.minor_ms ? 1 : 2) * max_semi_space_size_ +

kMaxNewLargeObjectSpaceSize + max_old_generation_size());

}

size_t Heap::YoungGenerationSizeFromOldGenerationSize(size_t old_generation) {

// Compute the semi space size and cap it.

bool is_low_memory = old_generation <= kOldGenerationLowMemory;

size_t semi_space;

if (v8_flags.minor_ms && !is_low_memory) {

semi_space = DefaultMaxSemiSpaceSize();

} else {

size_t ratio = is_low_memory ? OldGenerationToSemiSpaceRatioLowMemory()

: OldGenerationToSemiSpaceRatio();

semi_space = old_generation / ratio;

semi_space = std::min({semi_space, DefaultMaxSemiSpaceSize()});

semi_space = std::max({semi_space, DefaultMinSemiSpaceSize()});

semi_space = RoundUp(semi_space, PageMetadata::kPageSize);

}

return YoungGenerationSizeFromSemiSpaceSize(semi_space);

}

size_t Heap::HeapSizeFromPhysicalMemory(uint64_t physical_memory) {

// Compute the old generation size and cap it.

uint64_t old_generation = physical_memory /

kPhysicalMemoryToOldGenerationRatio *

kHeapLimitMultiplier;

old_generation =

std::min(old_generation,

static_cast<uint64_t>(MaxOldGenerationSize(physical_memory)));

old_generation =

std::max({old_generation, static_cast<uint64_t>(V8HeapTrait::kMinSize)});

old_generation = RoundUp(old_generation, PageMetadata::kPageSize);

size_t young_generation = YoungGenerationSizeFromOldGenerationSize(

static_cast<size_t>(old_generation));

return static_cast<size_t>(old_generation) + young_generation;

}

void Heap::GenerationSizesFromHeapSize(size_t heap_size,

size_t* young_generation_size,

size_t* old_generation_size) {

// Initialize values for the case when the given heap size is too small.

*young_generation_size = 0;

*old_generation_size = 0;

// Binary search for the largest old generation size that fits to the given

// heap limit considering the correspondingly sized young generation.

size_t lower = 0, upper = heap_size;

while (lower + 1 < upper) {

size_t old_generation = lower + (upper - lower) / 2;

size_t young_generation =

YoungGenerationSizeFromOldGenerationSize(old_generation);

if (old_generation + young_generation <= heap_size) {

// This size configuration fits into the given heap limit.

*young_generation_size = young_generation;

*old_generation_size = old_generation;

lower = old_generation;

} else {

upper = old_generation;

}

}

}

size_t Heap::MinYoungGenerationSize() {

return YoungGenerationSizeFromSemiSpaceSize(DefaultMinSemiSpaceSize());

}

size_t Heap::MinOldGenerationSize() {

size_t paged_space_count =

LAST_GROWABLE_PAGED_SPACE - FIRST_GROWABLE_PAGED_SPACE + 1;

return paged_space_count * PageMetadata::kPageSize;

}

size_t Heap::AllocatorLimitOnMaxOldGenerationSize() {

#ifdef V8_COMPRESS_POINTERS

// Isolate and the young generation are also allocated on the heap.

return kPtrComprCageReservationSize -

YoungGenerationSizeFromSemiSpaceSize(DefaultMaxSemiSpaceSize()) -

RoundUp(sizeof(Isolate), size_t{1} << kPageSizeBits);

#else

return std::numeric_limits<size_t>::max();

#endif

}

size_t Heap::MaxOldGenerationSize(uint64_t physical_memory) {

size_t max_size = V8HeapTrait::kMaxSize;

// Increase the heap size from 2GB to 4GB for 64-bit systems with physical

// memory at least 16GB. The threshold is set to 15GB to accommodate for some

// memory being reserved by the hardware.

#ifdef V8_HOST_ARCH_64_BIT

if ((physical_memory / GB) >= 15) {

#if V8_OS_ANDROID

// As of 2024, Android devices with 16GiB are shipping (for instance the

// Pixel 9 Pro). However, a large fraction of their memory is not usable,

// and there is no disk swap, so heaps are still smaller than on desktop for

// now.

DCHECK_EQ(max_size / GB, 1u);

#else

DCHECK_EQ(max_size / GB, 2u);

#endif

max_size *= 2;

}

#endif // V8_HOST_ARCH_64_BIT

return std::min(max_size, AllocatorLimitOnMaxOldGenerationSize());

}

namespace {

int NumberOfSemiSpaces() { return v8_flags.minor_ms ? 1 : 2; }

} // namespace

size_t Heap::YoungGenerationSizeFromSemiSpaceSize(size_t semi_space_size) {

return semi_space_size *

(NumberOfSemiSpaces() + kNewLargeObjectSpaceToSemiSpaceRatio);

}

size_t Heap::SemiSpaceSizeFromYoungGenerationSize(

size_t young_generation_size) {

return young_generation_size /

(NumberOfSemiSpaces() + kNewLargeObjectSpaceToSemiSpaceRatio);

}

size_t Heap::Capacity() {

if (!HasBeenSetUp()) {

return 0;

}

return NewSpaceCapacity() + OldGenerationCapacity();

}

size_t Heap::OldGenerationCapacity() const {

if (!HasBeenSetUp()) return 0;

PagedSpaceIterator spaces(this);

size_t total = 0;

for (PagedSpace* space = spaces.Next(); space != nullptr;

space = spaces.Next()) {

total += space->Capacity();

}

if (shared_lo_space_) {

total += shared_lo_space_->SizeOfObjects();

}

return total + lo_space_->SizeOfObjects() + code_lo_space_->SizeOfObjects() +

trusted_lo_space_->SizeOfObjects();

}

size_t Heap::CommittedOldGenerationMemory() {

if (!HasBeenSetUp()) return 0;

PagedSpaceIterator spaces(this);

size_t total = 0;

for (PagedSpace* space = spaces.Next(); space != nullptr;

space = spaces.Next()) {

total += space->CommittedMemory();

}

if (shared_lo_space_) {

total += shared_lo_space_->Size();

}

return total + lo_space_->Size() + code_lo_space_->Size() +

trusted_lo_space_->Size();

}

size_t Heap::CommittedMemory() {

if (!HasBeenSetUp()) return 0;

size_t new_space_committed = new_space_ ? new_space_->CommittedMemory() : 0;

size_t new_lo_space_committed = new_lo_space_ ? new_lo_space_->Size() : 0;

return new_space_committed + new_lo_space_committed +

CommittedOldGenerationMemory();

}

size_t Heap::CommittedPhysicalMemory() {

if (!HasBeenSetUp()) return 0;

size_t total = 0;

for (SpaceIterator it(this); it.HasNext();) {

total += it.Next()->CommittedPhysicalMemory();

}

return total;

}

size_t Heap::CommittedMemoryExecutable() {

if (!HasBeenSetUp()) return 0;

return static_cast<size_t>(memory_allocator()->SizeExecutable());

}

void Heap::UpdateMaximumCommitted() {

if (!HasBeenSetUp()) return;

const size_t current_committed_memory = CommittedMemory();

if (current_committed_memory > maximum_committed_) {

maximum_committed_ = current_committed_memory;

}

}

size_t Heap::Available() {

if (!HasBeenSetUp()) return 0;

size_t total = 0;

for (SpaceIterator it(this); it.HasNext();) {

total += it.Next()->Available();

}

total += memory_allocator()->Available();

return total;

}

bool Heap::CanExpandOldGeneration(size_t size) const {

if (force_oom_ || force_gc_on_next_allocation_) return false;

if (OldGenerationCapacity() + size > max_old_generation_size()) return false;

// Stay below `MaxReserved()` such that it is more likely that committing the

// second semi space at the beginning of a GC succeeds.

return memory_allocator()->Size() + size <= MaxReserved();

}

bool Heap::IsOldGenerationExpansionAllowed(

size_t size, const base::MutexGuard& expansion_mutex_witness) const {

return OldGenerationCapacity() + size <= max_old_generation_size();

}

bool Heap::CanPromoteYoungAndExpandOldGeneration(size_t size) const {

if (v8_flags.sticky_mark_bits) {

DCHECK_NULL(new_space());

size_t new_space_capacity =

sticky_space()->Capacity() - sticky_space()->young_objects_size();

size_t new_lo_space_capacity = new_lo_space_ ? new_lo_space_->Size() : 0;

return CanExpandOldGeneration(size + new_space_capacity +

new_lo_space_capacity);

}

if (!new_space()) {

DCHECK_NULL(new_lo_space());

return CanExpandOldGeneration(size);

}

size_t new_space_capacity =

new_space()->Capacity() + new_lo_space()->Size() +

(v8_flags.minor_ms ? 0

: semi_space_new_space()->QuarantinedPageCount() *

PageMetadata::kPageSize);

// Over-estimate the new space size using capacity to allow some slack.

return CanExpandOldGeneration(size + new_space_capacity);

}

bool Heap::HasBeenSetUp() const {

// We will always have an old space when the heap is set up.

return old_space_ != nullptr;

}

bool Heap::ShouldUseBackgroundThreads() const {

return !v8_flags.single_threaded_gc_in_background ||

!isolate()->EfficiencyModeEnabled();

}

bool Heap::ShouldUseIncrementalMarking() const {

if (v8_flags.single_threaded_gc_in_background &&

isolate()->EfficiencyModeEnabled()) {

return v8_flags.incremental_marking_for_gc_in_background;

} else {

return true;

}

}

bool Heap::ShouldOptimizeForBattery() const {

return v8_flags.optimize_gc_for_battery ||

isolate()->BatterySaverModeEnabled();

}

GarbageCollector Heap::SelectGarbageCollector(AllocationSpace space,

GarbageCollectionReason gc_reason,

const char** reason) const {

if (gc_reason == GarbageCollectionReason::kFinalizeMinorMSForMajorGC) {

DCHECK_NE(static_cast<bool>(new_space()),

v8_flags.sticky_mark_bits.value());

DCHECK(!ShouldReduceMemory());

*reason = "MinorMS finalization for starting major GC";

return GarbageCollector::MINOR_MARK_SWEEPER;

}

if (gc_reason == GarbageCollectionReason::kFinalizeConcurrentMinorMS) {

DCHECK_NE(static_cast<bool>(new_space()),

v8_flags.sticky_mark_bits.value());

DCHECK(!ShouldReduceMemory());

*reason = "Concurrent MinorMS needs finalization";

return GarbageCollector::MINOR_MARK_SWEEPER;

}

// Is global GC requested?

if (space != NEW_SPACE && space != NEW_LO_SPACE) {

isolate_->counters()->gc_compactor_caused_by_request()->Increment();

*reason = "GC in old space requested";

return GarbageCollector::MARK_COMPACTOR;

}

if (v8_flags.gc_global || ShouldStressCompaction() || !use_new_space()) {

*reason = "GC in old space forced by flags";

return GarbageCollector::MARK_COMPACTOR;

}

if (incremental_marking()->IsMajorMarking()) {

*reason = "Incremental marking forced finalization";

return GarbageCollector::MARK_COMPACTOR;

}

if (!CanPromoteYoungAndExpandOldGeneration(0)) {

isolate_->counters()

->gc_compactor_caused_by_oldspace_exhaustion()

->Increment();

*reason = "scavenge might not succeed";

return GarbageCollector::MARK_COMPACTOR;

}

DCHECK(!v8_flags.single_generation);

DCHECK(!v8_flags.gc_global);

// Default

*reason = nullptr;

return YoungGenerationCollector();

}

void Heap::SetGCState(HeapState state) {

gc_state_.store(state, std::memory_order_relaxed);

}

bool Heap::IsGCWithMainThreadStack() const {

return embedder_stack_state_ == StackState::kMayContainHeapPointers;

}

bool Heap::IsGCWithStack() const {

return IsGCWithMainThreadStack() || stack().HasBackgroundStacks();

}

bool Heap::CanShortcutStringsDuringGC(GarbageCollector collector) const {

if (!v8_flags.shortcut_strings_with_stack && IsGCWithStack()) return false;

switch (collector) {

case GarbageCollector::MINOR_MARK_SWEEPER:

if (!v8_flags.minor_ms_shortcut_strings) return false;

DCHECK(!incremental_marking()->IsMajorMarking());

// Minor MS cannot short cut strings during concurrent marking.

if (incremental_marking()->IsMinorMarking()) return false;

// Minor MS uses static roots to check for strings to shortcut.

if (!V8_STATIC_ROOTS_BOOL) return false;

break;

case GarbageCollector::SCAVENGER:

// Scavenger cannot short cut strings during incremental marking.

DCHECK(!incremental_marking()->IsMajorMarking());

if (isolate()->has_shared_space() &&

!isolate()->is_shared_space_isolate() &&

isolate()

->shared_space_isolate()

->heap()

->incremental_marking()

->IsMarking()) {

DCHECK(isolate()

->shared_space_isolate()

->heap()

->incremental_marking()

->IsMajorMarking());

return false;

}

break;

default:

UNREACHABLE();

}

return true;

}

void Heap::PrintShortHeapStatistics() {

if (!v8_flags.trace_gc_verbose) return;

PrintIsolate(isolate_,

"Memory allocator, used: %6zu KB"

", available: %7zu KB\n",

memory_allocator()->Size() / KB,

memory_allocator()->Available() / KB);

PrintIsolate(isolate_,

"Read-only space, used: %6zu KB"

", available: %7zu KB"

", committed: %6zu KB\n",

read_only_space_->Size() / KB, size_t{0},

read_only_space_->CommittedMemory() / KB);

PrintIsolate(isolate_,

"New space, used: %6zu KB"

", available:%c %7zu KB"

", committed: %6zu KB\n",

NewSpaceSize() / KB,

(v8_flags.minor_ms && minor_sweeping_in_progress()) ? '*' : ' ',

new_space_->Available() / KB,

new_space_->CommittedMemory() / KB);

PrintIsolate(isolate_,

"New large object space, used: %6zu KB"

", available: %7zu KB"

", committed: %6zu KB\n",

new_lo_space_->SizeOfObjects() / KB,

new_lo_space_->Available() / KB,

new_lo_space_->CommittedMemory() / KB);

PrintIsolate(isolate_,

"Old space, used: %6zu KB"

", available:%c %7zu KB"

", committed: %6zu KB\n",

old_space_->SizeOfObjects() / KB,

major_sweeping_in_progress() ? '*' : ' ',

old_space_->Available() / KB,

old_space_->CommittedMemory() / KB);

PrintIsolate(isolate_,

"Code space, used: %6zu KB"

", available:%c %7zu KB"

", committed: %6zu KB\n",

code_space_->SizeOfObjects() / KB,

major_sweeping_in_progress() ? '*' : ' ',

code_space_->Available() / KB,

code_space_->CommittedMemory() / KB);

PrintIsolate(isolate_,

"Large object space, used: %6zu KB"

", available: %7zu KB"

", committed: %6zu KB\n",

lo_space_->SizeOfObjects() / KB, lo_space_->Available() / KB,

lo_space_->CommittedMemory() / KB);

PrintIsolate(isolate_,

"Code large object space, used: %6zu KB"

", available: %7zu KB"

", committed: %6zu KB\n",

code_lo_space_->SizeOfObjects() / KB,

code_lo_space_->Available() / KB,

code_lo_space_->CommittedMemory() / KB);

PrintIsolate(isolate_,

"Trusted space, used: %6zu KB"

", available:%c %7zu KB"

", committed: %6zu KB\n",

trusted_space_->SizeOfObjects() / KB,

major_sweeping_in_progress() ? '*' : ' ',

trusted_space_->Available() / KB,

trusted_space_->CommittedMemory() / KB);

PrintIsolate(isolate_,

"Trusted large object space, used: %6zu KB"

", available: %7zu KB"

", committed: %6zu KB\n",

trusted_lo_space_->SizeOfObjects() / KB,

trusted_lo_space_->Available() / KB,

trusted_lo_space_->CommittedMemory() / KB);

ReadOnlySpace* const ro_space = read_only_space_;

PrintIsolate(isolate_,

"All spaces, used: %6zu KB"

", available:%c %7zu KB"

", committed: %6zu KB\n",

(this->SizeOfObjects() + ro_space->Size()) / KB,

sweeping_in_progress() ? '*' : ' ', (this->Available()) / KB,

(this->CommittedMemory() + ro_space->CommittedMemory()) / KB);

const size_t chunks = memory_allocator()->GetPooledChunksCount();

PrintIsolate(isolate_, "Pool buffering %4zu chunk(s) of committed: %7zu KB\n",

chunks, (chunks * PageMetadata::kPageSize) / KB);

PrintIsolate(isolate_,

"External memory reported: %7" PRId64 " KB\n",

external_memory() / KB);

PrintIsolate(isolate_,

"Backing store memory: %7" PRIu64 " KB\n",

backing_store_bytes() / KB);

PrintIsolate(isolate_, "External memory global: %7zu KB\n",

external_memory_callback_() / KB);

PrintIsolate(isolate_,

"Total time spent in GC: %7.1f ms\n",

total_gc_time_ms_.InMillisecondsF());

if (sweeping_in_progress()) {

PrintIsolate(isolate_,

"(*) Sweeping is still in progress, making available sizes "

"inaccurate.\n");

}

}

void Heap::PrintFreeListsStats() {

DCHECK(v8_flags.trace_gc_freelists);

if (v8_flags.trace_gc_freelists_verbose) {

PrintIsolate(isolate_,

"Freelists statistics per Page: "

"[category: length || total free bytes]\n");

}

std::vector<int> categories_lengths(

old_space()->free_list()->number_of_categories(), 0);

std::vector<size_t> categories_sums(

old_space()->free_list()->number_of_categories(), 0);

unsigned int pageCnt = 0;

// This loops computes freelists lengths and sum.

// If v8_flags.trace_gc_freelists_verbose is enabled, it also prints

// the stats of each FreeListCategory of each Page.

for (PageMetadata* page : *old_space()) {

std::ostringstream out_str;

if (v8_flags.trace_gc_freelists_verbose) {

out_str << "Page " << std::setw(4) << pageCnt;

}

for (int cat = kFirstCategory;

cat <= old_space()->free_list()->last_category(); cat++) {

FreeListCategory* free_list =

page->free_list_category(static_cast<FreeListCategoryType>(cat));

int length = free_list->FreeListLength();

size_t sum = free_list->SumFreeList();

if (v8_flags.trace_gc_freelists_verbose) {

out_str << "[" << cat << ": " << std::setw(4) << length << " || "

<< std::setw(6) << sum << " ]"

<< (cat == old_space()->free_list()->last_category() ? "\n"

: ", ");

}

categories_lengths[cat] += length;

categories_sums[cat] += sum;

}

if (v8_flags.trace_gc_freelists_verbose) {

PrintIsolate(isolate_, "%s", out_str.str().c_str());

}

pageCnt++;

}

// Print statistics about old_space (pages, free/wasted/used memory...).

PrintIsolate(

isolate_,

"%d pages. Free space: %.1f MB (waste: %.2f). "

"Usage: %.1f/%.1f (MB) -> %.2f%%.\n",

pageCnt, static_cast<double>(old_space_->Available()) / MB,

static_cast<double>(old_space_->Waste()) / MB,

static_cast<double>(old_space_->Size()) / MB,

static_cast<double>(old_space_->Capacity()) / MB,

static_cast<double>(old_space_->Size()) / old_space_->Capacity() * 100);

// Print global statistics of each FreeListCategory (length & sum).

PrintIsolate(isolate_,

"FreeLists global statistics: "

"[category: length || total free KB]\n");

std::ostringstream out_str;

for (int cat = kFirstCategory;

cat <= old_space()->free_list()->last_category(); cat++) {

out_str << "[" << cat << ": " << categories_lengths[cat] << " || "

<< std::fixed << std::setprecision(2)

<< static_cast<double>(categories_sums[cat]) / KB << " KB]"

<< (cat == old_space()->free_list()->last_category() ? "\n" : ", ");

}

PrintIsolate(isolate_, "%s", out_str.str().c_str());

}

void Heap::DumpJSONHeapStatistics(std::stringstream& stream) {

HeapStatistics stats;

reinterpret_cast<v8::Isolate*>(isolate())->GetHeapStatistics(&stats);

// clang-format off

#define DICT(s) "{" << s << "}"

#define LIST(s) "[" << s << "]"

#define QUOTE(s) "\"" << s << "\""

#define MEMBER(s) QUOTE(s) << ":"

auto SpaceStatistics = [this](int space_index) {

HeapSpaceStatistics space_stats;

reinterpret_cast<v8::Isolate*>(isolate())->GetHeapSpaceStatistics(

&space_stats, space_index);

std::stringstream stream;

stream << DICT(

MEMBER("name")

<< QUOTE(ToString(

static_cast<AllocationSpace>(space_index)))

<< ","

MEMBER("size") << space_stats.space_size() << ","

MEMBER("used_size") << space_stats.space_used_size() << ","

MEMBER("available_size") << space_stats.space_available_size() << ","

MEMBER("physical_size") << space_stats.physical_space_size());

return stream.str();

};

stream << DICT(

MEMBER("isolate") << QUOTE(reinterpret_cast<void*>(isolate())) << ","

MEMBER("id") << gc_count() << ","

MEMBER("time_ms") << isolate()->time_millis_since_init() << ","

MEMBER("total_heap_size") << stats.total_heap_size() << ","

MEMBER("total_heap_size_executable")

<< stats.total_heap_size_executable() << ","

MEMBER("total_physical_size") << stats.total_physical_size() << ","

MEMBER("total_available_size") << stats.total_available_size() << ","

MEMBER("used_heap_size") << stats.used_heap_size() << ","

MEMBER("heap_size_limit") << stats.heap_size_limit() << ","

MEMBER("malloced_memory") << stats.malloced_memory() << ","

MEMBER("external_memory") << stats.external_memory() << ","

MEMBER("peak_malloced_memory") << stats.peak_malloced_memory() << ","

MEMBER("spaces") << LIST(

SpaceStatistics(RO_SPACE) << "," <<

SpaceStatistics(NEW_SPACE) << "," <<

SpaceStatistics(OLD_SPACE) << "," <<

SpaceStatistics(CODE_SPACE) << "," <<

SpaceStatistics(LO_SPACE) << "," <<

SpaceStatistics(CODE_LO_SPACE) << "," <<

SpaceStatistics(NEW_LO_SPACE) << "," <<

SpaceStatistics(TRUSTED_SPACE) << "," <<

SpaceStatistics(TRUSTED_LO_SPACE)));

#undef DICT

#undef LIST

#undef QUOTE

#undef MEMBER

// clang-format on

}

void Heap::ReportStatisticsAfterGC() {

if (deferred_counters_.empty()) return;

// Move the contents into a new SmallVector first, in case

// {Isolate::CountUsage} puts the counters into {deferred_counters_} again.

decltype(deferred_counters_) to_report = std::move(deferred_counters_);

DCHECK(deferred_counters_.empty());

isolate()->CountUsage(base::VectorOf(to_report));

}

class Heap::AllocationTrackerForDebugging final

: public HeapObjectAllocationTracker {

public:

static bool IsNeeded() {

return v8_flags.verify_predictable || v8_flags.fuzzer_gc_analysis ||

(v8_flags.trace_allocation_stack_interval > 0);

}

explicit AllocationTrackerForDebugging(Heap* heap) : heap_(heap) {

CHECK(IsNeeded());

heap_->AddHeapObjectAllocationTracker(this);

}

~AllocationTrackerForDebugging() final {

heap_->RemoveHeapObjectAllocationTracker(this);

if (v8_flags.verify_predictable || v8_flags.fuzzer_gc_analysis) {

PrintAllocationsHash();

}

}

void AllocationEvent(Address addr, int size) final {

if (v8_flags.verify_predictable) {

allocations_count_.fetch_add(1, std::memory_order_relaxed);

// Advance synthetic time by making a time request.

heap_->MonotonicallyIncreasingTimeInMs();

UpdateAllocationsHash(HeapObject::FromAddress(addr));

UpdateAllocationsHash(size);

if (allocations_count_ % v8_flags.dump_allocations_digest_at_alloc == 0) {

PrintAllocationsHash();

}

} else if (v8_flags.fuzzer_gc_analysis) {

allocations_count_.fetch_add(1, std::memory_order_relaxed);

} else if (v8_flags.trace_allocation_stack_interval > 0) {

allocations_count_.fetch_add(1, std::memory_order_relaxed);

if (allocations_count_ % v8_flags.trace_allocation_stack_interval == 0) {

heap_->isolate()->PrintStack(stdout, Isolate::kPrintStackConcise);

}

}

}

void MoveEvent(Address source, Address target, int size) final {

if (v8_flags.verify_predictable) {

allocations_count_.fetch_add(1, std::memory_order_relaxed);

// Advance synthetic time by making a time request.

heap_->MonotonicallyIncreasingTimeInMs();

UpdateAllocationsHash(HeapObject::FromAddress(source));

UpdateAllocationsHash(HeapObject::FromAddress(target));

UpdateAllocationsHash(size);

if (allocations_count_ % v8_flags.dump_allocations_digest_at_alloc == 0) {

PrintAllocationsHash();

}

} else if (v8_flags.fuzzer_gc_analysis) {

allocations_count_.fetch_add(1, std::memory_order_relaxed);

}

}

void UpdateObjectSizeEvent(Address, int) final {}

private:

void UpdateAllocationsHash(Tagged<HeapObject> object) {

Address object_address = object.address();

MemoryChunk* memory_chunk = MemoryChunk::FromAddress(object_address);

AllocationSpace allocation_space =

MutablePageMetadata::cast(memory_chunk->Metadata())->owner_identity();

static_assert(kSpaceTagSize + kPageSizeBits <= 32);

uint32_t value =

static_cast<uint32_t>(memory_chunk->Offset(object_address)) |

(static_cast<uint32_t>(allocation_space) << kPageSizeBits);

UpdateAllocationsHash(value);

}

void UpdateAllocationsHash(uint32_t value) {

const uint16_t c1 = static_cast<uint16_t>(value);

const uint16_t c2 = static_cast<uint16_t>(value >> 16);

raw_allocations_hash_.AddCharacter(c1);

raw_allocations_hash_.AddCharacter(c2);

}

void PrintAllocationsHash() {

uint32_t hash = raw_allocations_hash_.Finalize();

PrintF("\n### Allocations = %zu, hash = 0x%08x\n",

allocations_count_.load(std::memory_order_relaxed), hash);

}

Heap* const heap_;

// Count of all allocations performed through C++ bottlenecks. This needs to

// be atomic as objects are moved in parallel in the GC which counts as

// allocations.

std::atomic<size_t> allocations_count_{0};

// Running hash over allocations performed.

RunningStringHasher raw_allocations_hash_{0};

};

void Heap::AddHeapObjectAllocationTracker(

HeapObjectAllocationTracker* tracker) {

if (allocation_trackers_.empty() && v8_flags.inline_new) {

DisableInlineAllocation();

}

allocation_trackers_.push_back(tracker);

if (allocation_trackers_.size() == 1) {

isolate_->UpdateLogObjectRelocation();

}

}

void Heap::RemoveHeapObjectAllocationTracker(

HeapObjectAllocationTracker* tracker) {

allocation_trackers_.erase(std::remove(allocation_trackers_.begin(),

allocation_trackers_.end(), tracker),

allocation_trackers_.end());

if (allocation_trackers_.empty()) {

isolate_->UpdateLogObjectRelocation();

}

if (allocation_trackers_.empty() && v8_flags.inline_new) {

EnableInlineAllocation();

}

}

void Heap::IncrementDeferredCounts(

base::Vector<const v8::Isolate::UseCounterFeature> features) {

deferred_counters_.insert(deferred_counters_.end(), features.begin(),

features.end());

}

void Heap::GarbageCollectionPrologue(

GarbageCollectionReason gc_reason,

const v8::GCCallbackFlags gc_callback_flags) {

TRACE_GC(tracer(), GCTracer::Scope::HEAP_PROLOGUE);

is_current_gc_forced_ = gc_callback_flags & v8::kGCCallbackFlagForced ||

current_gc_flags_ & GCFlag::kForced ||

force_gc_on_next_allocation_;

is_current_gc_for_heap_profiler_ =

gc_reason == GarbageCollectionReason::kHeapProfiler;

if (force_gc_on_next_allocation_) force_gc_on_next_allocation_ = false;

#ifdef V8_ENABLE_ALLOCATION_TIMEOUT

heap_allocator_->UpdateAllocationTimeout();

#endif // V8_ENABLE_ALLOCATION_TIMEOUT

if (minor_gc_job()) {

DCHECK(use_new_space());

minor_gc_job()->CancelTaskIfScheduled();

}

// Reset GC statistics.

promoted_objects_size_ = 0;

previous_new_space_surviving_object_size_ = new_space_surviving_object_size_;

new_space_surviving_object_size_ = 0;

nodes_died_in_new_space_ = 0;

nodes_copied_in_new_space_ = 0;

nodes_promoted_ = 0;

UpdateMaximumCommitted();

#ifdef DEBUG

DCHECK(!AllowGarbageCollection::IsAllowed());

DCHECK_EQ(gc_state(), NOT_IN_GC);

if (v8_flags.gc_verbose) Print();

#endif // DEBUG

}

void Heap::GarbageCollectionPrologueInSafepoint() {

TRACE_GC(tracer(), GCTracer::Scope::HEAP_PROLOGUE_SAFEPOINT);

gc_count_++;

new_space_allocation_counter_ = NewSpaceAllocationCounter();

}

size_t Heap::NewSpaceAllocationCounter() const {

size_t counter = new_space_allocation_counter_;

if (new_space_) {

DCHECK(!allocator()->new_space_allocator()->IsLabValid());

counter += new_space()->AllocatedSinceLastGC();

}

return counter;