/*

* Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.

* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

*

* This code is free software; you can redistribute it and/or modify it

* under the terms of the GNU General Public License version 2 only, as

* published by the Free Software Foundation.

*

* This code is distributed in the hope that it will be useful, but WITHOUT

* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License

* version 2 for more details (a copy is included in the LICENSE file that

* accompanied this code).

*

* You should have received a copy of the GNU General Public License version

* 2 along with this work; if not, write to the Free Software Foundation,

* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

*

* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

* or visit www.oracle.com if you need additional information or have any

* questions.

*

*/

#include "precompiled.hpp"

#include "jvm_io.h"

#include "code/codeBlob.hpp"

#include "code/codeCache.hpp"

#include "code/codeHeapState.hpp"

#include "code/compiledIC.hpp"

#include "code/dependencies.hpp"

#include "code/dependencyContext.hpp"

#include "code/icBuffer.hpp"

#include "code/nmethod.hpp"

#include "code/pcDesc.hpp"

#include "compiler/compilationPolicy.hpp"

#include "compiler/compileBroker.hpp"

#include "compiler/oopMap.hpp"

#include "gc/shared/collectedHeap.hpp"

#include "jfr/jfrEvents.hpp"

#include "logging/log.hpp"

#include "logging/logStream.hpp"

#include "memory/allocation.inline.hpp"

#include "memory/iterator.hpp"

#include "memory/resourceArea.hpp"

#include "memory/universe.hpp"

#include "oops/method.inline.hpp"

#include "oops/objArrayOop.hpp"

#include "oops/oop.inline.hpp"

#include "oops/verifyOopClosure.hpp"

#include "runtime/arguments.hpp"

#include "runtime/atomic.hpp"

#include "runtime/deoptimization.hpp"

#include "runtime/globals_extension.hpp"

#include "runtime/handles.inline.hpp"

#include "runtime/icache.hpp"

#include "runtime/java.hpp"

#include "runtime/mutexLocker.hpp"

#include "runtime/safepointVerifiers.hpp"

#include "runtime/sweeper.hpp"

#include "runtime/vmThread.hpp"

#include "services/memoryService.hpp"

#include "utilities/align.hpp"

#include "utilities/vmError.hpp"

#include "utilities/xmlstream.hpp"

#ifdef COMPILER1

#include "c1/c1_Compilation.hpp"

#include "c1/c1_Compiler.hpp"

#endif

#ifdef COMPILER2

#include "opto/c2compiler.hpp"

#include "opto/compile.hpp"

#include "opto/node.hpp"

#endif

// Helper class for printing in CodeCache

class CodeBlob_sizes {

private:

int count;

int total_size;

int header_size;

int code_size;

int stub_size;

int relocation_size;

int scopes_oop_size;

int scopes_metadata_size;

int scopes_data_size;

int scopes_pcs_size;

public:

CodeBlob_sizes() {

count = 0;

total_size = 0;

header_size = 0;

code_size = 0;

stub_size = 0;

relocation_size = 0;

scopes_oop_size = 0;

scopes_metadata_size = 0;

scopes_data_size = 0;

scopes_pcs_size = 0;

}

int total() { return total_size; }

bool is_empty() { return count == 0; }

void print(const char* title) {

tty->print_cr(" #%d %s = %dK (hdr %d%%, loc %d%%, code %d%%, stub %d%%, [oops %d%%, metadata %d%%, data %d%%, pcs %d%%])",

count,

title,

(int)(total() / K),

header_size * 100 / total_size,

relocation_size * 100 / total_size,

code_size * 100 / total_size,

stub_size * 100 / total_size,

scopes_oop_size * 100 / total_size,

scopes_metadata_size * 100 / total_size,

scopes_data_size * 100 / total_size,

scopes_pcs_size * 100 / total_size);

}

void add(CodeBlob* cb) {

count++;

total_size += cb->size();

header_size += cb->header_size();

relocation_size += cb->relocation_size();

if (cb->is_nmethod()) {

nmethod* nm = cb->as_nmethod_or_null();

code_size += nm->insts_size();

stub_size += nm->stub_size();

scopes_oop_size += nm->oops_size();

scopes_metadata_size += nm->metadata_size();

scopes_data_size += nm->scopes_data_size();

scopes_pcs_size += nm->scopes_pcs_size();

} else {

code_size += cb->code_size();

}

}

};

// Iterate over all CodeHeaps

#define FOR_ALL_HEAPS(heap) for (GrowableArrayIterator<CodeHeap*> heap = _heaps->begin(); heap != _heaps->end(); ++heap)

#define FOR_ALL_NMETHOD_HEAPS(heap) for (GrowableArrayIterator<CodeHeap*> heap = _nmethod_heaps->begin(); heap != _nmethod_heaps->end(); ++heap)

#define FOR_ALL_ALLOCABLE_HEAPS(heap) for (GrowableArrayIterator<CodeHeap*> heap = _allocable_heaps->begin(); heap != _allocable_heaps->end(); ++heap)

// Iterate over all CodeBlobs (cb) on the given CodeHeap

#define FOR_ALL_BLOBS(cb, heap) for (CodeBlob* cb = first_blob(heap); cb != NULL; cb = next_blob(heap, cb))

address CodeCache::_low_bound = 0;

address CodeCache::_high_bound = 0;

int CodeCache::_number_of_nmethods_with_dependencies = 0;

ExceptionCache* volatile CodeCache::_exception_cache_purge_list = NULL;

// Initialize arrays of CodeHeap subsets

GrowableArray<CodeHeap*>* CodeCache::_heaps = new(ResourceObj::C_HEAP, mtCode) GrowableArray<CodeHeap*> (CodeBlobType::All, mtCode);

GrowableArray<CodeHeap*>* CodeCache::_compiled_heaps = new(ResourceObj::C_HEAP, mtCode) GrowableArray<CodeHeap*> (CodeBlobType::All, mtCode);

GrowableArray<CodeHeap*>* CodeCache::_nmethod_heaps = new(ResourceObj::C_HEAP, mtCode) GrowableArray<CodeHeap*> (CodeBlobType::All, mtCode);

GrowableArray<CodeHeap*>* CodeCache::_allocable_heaps = new(ResourceObj::C_HEAP, mtCode) GrowableArray<CodeHeap*> (CodeBlobType::All, mtCode);

void CodeCache::check_heap_sizes(size_t non_nmethod_size, size_t profiled_size, size_t non_profiled_size, size_t cache_size, bool all_set) {

size_t total_size = non_nmethod_size + profiled_size + non_profiled_size;

// Prepare error message

const char* error = "Invalid code heap sizes";

err_msg message("NonNMethodCodeHeapSize (" SIZE_FORMAT "K) + ProfiledCodeHeapSize (" SIZE_FORMAT "K)"

" + NonProfiledCodeHeapSize (" SIZE_FORMAT "K) = " SIZE_FORMAT "K",

non_nmethod_size/K, profiled_size/K, non_profiled_size/K, total_size/K);

if (total_size > cache_size) {

// Some code heap sizes were explicitly set: total_size must be <= cache_size

message.append(" is greater than ReservedCodeCacheSize (" SIZE_FORMAT "K).", cache_size/K);

vm_exit_during_initialization(error, message);

} else if (all_set && total_size != cache_size) {

// All code heap sizes were explicitly set: total_size must equal cache_size

message.append(" is not equal to ReservedCodeCacheSize (" SIZE_FORMAT "K).", cache_size/K);

vm_exit_during_initialization(error, message);

}

}

void CodeCache::initialize_heaps() {

bool non_nmethod_set = FLAG_IS_CMDLINE(NonNMethodCodeHeapSize);

bool profiled_set = FLAG_IS_CMDLINE(ProfiledCodeHeapSize);

bool non_profiled_set = FLAG_IS_CMDLINE(NonProfiledCodeHeapSize);

size_t min_size = os::vm_page_size();

size_t cache_size = ReservedCodeCacheSize;

size_t non_nmethod_size = NonNMethodCodeHeapSize;

size_t profiled_size = ProfiledCodeHeapSize;

size_t non_profiled_size = NonProfiledCodeHeapSize;

// Check if total size set via command line flags exceeds the reserved size

check_heap_sizes((non_nmethod_set ? non_nmethod_size : min_size),

(profiled_set ? profiled_size : min_size),

(non_profiled_set ? non_profiled_size : min_size),

cache_size,

non_nmethod_set && profiled_set && non_profiled_set);

// Determine size of compiler buffers

size_t code_buffers_size = 0;

#ifdef COMPILER1

// C1 temporary code buffers (see Compiler::init_buffer_blob())

const int c1_count = CompilationPolicy::c1_count();

code_buffers_size += c1_count * Compiler::code_buffer_size();

#endif

#ifdef COMPILER2

// C2 scratch buffers (see Compile::init_scratch_buffer_blob())

const int c2_count = CompilationPolicy::c2_count();

// Initial size of constant table (this may be increased if a compiled method needs more space)

code_buffers_size += c2_count * C2Compiler::initial_code_buffer_size();

#endif

// Increase default non_nmethod_size to account for compiler buffers

if (!non_nmethod_set) {

non_nmethod_size += code_buffers_size;

}

// Calculate default CodeHeap sizes if not set by user

if (!non_nmethod_set && !profiled_set && !non_profiled_set) {

// Check if we have enough space for the non-nmethod code heap

if (cache_size > non_nmethod_size) {

// Use the default value for non_nmethod_size and one half of the

// remaining size for non-profiled and one half for profiled methods

size_t remaining_size = cache_size - non_nmethod_size;

profiled_size = remaining_size / 2;

non_profiled_size = remaining_size - profiled_size;

} else {

// Use all space for the non-nmethod heap and set other heaps to minimal size

non_nmethod_size = cache_size - 2 * min_size;

profiled_size = min_size;

non_profiled_size = min_size;

}

} else if (!non_nmethod_set || !profiled_set || !non_profiled_set) {

// The user explicitly set some code heap sizes. Increase or decrease the (default)

// sizes of the other code heaps accordingly. First adapt non-profiled and profiled

// code heap sizes and then only change non-nmethod code heap size if still necessary.

intx diff_size = cache_size - (non_nmethod_size + profiled_size + non_profiled_size);

if (non_profiled_set) {

if (!profiled_set) {

// Adapt size of profiled code heap

if (diff_size < 0 && ((intx)profiled_size + diff_size) <= 0) {

// Not enough space available, set to minimum size

diff_size += profiled_size - min_size;

profiled_size = min_size;

} else {

profiled_size += diff_size;

diff_size = 0;

}

}

} else if (profiled_set) {

// Adapt size of non-profiled code heap

if (diff_size < 0 && ((intx)non_profiled_size + diff_size) <= 0) {

// Not enough space available, set to minimum size

diff_size += non_profiled_size - min_size;

non_profiled_size = min_size;

} else {

non_profiled_size += diff_size;

diff_size = 0;

}

} else if (non_nmethod_set) {

// Distribute remaining size between profiled and non-profiled code heaps

diff_size = cache_size - non_nmethod_size;

profiled_size = diff_size / 2;

non_profiled_size = diff_size - profiled_size;

diff_size = 0;

}

if (diff_size != 0) {

// Use non-nmethod code heap for remaining space requirements

assert(!non_nmethod_set && ((intx)non_nmethod_size + diff_size) > 0, "sanity");

non_nmethod_size += diff_size;

}

}

// We do not need the profiled CodeHeap, use all space for the non-profiled CodeHeap

if (!heap_available(CodeBlobType::MethodProfiled)) {

non_profiled_size += profiled_size;

profiled_size = 0;

}

// We do not need the non-profiled CodeHeap, use all space for the non-nmethod CodeHeap

if (!heap_available(CodeBlobType::MethodNonProfiled)) {

non_nmethod_size += non_profiled_size;

non_profiled_size = 0;

}

// Make sure we have enough space for VM internal code

uint min_code_cache_size = CodeCacheMinimumUseSpace DEBUG_ONLY(* 3);

if (non_nmethod_size < min_code_cache_size) {

vm_exit_during_initialization(err_msg(

"Not enough space in non-nmethod code heap to run VM: " SIZE_FORMAT "K < " SIZE_FORMAT "K",

non_nmethod_size/K, min_code_cache_size/K));

}

// Verify sizes and update flag values

assert(non_profiled_size + profiled_size + non_nmethod_size == cache_size, "Invalid code heap sizes");

FLAG_SET_ERGO(NonNMethodCodeHeapSize, non_nmethod_size);

FLAG_SET_ERGO(ProfiledCodeHeapSize, profiled_size);

FLAG_SET_ERGO(NonProfiledCodeHeapSize, non_profiled_size);

// If large page support is enabled, align code heaps according to large

// page size to make sure that code cache is covered by large pages.

const size_t alignment = MAX2(page_size(false, 8), (size_t) os::vm_allocation_granularity());

non_nmethod_size = align_up(non_nmethod_size, alignment);

profiled_size = align_down(profiled_size, alignment);

// Reserve one continuous chunk of memory for CodeHeaps and split it into

// parts for the individual heaps. The memory layout looks like this:

// ---------- high -----------

// Non-profiled nmethods

// Profiled nmethods

// Non-nmethods

// ---------- low ------------

ReservedCodeSpace rs = reserve_heap_memory(cache_size);

ReservedSpace non_method_space = rs.first_part(non_nmethod_size);

ReservedSpace rest = rs.last_part(non_nmethod_size);

ReservedSpace profiled_space = rest.first_part(profiled_size);

ReservedSpace non_profiled_space = rest.last_part(profiled_size);

// Non-nmethods (stubs, adapters, ...)

add_heap(non_method_space, "CodeHeap 'non-nmethods'", CodeBlobType::NonNMethod);

// Tier 2 and tier 3 (profiled) methods

add_heap(profiled_space, "CodeHeap 'profiled nmethods'", CodeBlobType::MethodProfiled);

// Tier 1 and tier 4 (non-profiled) methods and native methods

add_heap(non_profiled_space, "CodeHeap 'non-profiled nmethods'", CodeBlobType::MethodNonProfiled);

}

size_t CodeCache::page_size(bool aligned, size_t min_pages) {

if (os::can_execute_large_page_memory()) {

if (InitialCodeCacheSize < ReservedCodeCacheSize) {

// Make sure that the page size allows for an incremental commit of the reserved space

min_pages = MAX2(min_pages, (size_t)8);

}

return aligned ? os::page_size_for_region_aligned(ReservedCodeCacheSize, min_pages) :

os::page_size_for_region_unaligned(ReservedCodeCacheSize, min_pages);

} else {

return os::vm_page_size();

}

}

ReservedCodeSpace CodeCache::reserve_heap_memory(size_t size) {

// Align and reserve space for code cache

const size_t rs_ps = page_size();

const size_t rs_align = MAX2(rs_ps, (size_t) os::vm_allocation_granularity());

const size_t rs_size = align_up(size, rs_align);

ReservedCodeSpace rs(rs_size, rs_align, rs_ps);

if (!rs.is_reserved()) {

vm_exit_during_initialization(err_msg("Could not reserve enough space for code cache (" SIZE_FORMAT "K)",

rs_size/K));

}

// Initialize bounds

_low_bound = (address)rs.base();

_high_bound = _low_bound + rs.size();

return rs;

}

// Heaps available for allocation

bool CodeCache::heap_available(int code_blob_type) {

if (!SegmentedCodeCache) {

// No segmentation: use a single code heap

return (code_blob_type == CodeBlobType::All);

} else if (Arguments::is_interpreter_only()) {

// Interpreter only: we don't need any method code heaps

return (code_blob_type == CodeBlobType::NonNMethod);

} else if (CompilerConfig::is_c1_profiling()) {

// Tiered compilation: use all code heaps

return (code_blob_type < CodeBlobType::All);

} else {

// No TieredCompilation: we only need the non-nmethod and non-profiled code heap

return (code_blob_type == CodeBlobType::NonNMethod) ||

(code_blob_type == CodeBlobType::MethodNonProfiled);

}

}

const char* CodeCache::get_code_heap_flag_name(int code_blob_type) {

switch(code_blob_type) {

case CodeBlobType::NonNMethod:

return "NonNMethodCodeHeapSize";

break;

case CodeBlobType::MethodNonProfiled:

return "NonProfiledCodeHeapSize";

break;

case CodeBlobType::MethodProfiled:

return "ProfiledCodeHeapSize";

break;

}

ShouldNotReachHere();

return NULL;

}

int CodeCache::code_heap_compare(CodeHeap* const &lhs, CodeHeap* const &rhs) {

if (lhs->code_blob_type() == rhs->code_blob_type()) {

return (lhs > rhs) ? 1 : ((lhs < rhs) ? -1 : 0);

} else {

return lhs->code_blob_type() - rhs->code_blob_type();

}

}

void CodeCache::add_heap(CodeHeap* heap) {

assert(!Universe::is_fully_initialized(), "late heap addition?");

_heaps->insert_sorted<code_heap_compare>(heap);

int type = heap->code_blob_type();

if (code_blob_type_accepts_compiled(type)) {

_compiled_heaps->insert_sorted<code_heap_compare>(heap);

}

if (code_blob_type_accepts_nmethod(type)) {

_nmethod_heaps->insert_sorted<code_heap_compare>(heap);

}

if (code_blob_type_accepts_allocable(type)) {

_allocable_heaps->insert_sorted<code_heap_compare>(heap);

}

}

void CodeCache::add_heap(ReservedSpace rs, const char* name, int code_blob_type) {

// Check if heap is needed

if (!heap_available(code_blob_type)) {

return;

}

// Create CodeHeap

CodeHeap* heap = new CodeHeap(name, code_blob_type);

add_heap(heap);

// Reserve Space

size_t size_initial = MIN2((size_t)InitialCodeCacheSize, rs.size());

size_initial = align_up(size_initial, os::vm_page_size());

if (!heap->reserve(rs, size_initial, CodeCacheSegmentSize)) {

vm_exit_during_initialization(err_msg("Could not reserve enough space in %s (" SIZE_FORMAT "K)",

heap->name(), size_initial/K));

}

// Register the CodeHeap

MemoryService::add_code_heap_memory_pool(heap, name);

}

CodeHeap* CodeCache::get_code_heap_containing(void* start) {

FOR_ALL_HEAPS(heap) {

if ((*heap)->contains(start)) {

return *heap;

}

}

return NULL;

}

CodeHeap* CodeCache::get_code_heap(const CodeBlob* cb) {

assert(cb != NULL, "CodeBlob is null");

FOR_ALL_HEAPS(heap) {

if ((*heap)->contains_blob(cb)) {

return *heap;

}

}

ShouldNotReachHere();

return NULL;

}

CodeHeap* CodeCache::get_code_heap(int code_blob_type) {

FOR_ALL_HEAPS(heap) {

if ((*heap)->accepts(code_blob_type)) {

return *heap;

}

}

return NULL;

}

CodeBlob* CodeCache::first_blob(CodeHeap* heap) {

assert_locked_or_safepoint(CodeCache_lock);

assert(heap != NULL, "heap is null");

return (CodeBlob*)heap->first();

}

CodeBlob* CodeCache::first_blob(int code_blob_type) {

if (heap_available(code_blob_type)) {

return first_blob(get_code_heap(code_blob_type));

} else {

return NULL;

}

}

CodeBlob* CodeCache::next_blob(CodeHeap* heap, CodeBlob* cb) {

assert_locked_or_safepoint(CodeCache_lock);

assert(heap != NULL, "heap is null");

return (CodeBlob*)heap->next(cb);

}

/**

* Do not seize the CodeCache lock here--if the caller has not

* already done so, we are going to lose bigtime, since the code

* cache will contain a garbage CodeBlob until the caller can

* run the constructor for the CodeBlob subclass he is busy

* instantiating.

*/

CodeBlob* CodeCache::allocate(int size, int code_blob_type, bool handle_alloc_failure, int orig_code_blob_type) {

// Possibly wakes up the sweeper thread.

NMethodSweeper::report_allocation(code_blob_type);

assert_locked_or_safepoint(CodeCache_lock);

assert(size > 0, "Code cache allocation request must be > 0 but is %d", size);

if (size <= 0) {

return NULL;

}

CodeBlob* cb = NULL;

// Get CodeHeap for the given CodeBlobType

CodeHeap* heap = get_code_heap(code_blob_type);

assert(heap != NULL, "heap is null");

while (true) {

cb = (CodeBlob*)heap->allocate(size);

if (cb != NULL) break;

if (!heap->expand_by(CodeCacheExpansionSize)) {

// Save original type for error reporting

if (orig_code_blob_type == CodeBlobType::All) {

orig_code_blob_type = code_blob_type;

}

// Expansion failed

if (SegmentedCodeCache) {

// Fallback solution: Try to store code in another code heap.

// NonNMethod -> MethodNonProfiled -> MethodProfiled (-> MethodNonProfiled)

// Note that in the sweeper, we check the reverse_free_ratio of the code heap

// and force stack scanning if less than 10% of the code heap are free.

int type = code_blob_type;

switch (type) {

case CodeBlobType::NonNMethod:

type = CodeBlobType::MethodNonProfiled;

break;

case CodeBlobType::MethodNonProfiled:

type = CodeBlobType::MethodProfiled;

break;

case CodeBlobType::MethodProfiled:

// Avoid loop if we already tried that code heap

if (type == orig_code_blob_type) {

type = CodeBlobType::MethodNonProfiled;

}

break;

}

if (type != code_blob_type && type != orig_code_blob_type && heap_available(type)) {

if (PrintCodeCacheExtension) {

tty->print_cr("Extension of %s failed. Trying to allocate in %s.",

heap->name(), get_code_heap(type)->name());

}

return allocate(size, type, handle_alloc_failure, orig_code_blob_type);

}

}

if (handle_alloc_failure) {

MutexUnlocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);

CompileBroker::handle_full_code_cache(orig_code_blob_type);

}

return NULL;

}

if (PrintCodeCacheExtension) {

ResourceMark rm;

if (_nmethod_heaps->length() >= 1) {

tty->print("%s", heap->name());

} else {

tty->print("CodeCache");

}

tty->print_cr(" extended to [" INTPTR_FORMAT ", " INTPTR_FORMAT "] (" SSIZE_FORMAT " bytes)",

(intptr_t)heap->low_boundary(), (intptr_t)heap->high(),

(address)heap->high() - (address)heap->low_boundary());

}

}

print_trace("allocation", cb, size);

return cb;

}

void CodeCache::free(CodeBlob* cb) {

assert_locked_or_safepoint(CodeCache_lock);

CodeHeap* heap = get_code_heap(cb);

print_trace("free", cb);

if (cb->is_nmethod()) {

nmethod* ptr = (nmethod *)cb;

heap->set_nmethod_count(heap->nmethod_count() - 1);

if (ptr->has_dependencies()) {

_number_of_nmethods_with_dependencies--;

}

ptr->free_native_invokers();

}

if (cb->is_adapter_blob()) {

heap->set_adapter_count(heap->adapter_count() - 1);

}

// Get heap for given CodeBlob and deallocate

get_code_heap(cb)->deallocate(cb);

assert(heap->blob_count() >= 0, "sanity check");

}

void CodeCache::free_unused_tail(CodeBlob* cb, size_t used) {

assert_locked_or_safepoint(CodeCache_lock);

guarantee(cb->is_buffer_blob() && strncmp("Interpreter", cb->name(), 11) == 0, "Only possible for interpreter!");

print_trace("free_unused_tail", cb);

// We also have to account for the extra space (i.e. header) used by the CodeBlob

// which provides the memory (see BufferBlob::create() in codeBlob.cpp).

used += CodeBlob::align_code_offset(cb->header_size());

// Get heap for given CodeBlob and deallocate its unused tail

get_code_heap(cb)->deallocate_tail(cb, used);

// Adjust the sizes of the CodeBlob

cb->adjust_size(used);

}

void CodeCache::commit(CodeBlob* cb) {

// this is called by nmethod::nmethod, which must already own CodeCache_lock

assert_locked_or_safepoint(CodeCache_lock);

CodeHeap* heap = get_code_heap(cb);

if (cb->is_nmethod()) {

heap->set_nmethod_count(heap->nmethod_count() + 1);

if (((nmethod *)cb)->has_dependencies()) {

_number_of_nmethods_with_dependencies++;

}

}

if (cb->is_adapter_blob()) {

heap->set_adapter_count(heap->adapter_count() + 1);

}

// flush the hardware I-cache

ICache::invalidate_range(cb->content_begin(), cb->content_size());

}

bool CodeCache::contains(void *p) {

// S390 uses contains() in current_frame(), which is used before

// code cache initialization if NativeMemoryTracking=detail is set.

S390_ONLY(if (_heaps == NULL) return false;)

// It should be ok to call contains without holding a lock.

FOR_ALL_HEAPS(heap) {

if ((*heap)->contains(p)) {

return true;

}

}

return false;

}

bool CodeCache::contains(nmethod *nm) {

return contains((void *)nm);

}

// This method is safe to call without holding the CodeCache_lock, as long as a dead CodeBlob is not

// looked up (i.e., one that has been marked for deletion). It only depends on the _segmap to contain

// valid indices, which it will always do, as long as the CodeBlob is not in the process of being recycled.

CodeBlob* CodeCache::find_blob(void* start) {

CodeBlob* result = find_blob_unsafe(start);

// We could potentially look up non_entrant methods

guarantee(result == NULL || !result->is_zombie() || result->is_locked_by_vm() || VMError::is_error_reported(), "unsafe access to zombie method");

return result;

}

// Lookup that does not fail if you lookup a zombie method (if you call this, be sure to know

// what you are doing)

CodeBlob* CodeCache::find_blob_unsafe(void* start) {

// NMT can walk the stack before code cache is created

if (_heaps != NULL) {

CodeHeap* heap = get_code_heap_containing(start);

if (heap != NULL) {

return heap->find_blob_unsafe(start);

}

}

return NULL;

}

nmethod* CodeCache::find_nmethod(void* start) {

CodeBlob* cb = find_blob(start);

assert(cb->is_nmethod(), "did not find an nmethod");

return (nmethod*)cb;

}

void CodeCache::blobs_do(void f(CodeBlob* nm)) {

assert_locked_or_safepoint(CodeCache_lock);

FOR_ALL_HEAPS(heap) {

FOR_ALL_BLOBS(cb, *heap) {

f(cb);

}

}

}

void CodeCache::nmethods_do(void f(nmethod* nm)) {

assert_locked_or_safepoint(CodeCache_lock);

NMethodIterator iter(NMethodIterator::all_blobs);

while(iter.next()) {

f(iter.method());

}

}

void CodeCache::metadata_do(MetadataClosure* f) {

assert_locked_or_safepoint(CodeCache_lock);

NMethodIterator iter(NMethodIterator::only_alive_and_not_unloading);

while(iter.next()) {

iter.method()->metadata_do(f);

}

}

int CodeCache::alignment_unit() {

return (int)_heaps->first()->alignment_unit();

}

int CodeCache::alignment_offset() {

return (int)_heaps->first()->alignment_offset();

}

// Mark nmethods for unloading if they contain otherwise unreachable oops.

void CodeCache::do_unloading(BoolObjectClosure* is_alive, bool unloading_occurred) {

assert_locked_or_safepoint(CodeCache_lock);

UnloadingScope scope(is_alive);

CompiledMethodIterator iter(CompiledMethodIterator::only_alive);

while(iter.next()) {

iter.method()->do_unloading(unloading_occurred);

}

}

void CodeCache::blobs_do(CodeBlobClosure* f) {

assert_locked_or_safepoint(CodeCache_lock);

FOR_ALL_ALLOCABLE_HEAPS(heap) {

FOR_ALL_BLOBS(cb, *heap) {

if (cb->is_alive()) {

f->do_code_blob(cb);

#ifdef ASSERT

if (cb->is_nmethod()) {

Universe::heap()->verify_nmethod((nmethod*)cb);

}

#endif //ASSERT

}

}

}

}

void CodeCache::verify_clean_inline_caches() {

#ifdef ASSERT

NMethodIterator iter(NMethodIterator::only_alive_and_not_unloading);

while(iter.next()) {

nmethod* nm = iter.method();

assert(!nm->is_unloaded(), "Tautology");

nm->verify_clean_inline_caches();

nm->verify();

}

#endif

}

void CodeCache::verify_icholder_relocations() {

#ifdef ASSERT

// make sure that we aren't leaking icholders

int count = 0;

FOR_ALL_HEAPS(heap) {

FOR_ALL_BLOBS(cb, *heap) {

CompiledMethod *nm = cb->as_compiled_method_or_null();

if (nm != NULL) {

count += nm->verify_icholder_relocations();

}

}

}

assert(count + InlineCacheBuffer::pending_icholder_count() + CompiledICHolder::live_not_claimed_count() ==

CompiledICHolder::live_count(), "must agree");

#endif

}

// Defer freeing of concurrently cleaned ExceptionCache entries until

// after a global handshake operation.

void CodeCache::release_exception_cache(ExceptionCache* entry) {

if (SafepointSynchronize::is_at_safepoint()) {

delete entry;

} else {

for (;;) {

ExceptionCache* purge_list_head = Atomic::load(&_exception_cache_purge_list);

entry->set_purge_list_next(purge_list_head);

if (Atomic::cmpxchg(&_exception_cache_purge_list, purge_list_head, entry) == purge_list_head) {

break;

}

}

}

}

// Delete exception caches that have been concurrently unlinked,

// followed by a global handshake operation.

void CodeCache::purge_exception_caches() {

ExceptionCache* curr = _exception_cache_purge_list;

while (curr != NULL) {

ExceptionCache* next = curr->purge_list_next();

delete curr;

curr = next;

}

_exception_cache_purge_list = NULL;

}

uint8_t CodeCache::_unloading_cycle = 1;

void CodeCache::increment_unloading_cycle() {

// 2-bit value (see IsUnloadingState in nmethod.cpp for details)

// 0 is reserved for new methods.

_unloading_cycle = (_unloading_cycle + 1) % 4;

if (_unloading_cycle == 0) {

_unloading_cycle = 1;

}

}

CodeCache::UnloadingScope::UnloadingScope(BoolObjectClosure* is_alive)

: _is_unloading_behaviour(is_alive)

{

_saved_behaviour = IsUnloadingBehaviour::current();

IsUnloadingBehaviour::set_current(&_is_unloading_behaviour);

increment_unloading_cycle();

DependencyContext::cleaning_start();

}

CodeCache::UnloadingScope::~UnloadingScope() {

IsUnloadingBehaviour::set_current(_saved_behaviour);

DependencyContext::cleaning_end();

}

void CodeCache::verify_oops() {

MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);

VerifyOopClosure voc;

NMethodIterator iter(NMethodIterator::only_alive_and_not_unloading);

while(iter.next()) {

nmethod* nm = iter.method();

nm->oops_do(&voc);

nm->verify_oop_relocations();

}

}

int CodeCache::blob_count(int code_blob_type) {

CodeHeap* heap = get_code_heap(code_blob_type);

return (heap != NULL) ? heap->blob_count() : 0;

}

int CodeCache::blob_count() {

int count = 0;

FOR_ALL_HEAPS(heap) {

count += (*heap)->blob_count();

}

return count;

}

int CodeCache::nmethod_count(int code_blob_type) {

CodeHeap* heap = get_code_heap(code_blob_type);

return (heap != NULL) ? heap->nmethod_count() : 0;

}

int CodeCache::nmethod_count() {

int count = 0;

FOR_ALL_NMETHOD_HEAPS(heap) {

count += (*heap)->nmethod_count();

}

return count;

}

int CodeCache::adapter_count(int code_blob_type) {

CodeHeap* heap = get_code_heap(code_blob_type);

return (heap != NULL) ? heap->adapter_count() : 0;

}

int CodeCache::adapter_count() {

int count = 0;

FOR_ALL_HEAPS(heap) {

count += (*heap)->adapter_count();

}

return count;

}

address CodeCache::low_bound(int code_blob_type) {

CodeHeap* heap = get_code_heap(code_blob_type);

return (heap != NULL) ? (address)heap->low_boundary() : NULL;

}

address CodeCache::high_bound(int code_blob_type) {

CodeHeap* heap = get_code_heap(code_blob_type);

return (heap != NULL) ? (address)heap->high_boundary() : NULL;

}

size_t CodeCache::capacity() {

size_t cap = 0;

FOR_ALL_ALLOCABLE_HEAPS(heap) {

cap += (*heap)->capacity();

}

return cap;

}

size_t CodeCache::unallocated_capacity(int code_blob_type) {

CodeHeap* heap = get_code_heap(code_blob_type);

return (heap != NULL) ? heap->unallocated_capacity() : 0;

}

size_t CodeCache::unallocated_capacity() {

size_t unallocated_cap = 0;

FOR_ALL_ALLOCABLE_HEAPS(heap) {

unallocated_cap += (*heap)->unallocated_capacity();

}

return unallocated_cap;

}

size_t CodeCache::max_capacity() {

size_t max_cap = 0;

FOR_ALL_ALLOCABLE_HEAPS(heap) {

max_cap += (*heap)->max_capacity();

}

return max_cap;

}

/**

* Returns the reverse free ratio. E.g., if 25% (1/4) of the code heap

* is free, reverse_free_ratio() returns 4.

*/

double CodeCache::reverse_free_ratio(int code_blob_type) {

CodeHeap* heap = get_code_heap(code_blob_type);

if (heap == NULL) {

return 0;

}

double unallocated_capacity = MAX2((double)heap->unallocated_capacity(), 1.0); // Avoid division by 0;

double max_capacity = (double)heap->max_capacity();

double result = max_capacity / unallocated_capacity;

assert (max_capacity >= unallocated_capacity, "Must be");

assert (result >= 1.0, "reverse_free_ratio must be at least 1. It is %f", result);

return result;

}

size_t CodeCache::bytes_allocated_in_freelists() {

size_t allocated_bytes = 0;

FOR_ALL_ALLOCABLE_HEAPS(heap) {

allocated_bytes += (*heap)->allocated_in_freelist();

}

return allocated_bytes;

}

int CodeCache::allocated_segments() {

int number_of_segments = 0;

FOR_ALL_ALLOCABLE_HEAPS(heap) {

number_of_segments += (*heap)->allocated_segments();

}

return number_of_segments;

}

size_t CodeCache::freelists_length() {

size_t length = 0;

FOR_ALL_ALLOCABLE_HEAPS(heap) {

length += (*heap)->freelist_length();

}

return length;

}

void icache_init();

void CodeCache::initialize() {

assert(CodeCacheSegmentSize >= (uintx)CodeEntryAlignment, "CodeCacheSegmentSize must be large enough to align entry points");

#ifdef COMPILER2

assert(CodeCacheSegmentSize >= (uintx)OptoLoopAlignment, "CodeCacheSegmentSize must be large enough to align inner loops");

#endif

assert(CodeCacheSegmentSize >= sizeof(jdouble), "CodeCacheSegmentSize must be large enough to align constants");

// This was originally just a check of the alignment, causing failure, instead, round

// the code cache to the page size. In particular, Solaris is moving to a larger

// default page size.

CodeCacheExpansionSize = align_up(CodeCacheExpansionSize, os::vm_page_size());

if (SegmentedCodeCache) {

// Use multiple code heaps

initialize_heaps();

} else {

// Use a single code heap

FLAG_SET_ERGO(NonNMethodCodeHeapSize, 0);

FLAG_SET_ERGO(ProfiledCodeHeapSize, 0);

FLAG_SET_ERGO(NonProfiledCodeHeapSize, 0);

ReservedCodeSpace rs = reserve_heap_memory(ReservedCodeCacheSize);

add_heap(rs, "CodeCache", CodeBlobType::All);

}

// Initialize ICache flush mechanism

// This service is needed for os::register_code_area

icache_init();

// Give OS a chance to register generated code area.

// This is used on Windows 64 bit platforms to register

// Structured Exception Handlers for our generated code.

os::register_code_area((char*)low_bound(), (char*)high_bound());

}

void codeCache_init() {

CodeCache::initialize();

}

//------------------------------------------------------------------------------------------------

int CodeCache::number_of_nmethods_with_dependencies() {

return _number_of_nmethods_with_dependencies;

}

void CodeCache::clear_inline_caches() {

assert_locked_or_safepoint(CodeCache_lock);

CompiledMethodIterator iter(CompiledMethodIterator::only_alive_and_not_unloading);

while(iter.next()) {

iter.method()->clear_inline_caches();

}

}

void CodeCache::cleanup_inline_caches() {

assert_locked_or_safepoint(CodeCache_lock);

NMethodIterator iter(NMethodIterator::only_alive_and_not_unloading);

while(iter.next()) {

iter.method()->cleanup_inline_caches(/*clean_all=*/true);

}

}

// Keeps track of time spent for checking dependencies

NOT_PRODUCT(static elapsedTimer dependentCheckTime;)

int CodeCache::mark_for_deoptimization(KlassDepChange& changes) {

MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);

int number_of_marked_CodeBlobs = 0;