From 8e41ca2b4439ceb80392f8c09f48c37385fc1a1e Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Sun, 13 Oct 2024 19:14:23 -0400 Subject: [PATCH 01/32] Refactor SCCache address table setup Signed-off-by: Ashutosh Mehra --- src/hotspot/share/code/SCCache.cpp | 135 +++++++++++++------ src/hotspot/share/code/SCCache.hpp | 16 ++- src/hotspot/share/compiler/compileBroker.cpp | 5 - src/hotspot/share/runtime/init.cpp | 2 + 4 files changed, 113 insertions(+), 45 deletions(-) diff --git a/src/hotspot/share/code/SCCache.cpp b/src/hotspot/share/code/SCCache.cpp index 7031f1a678d..6564d5100d8 100644 --- a/src/hotspot/share/code/SCCache.cpp +++ b/src/hotspot/share/code/SCCache.cpp @@ -175,6 +175,13 @@ void SCCache::init2() { close(); exit_vm_on_load_failure(); } + + // initialize the table of external routines so we can save + // generated code blobs that reference them + init_extrs_table(); + // initialize the table of initial stubs so we can save + // generated code blobs that reference them + init_early_stubs_table(); } void SCCache::print_timers_on(outputStream* st) { @@ -243,6 +250,8 @@ void SCCache::close() { } LogStreamHandle(Info, scc, codecache) info_scc; + // need a lock to traverse the code cache + MutexLocker locker(CodeCache_lock, Mutex::_no_safepoint_check_flag); if (info_scc.is_enabled()) { NMethodIterator iter(NMethodIterator::all); while (iter.next()) { @@ -580,10 +589,28 @@ SCCache::SCCache(const char* cache_path, int fd, uint load_size) { _table = new SCAddressTable(); } -void SCCache::init_table() { +void SCCache::init_extrs_table() { + SCCache* cache = SCCache::cache(); + if (cache != nullptr && cache->_table != nullptr) { + cache->_table->init_extrs(); + } +} +void SCCache::init_early_stubs_table() { SCCache* cache = SCCache::cache(); if (cache != nullptr && cache->_table != nullptr) { - cache->_table->init(); + cache->_table->init_early_stubs(); + } +} +void SCCache::init_shared_blobs_table() { + SCCache* cache = SCCache::cache(); + if (cache != nullptr && cache->_table != nullptr) { + cache->_table->init_shared_blobs(); + } +} +void SCCache::init_stubs_table() { + SCCache* cache = SCCache::cache(); + if (cache != nullptr && cache->_table != nullptr) { + cache->_table->init_stubs(); } } @@ -3535,17 +3562,11 @@ void SCCReader::print_on(outputStream* st) { assert(type##_length <= type##_max, "increase size"); \ } -static bool initializing = false; -void SCAddressTable::init() { - if (_complete || initializing) return; // Done already - initializing = true; +static bool initializing_extrs = false; +void SCAddressTable::init_extrs() { + if (_extrs_complete || initializing_extrs) return; // Done already + initializing_extrs = true; _extrs_addr = NEW_C_HEAP_ARRAY(address, _extrs_max, mtCode); - _stubs_addr = NEW_C_HEAP_ARRAY(address, _stubs_max, mtCode); - _blobs_addr = NEW_C_HEAP_ARRAY(address, _blobs_max, mtCode); - - // Divide _blobs_addr array to chunks because they could be initialized in parrallel - _C2_blobs_addr = _blobs_addr + _shared_blobs_max;// C2 blobs addresses stored after shared blobs - _C1_blobs_addr = _C2_blobs_addr + _C2_blobs_max; // C1 blobs addresses stored after C2 blobs _extrs_length = 0; _stubs_length = 0; @@ -3654,9 +3675,68 @@ void SCAddressTable::init() { while (*p != nullptr) { SET_ADDRESS(_extrs, *p++); } + + _extrs_complete = true; + log_info(scc,init)("External addresses recorded"); +} + +static bool initializing_early_stubs = false; +void SCAddressTable::init_early_stubs() { + if (_complete || initializing_early_stubs) return; // Done already + initializing_early_stubs = true; + _stubs_addr = NEW_C_HEAP_ARRAY(address, _stubs_max, mtCode); + _stubs_length = 0; + SET_ADDRESS(_stubs, StubRoutines::forward_exception_entry()); + _early_stubs_complete = true; + log_info(scc,init)("early stubs recorded"); +} + +static bool initializing_shared_blobs = false; +void SCAddressTable::init_shared_blobs() { + if (_complete || initializing_shared_blobs) return; // Done already + initializing_shared_blobs = true; + _blobs_addr = NEW_C_HEAP_ARRAY(address, _blobs_max, mtCode); + // Divide _blobs_addr array to chunks because they could be initialized in parrallel + _C2_blobs_addr = _blobs_addr + _shared_blobs_max;// C2 blobs addresses stored after shared blobs + _C1_blobs_addr = _C2_blobs_addr + _C2_blobs_max; // C1 blobs addresses stored after C2 blobs + _blobs_length = 0; // for shared blobs + _C1_blobs_length = 0; + _C2_blobs_length = 0; + _final_blobs_length = 0; // Depends on numnber of C1 blobs + // Blobs + SET_ADDRESS(_blobs, SharedRuntime::get_handle_wrong_method_stub()); + SET_ADDRESS(_blobs, SharedRuntime::get_ic_miss_stub()); + SET_ADDRESS(_blobs, SharedRuntime::get_resolve_opt_virtual_call_stub()); + SET_ADDRESS(_blobs, SharedRuntime::get_resolve_virtual_call_stub()); + SET_ADDRESS(_blobs, SharedRuntime::get_resolve_static_call_stub()); + SET_ADDRESS(_blobs, SharedRuntime::deopt_blob()->entry_point()); + SET_ADDRESS(_blobs, SharedRuntime::polling_page_safepoint_handler_blob()->entry_point()); + SET_ADDRESS(_blobs, SharedRuntime::polling_page_return_handler_blob()->entry_point()); +#ifdef COMPILER2 + SET_ADDRESS(_blobs, SharedRuntime::polling_page_vectors_safepoint_handler_blob()->entry_point()); +#endif + assert(_blobs_length <= _shared_blobs_max, "increase _shared_blobs_max to %d", _blobs_length); + _final_blobs_length = _blobs_length; + log_info(scc,init)("Early shared blobs recorded"); +} + +static bool initializing_stubs = false; +void SCAddressTable::init_stubs() { + if (_complete || initializing_stubs) return; // Done already + initializing_stubs = true; + // final blobs + SET_ADDRESS(_blobs, SharedRuntime::throw_AbstractMethodError_entry()); + SET_ADDRESS(_blobs, SharedRuntime::throw_IncompatibleClassChangeError_entry()); + SET_ADDRESS(_blobs, SharedRuntime::throw_NullPointerException_at_call_entry()); + SET_ADDRESS(_blobs, SharedRuntime::throw_StackOverflowError_entry()); + SET_ADDRESS(_blobs, SharedRuntime::throw_delayed_StackOverflowError_entry()); + assert(_blobs_length <= _shared_blobs_max, "increase _shared_blobs_max to %d", _blobs_length); + _final_blobs_length = _blobs_length; + _shared_blobs_complete = true; + log_info(scc,init)("All shared blobs recorded"); + // Stubs SET_ADDRESS(_stubs, StubRoutines::method_entry_barrier()); - SET_ADDRESS(_stubs, StubRoutines::forward_exception_entry()); /* SET_ADDRESS(_stubs, StubRoutines::throw_AbstractMethodError_entry()); SET_ADDRESS(_stubs, StubRoutines::throw_IncompatibleClassChangeError_entry()); @@ -3819,29 +3899,8 @@ void SCAddressTable::init() { SET_ADDRESS(_stubs, StubRoutines::aarch64::spin_wait()); #endif - // Blobs - SET_ADDRESS(_blobs, SharedRuntime::get_handle_wrong_method_stub()); - SET_ADDRESS(_blobs, SharedRuntime::get_ic_miss_stub()); - SET_ADDRESS(_blobs, SharedRuntime::get_resolve_opt_virtual_call_stub()); - SET_ADDRESS(_blobs, SharedRuntime::get_resolve_virtual_call_stub()); - SET_ADDRESS(_blobs, SharedRuntime::get_resolve_static_call_stub()); - SET_ADDRESS(_blobs, SharedRuntime::deopt_blob()->entry_point()); - SET_ADDRESS(_blobs, SharedRuntime::polling_page_safepoint_handler_blob()->entry_point()); - SET_ADDRESS(_blobs, SharedRuntime::polling_page_return_handler_blob()->entry_point()); -#ifdef COMPILER2 - SET_ADDRESS(_blobs, SharedRuntime::polling_page_vectors_safepoint_handler_blob()->entry_point()); -#endif - - SET_ADDRESS(_blobs, SharedRuntime::throw_AbstractMethodError_entry()); - SET_ADDRESS(_blobs, SharedRuntime::throw_IncompatibleClassChangeError_entry()); - SET_ADDRESS(_blobs, SharedRuntime::throw_NullPointerException_at_call_entry()); - SET_ADDRESS(_blobs, SharedRuntime::throw_StackOverflowError_entry()); - SET_ADDRESS(_blobs, SharedRuntime::throw_delayed_StackOverflowError_entry()); - - assert(_blobs_length <= _shared_blobs_max, "increase _shared_blobs_max to %d", _blobs_length); - _final_blobs_length = _blobs_length; _complete = true; - log_info(scc,init)("External addresses and stubs recorded"); + log_info(scc,init)("Stubs recorded"); } void SCAddressTable::init_opto() { @@ -4033,7 +4092,7 @@ void SCCache::add_new_C_string(const char* str) { } void SCAddressTable::add_C_string(const char* str) { - if (str != nullptr && _complete && (_opto_complete || _c1_complete)) { + if (str != nullptr && _extrs_complete) { // Check previous strings address for (int i = 0; i < _C_strings_count; i++) { if (_C_strings[i] == str) { @@ -4097,7 +4156,7 @@ int search_address(address addr, address* table, uint length) { } address SCAddressTable::address_for_id(int idx) { - if (!_complete) { + if (!_extrs_complete) { fatal("SCA table is not complete"); } if (idx == -1) { @@ -4132,7 +4191,7 @@ int SCAddressTable::id_for_address(address addr, RelocIterator reloc, CodeBuffer if (addr == (address)-1) { // Static call stub has jump to itself return id; } - if (!_complete) { + if (!_extrs_complete) { fatal("SCA table is not complete"); } // Seach for C string diff --git a/src/hotspot/share/code/SCCache.hpp b/src/hotspot/share/code/SCCache.hpp index d818a007d16..d68b6021489 100644 --- a/src/hotspot/share/code/SCCache.hpp +++ b/src/hotspot/share/code/SCCache.hpp @@ -273,6 +273,9 @@ class SCAddressTable : public CHeapObj { uint _C2_blobs_length; uint _final_blobs_length; + bool _extrs_complete; + bool _early_stubs_complete; + bool _shared_blobs_complete; bool _complete; bool _opto_complete; bool _c1_complete; @@ -282,12 +285,18 @@ class SCAddressTable : public CHeapObj { _extrs_addr = nullptr; _stubs_addr = nullptr; _blobs_addr = nullptr; + _extrs_complete = false; + _early_stubs_complete = false; + _shared_blobs_complete = false; _complete = false; _opto_complete = false; _c1_complete = false; } ~SCAddressTable(); - void init(); + void init_extrs(); + void init_early_stubs(); + void init_shared_blobs(); + void init_stubs(); void init_opto(); void init_c1(); void add_C_string(const char* str); @@ -470,7 +479,10 @@ class SCCache : public CHeapObj { void load_strings(); int store_strings(); - static void init_table(); + static void init_extrs_table(); + static void init_early_stubs_table(); + static void init_shared_blobs_table(); + static void init_stubs_table(); static void init_opto_table(); static void init_c1_table(); address address_for_id(int id) const { return _table->address_for_id(id); } diff --git a/src/hotspot/share/compiler/compileBroker.cpp b/src/hotspot/share/compiler/compileBroker.cpp index 8cab1d750ee..3273f9b5030 100644 --- a/src/hotspot/share/compiler/compileBroker.cpp +++ b/src/hotspot/share/compiler/compileBroker.cpp @@ -2043,11 +2043,6 @@ bool CompileBroker::init_compiler_runtime() { // Switch back to VM state to do compiler initialization ThreadInVMfromNative tv(thread); - // Perform per-thread and global initializations - { - MutexLocker only_one (thread, CompileThread_lock); - SCCache::init_table(); - } comp->initialize(); } diff --git a/src/hotspot/share/runtime/init.cpp b/src/hotspot/share/runtime/init.cpp index 7180f452ef1..704440bf16d 100644 --- a/src/hotspot/share/runtime/init.cpp +++ b/src/hotspot/share/runtime/init.cpp @@ -166,6 +166,7 @@ jint init_globals() { InterfaceSupport_init(); VMRegImpl::set_regName(); // need this before generate_stubs (for printing oop maps). SharedRuntime::generate_stubs(); + SCCache::init_shared_blobs_table(); // need this after generate_stubs return JNI_OK; } @@ -204,6 +205,7 @@ jint init_globals2() { } compiler_stubs_init(false /* in_compiler_thread */); // compiler's intrinsics stubs final_stubs_init(); // final StubRoutines stubs + SCCache::init_stubs_table(); MethodHandles::generate_adapters(); SystemDictionary::restore_archived_method_handle_intrinsics(); From 4f96b309a28d9c43335052d4118986353d7b7a91 Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Sun, 13 Oct 2024 19:33:58 -0400 Subject: [PATCH 02/32] Allocate address table entries for different types in fixed, independent ranges Signed-off-by: Ashutosh Mehra --- src/hotspot/share/code/SCCache.cpp | 124 +++++++++++++++++++---------- src/hotspot/share/code/SCCache.hpp | 1 - 2 files changed, 83 insertions(+), 42 deletions(-) diff --git a/src/hotspot/share/code/SCCache.cpp b/src/hotspot/share/code/SCCache.cpp index 6564d5100d8..b992b78bad3 100644 --- a/src/hotspot/share/code/SCCache.cpp +++ b/src/hotspot/share/code/SCCache.cpp @@ -3548,14 +3548,32 @@ void SCCReader::print_on(outputStream* st) { st->print_cr(" name: %s", name); } +// address table ids for generated routines, external addresses and C +// string addresses are partitioned into positive integer ranges +// defined by the following positive base and max values +// i.e. [_extrs_base, _extrs_base + _extrs_max -1], +// [_stubs_base, _stubs_base + _stubs_max -1], +// ... +// [_c_str_base, _c_str_base + _c_str_max -1], #define _extrs_max 80 #define _stubs_max 120 -#define _blobs_max 100 -#define _shared_blobs_max 24 +#define _all_blobs_max 100 +#define _blobs_max 24 #define _C2_blobs_max 25 -#define _C1_blobs_max (_blobs_max - _shared_blobs_max - _C2_blobs_max) +#define _C1_blobs_max (_all_blobs_max - _blobs_max - _C2_blobs_max) #define _all_max 300 +#define _c_str_max MAX_STR_COUNT +#define _extrs_base 0 +#define _stubs_base (_extrs_base + _extrs_max) +#define _blobs_base (_stubs_base + _stubs_max) +#define _C1_blobs_base (_blobs_base + _blobs_max) +#define _C2_blobs_base (_C1_blobs_base + _C1_blobs_max) +#if (_C2_blobs_base >= _all_max) +#error SCAddress table ranges need adjusting +#endif +#define _c_str_base _all_max + #define SET_ADDRESS(type, addr) \ { \ type##_addr[type##_length++] = (address) (addr); \ @@ -3570,10 +3588,6 @@ void SCAddressTable::init_extrs() { _extrs_length = 0; _stubs_length = 0; - _blobs_length = 0; // for shared blobs - _C1_blobs_length = 0; - _C2_blobs_length = 0; - _final_blobs_length = 0; // Depends on numnber of C1 blobs // Runtime methods #ifdef COMPILER2 @@ -3695,14 +3709,16 @@ static bool initializing_shared_blobs = false; void SCAddressTable::init_shared_blobs() { if (_complete || initializing_shared_blobs) return; // Done already initializing_shared_blobs = true; - _blobs_addr = NEW_C_HEAP_ARRAY(address, _blobs_max, mtCode); + _blobs_addr = NEW_C_HEAP_ARRAY(address, _all_blobs_max, mtCode); + // Divide _blobs_addr array to chunks because they could be initialized in parrallel - _C2_blobs_addr = _blobs_addr + _shared_blobs_max;// C2 blobs addresses stored after shared blobs - _C1_blobs_addr = _C2_blobs_addr + _C2_blobs_max; // C1 blobs addresses stored after C2 blobs + _C1_blobs_addr = _blobs_addr + _blobs_max;// C1 blobs addresses stored after shared blobs + _C2_blobs_addr = _C1_blobs_addr + _C1_blobs_max; // C2 blobs addresses stored after C1 blobs + _blobs_length = 0; // for shared blobs _C1_blobs_length = 0; _C2_blobs_length = 0; - _final_blobs_length = 0; // Depends on numnber of C1 blobs + // Blobs SET_ADDRESS(_blobs, SharedRuntime::get_handle_wrong_method_stub()); SET_ADDRESS(_blobs, SharedRuntime::get_ic_miss_stub()); @@ -3715,8 +3731,8 @@ void SCAddressTable::init_shared_blobs() { #ifdef COMPILER2 SET_ADDRESS(_blobs, SharedRuntime::polling_page_vectors_safepoint_handler_blob()->entry_point()); #endif - assert(_blobs_length <= _shared_blobs_max, "increase _shared_blobs_max to %d", _blobs_length); - _final_blobs_length = _blobs_length; + + assert(_blobs_length <= _blobs_max, "increase _blobs_max to %d", _blobs_length); log_info(scc,init)("Early shared blobs recorded"); } @@ -3730,8 +3746,9 @@ void SCAddressTable::init_stubs() { SET_ADDRESS(_blobs, SharedRuntime::throw_NullPointerException_at_call_entry()); SET_ADDRESS(_blobs, SharedRuntime::throw_StackOverflowError_entry()); SET_ADDRESS(_blobs, SharedRuntime::throw_delayed_StackOverflowError_entry()); - assert(_blobs_length <= _shared_blobs_max, "increase _shared_blobs_max to %d", _blobs_length); - _final_blobs_length = _blobs_length; + + assert(_blobs_length <= _blobs_max, "increase _blobs_max to %d", _blobs_length); + _shared_blobs_complete = true; log_info(scc,init)("All shared blobs recorded"); @@ -3933,7 +3950,6 @@ void SCAddressTable::init_opto() { #endif assert(_C2_blobs_length <= _C2_blobs_max, "increase _C2_blobs_max to %d", _C2_blobs_length); - _final_blobs_length = MAX2(_final_blobs_length, (_shared_blobs_max + _C2_blobs_length)); _opto_complete = true; log_info(scc,init)("OptoRuntime Blobs recorded"); } @@ -3985,18 +4001,11 @@ void SCAddressTable::init_c1() { #endif // COMPILER1 assert(_C1_blobs_length <= _C1_blobs_max, "increase _C1_blobs_max to %d", _C1_blobs_length); - _final_blobs_length = MAX2(_final_blobs_length, (_shared_blobs_max + _C2_blobs_max + _C1_blobs_length)); _c1_complete = true; log_info(scc,init)("Runtime1 Blobs recorded"); } #undef SET_ADDRESS -#undef _extrs_max -#undef _stubs_max -#undef _blobs_max -#undef _shared_blobs_max -#undef _C1_blobs_max -#undef _C2_blobs_max SCAddressTable::~SCAddressTable() { if (_extrs_addr != nullptr) { @@ -4157,32 +4166,39 @@ int search_address(address addr, address* table, uint length) { address SCAddressTable::address_for_id(int idx) { if (!_extrs_complete) { - fatal("SCA table is not complete"); + fatal("SCA extrs table is not complete"); } if (idx == -1) { return (address)-1; } uint id = (uint)idx; - if (id >= _all_max && idx < (_all_max + _C_strings_count)) { - return address_for_C_string(idx - _all_max); + // special case for symbols based relative to os::init + if (id > (_c_str_base + _c_str_max)) { + return (address)os::init + idx; } - if (idx < 0 || id == (_extrs_length + _stubs_length + _final_blobs_length)) { + if (idx < 0) { fatal("Incorrect id %d for SCA table", id); } - if (idx > (_all_max + _C_strings_count)) { - return (address)os::init + idx; + // no need to compare unsigned id against 0 + if (/* id >= _extrs_base && */ id < _extrs_length) { + return _extrs_addr[id - _extrs_base]; } - if (id < _extrs_length) { - return _extrs_addr[id]; + if (id >= _stubs_base && id < _stubs_base + _stubs_length) { + return _stubs_addr[id - _stubs_base]; } - id -= _extrs_length; - if (id < _stubs_length) { - return _stubs_addr[id]; + if (id >= _blobs_base && id < _blobs_base + _blobs_length) { + return _blobs_addr[id - _blobs_base]; } - id -= _stubs_length; - if (id < _final_blobs_length) { - return _blobs_addr[id]; + if (id >= _C1_blobs_base && id < _C1_blobs_base + _C1_blobs_length) { + return _C1_blobs_addr[id - _C1_blobs_base]; } + if (id >= _C2_blobs_base && id < _C2_blobs_base + _C2_blobs_length) { + return _C2_blobs_addr[id - _C2_blobs_base]; + } + if (id >= _c_str_base && id < (_c_str_base + (uint)_C_strings_count)) { + return address_for_C_string(id - _c_str_base); + } + fatal("Incorrect id %d for SCA table", id); return nullptr; } @@ -4197,7 +4213,7 @@ int SCAddressTable::id_for_address(address addr, RelocIterator reloc, CodeBuffer // Seach for C string id = id_for_C_string(addr); if (id >=0) { - return id + _all_max; + return id + _c_str_base; } if (StubRoutines::contains(addr)) { // Search in stubs @@ -4210,17 +4226,28 @@ int SCAddressTable::id_for_address(address addr, RelocIterator reloc, CodeBuffer const char* sub_name = (desc != nullptr) ? desc->name() : ""; fatal("Address " INTPTR_FORMAT " for Stub:%s is missing in SCA table", p2i(addr), sub_name); } else { - id += _extrs_length; + return _stubs_base + id; } } else { CodeBlob* cb = CodeCache::find_blob(addr); if (cb != nullptr) { + int id_base = _blobs_base; // Search in code blobs - id = search_address(addr, _blobs_addr, _final_blobs_length); + id = search_address(addr, _blobs_addr, _blobs_length); + if (id == -1) { + id_base = _C1_blobs_base; + // search C1 blobs + id = search_address(addr, _C1_blobs_addr, _C1_blobs_length); + } + if (id == -1) { + id_base = _C2_blobs_base; + // search C2 blobs + id = search_address(addr, _C2_blobs_addr, _C2_blobs_length); + } if (id < 0) { fatal("Address " INTPTR_FORMAT " for Blob:%s is missing in SCA table", p2i(addr), cb->name()); } else { - id += _extrs_length + _stubs_length; + return id_base + id; } } else { // Search in runtime functions @@ -4256,12 +4283,27 @@ int SCAddressTable::id_for_address(address addr, RelocIterator reloc, CodeBuffer #endif // !PRODUCT fatal("Address " INTPTR_FORMAT " for is missing in SCA table", p2i(addr)); } + } else { + return _extrs_base + id; } } } return id; } +#undef _extrs_max +#undef _stubs_max +#undef _all_blobs_max +#undef _blobs_max +#undef _C1_blobs_max +#undef _C2_blobs_max +#undef _extrs_base +#undef _stubs_base +#undef _blobs_base +#undef _C1_blobs_base +#undef _C2_blobs_base +#undef _c_str_base + void AOTRuntimeConstants::initialize_from_runtime() { BarrierSet* bs = BarrierSet::barrier_set(); if (bs->is_a(BarrierSet::CardTableBarrierSet)) { diff --git a/src/hotspot/share/code/SCCache.hpp b/src/hotspot/share/code/SCCache.hpp index d68b6021489..8da8cb3662e 100644 --- a/src/hotspot/share/code/SCCache.hpp +++ b/src/hotspot/share/code/SCCache.hpp @@ -271,7 +271,6 @@ class SCAddressTable : public CHeapObj { uint _blobs_length; uint _C1_blobs_length; uint _C2_blobs_length; - uint _final_blobs_length; bool _extrs_complete; bool _early_stubs_complete; From 0fc149a7f6d7667d916a718d0ca619842b610615 Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Tue, 15 Oct 2024 11:05:41 -0400 Subject: [PATCH 03/32] save and restore i2c and c2i adapter Signed-off-by: Ashutosh Mehra --- .../cpu/aarch64/sharedRuntime_aarch64.cpp | 27 +++ src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp | 53 +++-- src/hotspot/share/code/SCCache.cpp | 193 +++++++++++++++++- src/hotspot/share/code/SCCache.hpp | 12 +- src/hotspot/share/runtime/sharedRuntime.cpp | 11 +- src/hotspot/share/runtime/sharedRuntime.hpp | 2 + 6 files changed, 276 insertions(+), 22 deletions(-) diff --git a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp index 52996f4c4a5..2c2ecd139cb 100644 --- a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp @@ -720,6 +720,22 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm AdapterFingerPrint* fingerprint) { address i2c_entry = __ pc(); + const char* name = AdapterHandlerLibrary::name(fingerprint); + const uint32_t id = AdapterHandlerLibrary::id(fingerprint); + // we need to avoid restoring code that might refer to stub routines + // until we know they are available likewise we don't want to save + // early generated adapters that try to refernce them + bool attempt_save_restore = (StubRoutines::final_stubs_code() != nullptr); + if (attempt_save_restore) { + // see if we can load the code rather than generate it + CodeBuffer* buffer = masm->code(); + uint32_t offsets[4]; + if (SCCache::load_adapter(buffer, id, name, offsets)) { + assert(offsets[0] == 0, "sanity check"); + return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, i2c_entry + offsets[1], i2c_entry + offsets[2], i2c_entry + offsets[3]); + } + } + gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs); address c2i_unverified_entry = __ pc(); @@ -778,6 +794,17 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); + if (attempt_save_restore) { + // try to save generated code + CodeBuffer* buffer = masm->code(); + uint32_t offsets[4]; + offsets[0] = 0; + offsets[1] = c2i_entry - i2c_entry; + offsets[2] = c2i_unverified_entry - i2c_entry; + offsets[3] = c2i_no_clinit_check_entry - i2c_entry; + SCCache::store_adapter(buffer, id, name, offsets); + } + return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); } diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp index 174e2e02779..dd5a9003d95 100644 --- a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp +++ b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp @@ -28,6 +28,7 @@ #endif #include "asm/macroAssembler.hpp" #include "asm/macroAssembler.inline.hpp" +#include "code/SCCache.hpp" #include "code/compiledIC.hpp" #include "code/debugInfoRec.hpp" #include "code/nativeInst.hpp" @@ -676,6 +677,18 @@ static void patch_callers_callsite(MacroAssembler *masm) { __ bind(L); } +static void range_check(MacroAssembler* masm, Register pc_reg, Register temp_reg, + address code_start, address code_end, + Label& L_ok) { + Label L_fail; + __ lea(temp_reg, AddressLiteral(code_start, relocInfo::none)); + __ cmpptr(pc_reg, temp_reg); + __ jcc(Assembler::belowEqual, L_fail); + __ lea(temp_reg, AddressLiteral(code_end, relocInfo::none)); + __ cmpptr(pc_reg, temp_reg); + __ jcc(Assembler::below, L_ok); + __ bind(L_fail); +} static void gen_c2i_adapter(MacroAssembler *masm, int total_args_passed, @@ -827,19 +840,6 @@ static void gen_c2i_adapter(MacroAssembler *masm, __ jmp(rcx); } -static void range_check(MacroAssembler* masm, Register pc_reg, Register temp_reg, - address code_start, address code_end, - Label& L_ok) { - Label L_fail; - __ lea(temp_reg, AddressLiteral(code_start, relocInfo::none)); - __ cmpptr(pc_reg, temp_reg); - __ jcc(Assembler::belowEqual, L_fail); - __ lea(temp_reg, AddressLiteral(code_end, relocInfo::none)); - __ cmpptr(pc_reg, temp_reg); - __ jcc(Assembler::below, L_ok); - __ bind(L_fail); -} - void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, int total_args_passed, int comp_args_on_stack, @@ -1059,6 +1059,22 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm AdapterFingerPrint* fingerprint) { address i2c_entry = __ pc(); + const char* name = AdapterHandlerLibrary::name(fingerprint); + const uint32_t id = AdapterHandlerLibrary::id(fingerprint); + // we need to avoid restoring code that might refer to stub routines + // until we know they are available likewise we don't want to save + // early generated adapters that try to refernce them + bool attempt_save_restore = (StubRoutines::final_stubs_code() != nullptr); + if (attempt_save_restore) { + // see if we can load the code rather than generate it + CodeBuffer* buffer = masm->code(); + uint32_t offsets[4]; + if (SCCache::load_adapter(buffer, id, name, offsets)) { + assert(offsets[0] == 0, "sanity check"); + return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, i2c_entry + offsets[1], i2c_entry + offsets[2], i2c_entry + offsets[3]); + } + } + gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs); // ------------------------------------------------------------------------- @@ -1118,6 +1134,17 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); + if (attempt_save_restore) { + // try to save generated code + CodeBuffer* buffer = masm->code(); + uint32_t offsets[4]; + offsets[0] = 0; + offsets[1] = c2i_entry - i2c_entry; + offsets[2] = c2i_unverified_entry - i2c_entry; + offsets[3] = c2i_no_clinit_check_entry - i2c_entry; + SCCache::store_adapter(buffer, id, name, offsets); + } + return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); } diff --git a/src/hotspot/share/code/SCCache.cpp b/src/hotspot/share/code/SCCache.cpp index b992b78bad3..d0d895cdb19 100644 --- a/src/hotspot/share/code/SCCache.cpp +++ b/src/hotspot/share/code/SCCache.cpp @@ -1139,6 +1139,12 @@ bool SCCache::finish_write() { SCCEntry* entries_address = _store_entries; // Pointer to latest entry uint not_entrant_nb = 0; + uint stubs_count = 0; + uint adapters_count = 0; + uint shared_blobs_count = 0; + uint c1_blobs_count = 0; + uint opto_blobs_count = 0; + uint total_blobs_count = 0; uint max_size = 0; // Add old entries first if (_for_read && (_load_header != nullptr)) { @@ -1158,6 +1164,20 @@ bool SCCache::finish_write() { } else if (_load_entries[i].for_preload() && _load_entries[i].method() != nullptr) { // record entrant first version code for pre-loading preload_entries[preload_entries_cnt++] = entries_count; + } else if (entries_address[i].kind() == SCCEntry::Adapter) { + adapters_count++; + } else if (entries_address[i].kind() == SCCEntry::Stub) { + stubs_count++; + } else if (entries_address[i].kind() == SCCEntry::Blob) { + total_blobs_count++; + if (entries_address[i].comp_level() == CompLevel_none) { + shared_blobs_count++; + } else if (entries_address[i].comp_level() == CompLevel_simple) { + c1_blobs_count++; + } else { + assert(entries_address[i].comp_level() == CompLevel_full_optimization, "must be!"); + opto_blobs_count++; + } } { uint size = align_up(_load_entries[i].size(), DATA_ALIGNMENT); @@ -1256,7 +1276,12 @@ bool SCCache::finish_write() { copy_bytes((_store_buffer + entries_offset), (address)current, entries_size); current += entries_size; log_info(scc, exit)("Wrote %d SCCEntry entries (%d were not entrant, %d max size) to Startup Code Cache '%s'", entries_count, not_entrant_nb, max_size, _cache_path); - + log_info(scc, exit)(" Stubs: total=%d", stubs_count); + log_info(scc, exit)(" Adapters: total=%d", adapters_count); + log_info(scc, exit)(" Shared Blobs: total=%d",shared_blobs_count); + log_info(scc, exit)(" C1 Blobs: total=%d", c1_blobs_count); + log_info(scc, exit)(" Opto Blobs: total=%d", opto_blobs_count); + log_info(scc, exit)(" All Blobs: total=%d", total_blobs_count); uint size = (current - start); assert(size <= total_size, "%d > %d", size , total_size); @@ -1972,7 +1997,6 @@ bool SCCReader::read_relocations(CodeBuffer* buffer, CodeBuffer* orig_buffer, bool SCCReader::read_code(CodeBuffer* buffer, CodeBuffer* orig_buffer, uint code_offset) { assert(code_offset == align_up(code_offset, DATA_ALIGNMENT), "%d not aligned to %d", code_offset, DATA_ALIGNMENT); - assert(buffer->blob() != nullptr, "sanity"); SCCodeSection* scc_cs = (SCCodeSection*)addr(code_offset); for (int i = 0; i < (int)CodeBuffer::SECT_LIMIT; i++) { CodeSection* cs = buffer->code_section(i); @@ -2009,6 +2033,82 @@ bool SCCReader::read_code(CodeBuffer* buffer, CodeBuffer* orig_buffer, uint code return true; } +bool SCCache::load_adapter(CodeBuffer* buffer, uint32_t id, const char* name, uint32_t offsets[4]) { +#ifdef ASSERT + LogStreamHandle(Debug, scc, stubs) log; + if (log.is_enabled()) { + FlagSetting fs(PrintRelocations, true); + buffer->print_on(&log); + } +#endif + SCCache* cache = open_for_read(); + if (cache == nullptr) { + return false; + } + log_info(scc, stubs)("Looking up adapter %s (0x%x) in Startup Code Cache '%s'", + name, id, _cache->cache_path()); + SCCEntry* entry = cache->find_entry(SCCEntry::Adapter, id); + if (entry == nullptr) { + return false; + } + SCCReader reader(cache, entry, nullptr); + return reader.compile_adapter(buffer, name, offsets); +} +bool SCCReader::compile_adapter(CodeBuffer* buffer, const char* name, uint32_t offsets[4]) { + uint entry_position = _entry->offset(); + // Read name + uint name_offset = entry_position + _entry->name_offset(); + uint name_size = _entry->name_size(); // Includes '/0' + const char* stored_name = addr(name_offset); + log_info(scc, stubs)("%d (L%d): Reading adapter '%s' from Startup Code Cache '%s'", + compile_id(), comp_level(), name, _cache->cache_path()); + if (strncmp(stored_name, name, (name_size - 1)) != 0) { + log_warning(scc)("%d (L%d): Saved adapter's name '%s' is different from '%s'", + compile_id(), comp_level(), stored_name, name); + // n.b. this is not fatal -- we have just seen a hash id clash + // so no need to call cache->set_failed() + return false; + } + // Create fake original CodeBuffer + CodeBuffer orig_buffer(name); + // Read code + uint code_offset = entry_position + _entry->code_offset(); + if (!read_code(buffer, &orig_buffer, code_offset)) { + return false; + } + // Read relocations + uint reloc_offset = entry_position + _entry->reloc_offset(); + set_read_position(reloc_offset); + if (!read_relocations(buffer, &orig_buffer, nullptr, nullptr)) { + return false; + } + uint offset = read_position(); + int offsets_count = *(int*)addr(offset); + offset += sizeof(int); + assert(offsets_count == 4, "wrong caller expectations"); + set_read_position(offset); + for (int i = 0; i < offsets_count; i++) { + uint32_t arg = *(uint32_t*)addr(offset); + offset += sizeof(uint32_t); + log_debug(scc, stubs)("%d (L%d): Reading adapter '%s' offsets[%d] == 0x%x from Startup Code Cache '%s'", + compile_id(), comp_level(), stored_name, i, arg, _cache->cache_path()); + offsets[i] = arg; + } + log_debug(scc, stubs)("%d (L%d): Read adapter '%s' with '%d' args from Startup Code Cache '%s'", + compile_id(), comp_level(), stored_name, offsets_count, _cache->cache_path()); +#ifdef ASSERT + LogStreamHandle(Debug, scc, stubs) log; + if (log.is_enabled()) { + FlagSetting fs(PrintRelocations, true); + buffer->print_on(&log); + buffer->decode(); + } +#endif + // mark entry as loaded + ((SCCEntry *)_entry)->set_loaded(); + return true; +} + bool SCCache::load_exception_blob(CodeBuffer* buffer, int* pc_offset) { #ifdef ASSERT LogStreamHandle(Debug, scc, nmethod) log; @@ -2252,6 +2352,72 @@ bool SCCache::write_relocations(CodeBuffer* buffer, uint& all_reloc_size) { return success; } +bool SCCache::store_adapter(CodeBuffer* buffer, uint32_t id, const char* name, uint32_t offsets[4]) { + SCCache* cache = open_for_write(); + if (cache == nullptr) { + return false; + } + log_info(scc, stubs)("Writing adapter '%s' (0x%x) to Startup Code Cache '%s'", name, id, cache->_cache_path); +#ifdef ASSERT + LogStreamHandle(Debug, scc, stubs) log; + if (log.is_enabled()) { + FlagSetting fs(PrintRelocations, true); + buffer->print_on(&log); + buffer->decode(); + } +#endif + // we need to take a lock to stop C1 and C2 compiler threads racing to + // write blobs in parallel with each other or with later nmethods + // TODO - maybe move this up to selected callers so we only lock + // when saving a c1 or opto blob + MutexLocker ml(Compile_lock); + if (!cache->align_write()) { + return false; + } + uint entry_position = cache->_write_position; + // Write name + uint name_offset = cache->_write_position - entry_position; + uint name_size = (uint)strlen(name) + 1; // Includes '/0' + uint n = cache->write_bytes(name, name_size); + if (n != name_size) { + return false; + } + // Write code section + if (!cache->align_write()) { + return false; + } + uint code_offset = cache->_write_position - entry_position; + uint code_size = 0; + if (!cache->write_code(buffer, code_size)) { + return false; + } + // Write relocInfo array + uint reloc_offset = cache->_write_position - entry_position; + uint reloc_size = 0; + if (!cache->write_relocations(buffer, reloc_size)) { + return false; + } + int extras_count = 4; + n = cache->write_bytes(&extras_count, sizeof(int)); + if (n != sizeof(int)) { + return false; + } + for (int i = 0; i < 4; i++) { + uint32_t arg = offsets[i]; + log_debug(scc, stubs)("Writing adapter '%s' (0x%x) offsets[%d] == 0x%x to Startup Code Cache '%s'", name, id, i, arg, cache->_cache_path); + n = cache->write_bytes(&arg, sizeof(uint32_t)); + if (n != sizeof(uint32_t)) { + return false; + } + } + uint entry_size = cache->_write_position - entry_position; + SCCEntry* entry = new (cache) SCCEntry(entry_position, entry_size, name_offset, name_size, + code_offset, code_size, reloc_offset, reloc_size, + SCCEntry::Adapter, id, 0); + log_info(scc, stubs)("Wrote adapter '%s' (0x%x) to Startup Code Cache '%s'", name, id, cache->_cache_path); + return true; +} + bool SCCache::write_code(CodeBuffer* buffer, uint& code_size) { assert(_write_position == align_up(_write_position, DATA_ALIGNMENT), "%d not aligned to %d", _write_position, DATA_ALIGNMENT); //assert(buffer->blob() != nullptr, "sanity"); @@ -3393,7 +3559,7 @@ static void print_helper1(outputStream* st, const char* name, int count) { st->print(" %s=%d", name, count); } } -static void print_helper(outputStream* st, const char* name, int stats[6+3][6], int idx) { +static void print_helper(outputStream* st, const char* name, int stats[6+3+1][6], int idx) { int total = stats[idx][0]; if (total > 0) { st->print(" %s:", name); @@ -3420,7 +3586,8 @@ void SCCache::print_statistics_on(outputStream* st) { uint* search_entries = (uint*)cache->addr(cache->_load_header->entries_offset()); // [id, index] SCCEntry* load_entries = (SCCEntry*)(search_entries + 2 * count); - int stats[6 + 3][6] = {0}; + // Entries are None, Adapter, Stub, Blob x 3 levels, Code x 6 levels + int stats[6 + 3 + 1][6] = {0}; for (uint i = 0; i < count; i++) { int index = search_entries[2*i + 1]; SCCEntry* entry = &(load_entries[index]); @@ -3450,6 +3617,7 @@ void SCCache::print_statistics_on(outputStream* st) { print_helper(st, "None", stats, SCCEntry::None); print_helper(st, "Stub", stats, SCCEntry::Stub); print_helper(st, "Blob", stats, SCCEntry::Blob); + print_helper(st, "Adapters", stats, SCCEntry::Adapter); for (int lvl = 0; lvl <= CompLevel_full_optimization + 1; lvl++) { ResourceMark rm; stringStream ss; @@ -3616,6 +3784,19 @@ void SCAddressTable::init_extrs() { SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_phantom); SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_phantom_narrow); #endif + SET_ADDRESS(_extrs, SharedRuntime::fixup_callers_callsite); + + SET_ADDRESS(_extrs, SharedRuntime::log_jni_monitor_still_held); + SET_ADDRESS(_extrs, SharedRuntime::rc_trace_method_entry); + SET_ADDRESS(_extrs, SharedRuntime::reguard_yellow_pages); + SET_ADDRESS(_extrs, SharedRuntime::dtrace_method_exit); + + SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method); + SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method_abstract); + SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method_ic_miss); + SET_ADDRESS(_extrs, SharedRuntime::resolve_opt_virtual_call_C); + SET_ADDRESS(_extrs, SharedRuntime::resolve_virtual_call_C); + SET_ADDRESS(_extrs, SharedRuntime::resolve_static_call_C); SET_ADDRESS(_extrs, SharedRuntime::complete_monitor_unlocking_C); SET_ADDRESS(_extrs, SharedRuntime::enable_stack_reserved_zone); @@ -4019,7 +4200,11 @@ SCAddressTable::~SCAddressTable() { } } +#ifdef PRODUCT #define MAX_STR_COUNT 200 +#else +#define MAX_STR_COUNT 500 +#endif static const char* _C_strings[MAX_STR_COUNT] = {nullptr}; static int _C_strings_count = 0; static int _C_strings_s[MAX_STR_COUNT] = {0}; diff --git a/src/hotspot/share/code/SCCache.hpp b/src/hotspot/share/code/SCCache.hpp index 8da8cb3662e..a1ca68a9e45 100644 --- a/src/hotspot/share/code/SCCache.hpp +++ b/src/hotspot/share/code/SCCache.hpp @@ -145,9 +145,10 @@ class SCCEntry { public: enum Kind { None = 0, - Stub = 1, - Blob = 2, - Code = 3 + Adapter = 1, + Stub = 2, + Blob = 3, + Code = 4 }; private: @@ -359,6 +360,8 @@ class SCCReader { // Concurent per compilation request bool compile(ciEnv* env, ciMethod* target, int entry_bci, AbstractCompiler* compiler); bool compile_blob(CodeBuffer* buffer, int* pc_offset); + bool compile_adapter(CodeBuffer* buffer, const char* name, uint32_t offsets[4]); + Klass* read_klass(const methodHandle& comp_method, bool shared); Method* read_method(const methodHandle& comp_method, bool shared); @@ -531,6 +534,9 @@ class SCCache : public CHeapObj { static bool load_exception_blob(CodeBuffer* buffer, int* pc_offset); static bool store_exception_blob(CodeBuffer* buffer, int pc_offset); + static bool load_adapter(CodeBuffer* buffer, uint32_t id, const char* basic_sig, uint32_t offsets[4]); + static bool store_adapter(CodeBuffer* buffer, uint32_t id, const char* basic_sig, uint32_t offsets[4]); + static bool load_nmethod(ciEnv* env, ciMethod* target, int entry_bci, AbstractCompiler* compiler, CompLevel comp_level); static SCCEntry* store_nmethod(const methodHandle& method, diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index ca8a3a52e55..14297b99ec4 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -2387,7 +2387,6 @@ class AdapterFingerPrint : public CHeapObj { return st.as_string(); } -#ifndef PRODUCT // Reconstitutes the basic type arguments from the fingerprint, // producing strings like LIJDF const char* as_basic_args_string() { @@ -2426,7 +2425,6 @@ class AdapterFingerPrint : public CHeapObj { } return st.as_string(); } -#endif // !product bool equals(AdapterFingerPrint* other) { if (other->_length != _length) { @@ -3146,6 +3144,15 @@ bool AdapterHandlerLibrary::contains(const CodeBlob* b) { return found; } +const char* AdapterHandlerLibrary::name(AdapterFingerPrint* fingerprint) { + return fingerprint->as_basic_args_string(); +} + +uint32_t AdapterHandlerLibrary::id(AdapterFingerPrint* fingerprint) { + unsigned int hash = fingerprint->compute_hash(); + return hash; +} + void AdapterHandlerLibrary::print_handler_on(outputStream* st, const CodeBlob* b) { bool found = false; auto findblob = [&] (AdapterFingerPrint* key, AdapterHandlerEntry* a) { diff --git a/src/hotspot/share/runtime/sharedRuntime.hpp b/src/hotspot/share/runtime/sharedRuntime.hpp index 88ee4a63e27..6ec01114eea 100644 --- a/src/hotspot/share/runtime/sharedRuntime.hpp +++ b/src/hotspot/share/runtime/sharedRuntime.hpp @@ -758,6 +758,8 @@ class AdapterHandlerLibrary: public AllStatic { static void print_handler(const CodeBlob* b) { print_handler_on(tty, b); } static void print_handler_on(outputStream* st, const CodeBlob* b); static bool contains(const CodeBlob* b); + static const char* name(AdapterFingerPrint* fingerprint); + static uint32_t id(AdapterFingerPrint* fingerprint); #ifndef PRODUCT static void print_statistics_on(outputStream* st); #endif // PRODUCT From d5be82536f1a4c7dae6b19af2329c8c887f65b8a Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Tue, 15 Oct 2024 12:23:21 -0400 Subject: [PATCH 04/32] Save AdapterHandlerEntry and AdapterFingerPrint in AOT cache Signed-off-by: Ashutosh Mehra --- src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp | 76 +--- src/hotspot/share/cds/cdsConfig.cpp | 4 + src/hotspot/share/cds/cdsConfig.hpp | 1 + src/hotspot/share/cds/cds_globals.hpp | 3 + src/hotspot/share/cds/cppVtables.cpp | 2 + src/hotspot/share/cds/metaspaceShared.cpp | 5 + src/hotspot/share/memory/allocation.hpp | 4 +- src/hotspot/share/oops/method.cpp | 30 +- src/hotspot/share/runtime/init.cpp | 1 + src/hotspot/share/runtime/sharedRuntime.cpp | 344 ++++++++++++++----- src/hotspot/share/runtime/sharedRuntime.hpp | 68 +++- 11 files changed, 368 insertions(+), 170 deletions(-) diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp index dd5a9003d95..52d7c6f7bcc 100644 --- a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp +++ b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp @@ -28,6 +28,7 @@ #endif #include "asm/macroAssembler.hpp" #include "asm/macroAssembler.inline.hpp" +#include "cds/cdsConfig.hpp" #include "code/SCCache.hpp" #include "code/compiledIC.hpp" #include "code/debugInfoRec.hpp" @@ -677,19 +678,6 @@ static void patch_callers_callsite(MacroAssembler *masm) { __ bind(L); } -static void range_check(MacroAssembler* masm, Register pc_reg, Register temp_reg, - address code_start, address code_end, - Label& L_ok) { - Label L_fail; - __ lea(temp_reg, AddressLiteral(code_start, relocInfo::none)); - __ cmpptr(pc_reg, temp_reg); - __ jcc(Assembler::belowEqual, L_fail); - __ lea(temp_reg, AddressLiteral(code_end, relocInfo::none)); - __ cmpptr(pc_reg, temp_reg); - __ jcc(Assembler::below, L_ok); - __ bind(L_fail); -} - static void gen_c2i_adapter(MacroAssembler *masm, int total_args_passed, int comp_args_on_stack, @@ -872,41 +860,6 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, // If this happens, control eventually transfers back to the compiled // caller, but with an uncorrected stack, causing delayed havoc. - if (VerifyAdapterCalls && - (Interpreter::code() != nullptr || StubRoutines::final_stubs_code() != nullptr)) { - // So, let's test for cascading c2i/i2c adapters right now. - // assert(Interpreter::contains($return_addr) || - // StubRoutines::contains($return_addr), - // "i2c adapter must return to an interpreter frame"); - __ block_comment("verify_i2c { "); - // Pick up the return address - __ movptr(rax, Address(rsp, 0)); - Label L_ok; - if (Interpreter::code() != nullptr) { - range_check(masm, rax, r11, - Interpreter::code()->code_start(), - Interpreter::code()->code_end(), - L_ok); - } - if (StubRoutines::initial_stubs_code() != nullptr) { - range_check(masm, rax, r11, - StubRoutines::initial_stubs_code()->code_begin(), - StubRoutines::initial_stubs_code()->code_end(), - L_ok); - } - if (StubRoutines::final_stubs_code() != nullptr) { - range_check(masm, rax, r11, - StubRoutines::final_stubs_code()->code_begin(), - StubRoutines::final_stubs_code()->code_end(), - L_ok); - } - const char* msg = "i2c adapter must return to an interpreter frame"; - __ block_comment(msg); - __ stop(msg); - __ bind(L_ok); - __ block_comment("} verify_i2ce "); - } - // Must preserve original SP for loading incoming arguments because // we need to align the outgoing SP for compiled code. __ movptr(r11, rsp); @@ -1051,27 +1004,27 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, } // --------------------------------------------------------------- -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint) { +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* entry) { address i2c_entry = __ pc(); - const char* name = AdapterHandlerLibrary::name(fingerprint); - const uint32_t id = AdapterHandlerLibrary::id(fingerprint); + const char* name = AdapterHandlerLibrary::name(entry->fingerprint()); + const uint32_t id = AdapterHandlerLibrary::id(entry->fingerprint()); // we need to avoid restoring code that might refer to stub routines // until we know they are available likewise we don't want to save // early generated adapters that try to refernce them - bool attempt_save_restore = (StubRoutines::final_stubs_code() != nullptr); - if (attempt_save_restore) { + { // see if we can load the code rather than generate it CodeBuffer* buffer = masm->code(); uint32_t offsets[4]; if (SCCache::load_adapter(buffer, id, name, offsets)) { assert(offsets[0] == 0, "sanity check"); - return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, i2c_entry + offsets[1], i2c_entry + offsets[2], i2c_entry + offsets[3]); + entry->set_entry_points(i2c_entry, i2c_entry + offsets[1], i2c_entry + offsets[2], i2c_entry + offsets[3]); + return; } } @@ -1134,7 +1087,7 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - if (attempt_save_restore) { + if (CDSConfig::is_dumping_adapters()) { // try to save generated code CodeBuffer* buffer = masm->code(); uint32_t offsets[4]; @@ -1145,7 +1098,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm SCCache::store_adapter(buffer, id, name, offsets); } - return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + entry->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + return; } int SharedRuntime::c_calling_convention(const BasicType *sig_bt, diff --git a/src/hotspot/share/cds/cdsConfig.cpp b/src/hotspot/share/cds/cdsConfig.cpp index 04fab431e7e..c296b4d29b1 100644 --- a/src/hotspot/share/cds/cdsConfig.cpp +++ b/src/hotspot/share/cds/cdsConfig.cpp @@ -867,3 +867,7 @@ void CDSConfig::disable_dumping_cached_code() { void CDSConfig::enable_dumping_cached_code() { _is_dumping_cached_code = true; } + +bool CDSConfig::is_dumping_adapters() { + return (ArchiveAdapters && is_dumping_final_static_archive()); +} diff --git a/src/hotspot/share/cds/cdsConfig.hpp b/src/hotspot/share/cds/cdsConfig.hpp index c37c710a9b8..db522420b32 100644 --- a/src/hotspot/share/cds/cdsConfig.hpp +++ b/src/hotspot/share/cds/cdsConfig.hpp @@ -161,6 +161,7 @@ class CDSConfig : public AllStatic { static void disable_dumping_cached_code() NOT_CDS_RETURN; static void enable_dumping_cached_code() NOT_CDS_RETURN; + static bool is_dumping_adapters() NOT_CDS_RETURN_(false); }; #endif // SHARE_CDS_CDSCONFIG_HPP diff --git a/src/hotspot/share/cds/cds_globals.hpp b/src/hotspot/share/cds/cds_globals.hpp index 053b91b5f10..ced6b1a2612 100644 --- a/src/hotspot/share/cds/cds_globals.hpp +++ b/src/hotspot/share/cds/cds_globals.hpp @@ -168,6 +168,9 @@ \ product(bool, SkipArchiveHeapVerification, false, \ "Skip verification of CDS archive heap") \ + \ + product(bool, ArchiveAdapters, false, \ + "Archive AdapterFingerPrint and AdapterHandlerEntry") \ // end of CDS_FLAGS diff --git a/src/hotspot/share/cds/cppVtables.cpp b/src/hotspot/share/cds/cppVtables.cpp index 138268ca4fd..9faf3e4990c 100644 --- a/src/hotspot/share/cds/cppVtables.cpp +++ b/src/hotspot/share/cds/cppVtables.cpp @@ -282,6 +282,8 @@ intptr_t* CppVtables::get_archived_vtable(MetaspaceObj::Type msotype, address ob case MetaspaceObj::AnnotationsType: case MetaspaceObj::SharedClassPathEntryType: case MetaspaceObj::RecordComponentType: + case MetaspaceObj::AdapterHandlerEntryType: + case MetaspaceObj::AdapterFingerPrintType: // These have no vtables. break; default: diff --git a/src/hotspot/share/cds/metaspaceShared.cpp b/src/hotspot/share/cds/metaspaceShared.cpp index c7531c751bb..d550807f33a 100644 --- a/src/hotspot/share/cds/metaspaceShared.cpp +++ b/src/hotspot/share/cds/metaspaceShared.cpp @@ -450,6 +450,8 @@ void MetaspaceShared::serialize(SerializeClosure* soc) { LambdaFormInvokers::serialize(soc); + AdapterHandlerLibrary::serialize(soc); + soc->do_tag(666); } @@ -556,6 +558,9 @@ char* VM_PopulateDumpSharedSpace::dump_read_only_tables() { // Write lambform lines into archive LambdaFormInvokers::dump_static_archive_invokers(); + if (CDSConfig::is_dumping_adapters()) { + AdapterHandlerLibrary::dump_adapter_handler_list(); + } // Write module name into archive CDS_JAVA_HEAP_ONLY(Modules::dump_main_module_name();) // Write the other data to the output array. diff --git a/src/hotspot/share/memory/allocation.hpp b/src/hotspot/share/memory/allocation.hpp index 500b4d29f16..7ee0ff0c47b 100644 --- a/src/hotspot/share/memory/allocation.hpp +++ b/src/hotspot/share/memory/allocation.hpp @@ -318,7 +318,9 @@ class MetaspaceObj { f(KlassTrainingData) \ f(MethodTrainingData) \ f(CompileTrainingData) \ - f(SharedClassPathEntry) + f(SharedClassPathEntry) \ + f(AdapterHandlerEntry) \ + f(AdapterFingerPrint) #define METASPACE_OBJ_TYPE_DECLARE(name) name ## Type, #define METASPACE_OBJ_TYPE_NAME_CASE(name) case name ## Type: return #name; diff --git a/src/hotspot/share/oops/method.cpp b/src/hotspot/share/oops/method.cpp index 7a210acf496..b3d4156a94a 100644 --- a/src/hotspot/share/oops/method.cpp +++ b/src/hotspot/share/oops/method.cpp @@ -399,6 +399,11 @@ void Method::metaspace_pointers_do(MetaspaceClosure* it) { } else { it->push(&_constMethod); } + if (CDSConfig::is_dumping_adapters()) { + if (!AdapterHandlerLibrary::is_abstract_method_adapter(_adapter)) { + it->push(&_adapter); + } + } it->push(&_method_data); it->push(&_method_counters); NOT_PRODUCT(it->push(&_name);) @@ -412,6 +417,9 @@ void Method::metaspace_pointers_do(MetaspaceClosure* it) { void Method::remove_unshareable_info() { unlink_method(); + if (CDSConfig::is_dumping_adapters() && _adapter != nullptr) { + _adapter->remove_unshareable_info(); + } if (method_data() != nullptr) { method_data()->remove_unshareable_info(); } @@ -423,6 +431,10 @@ void Method::remove_unshareable_info() { void Method::restore_unshareable_info(TRAPS) { assert(is_method() && is_valid_method(this), "ensure C++ vtable is restored"); + if (_adapter != nullptr) { + _adapter->restore_unshareable_info(CHECK); + _from_compiled_entry = _adapter->get_c2i_entry(); + } if (method_data() != nullptr) { method_data()->restore_unshareable_info(CHECK); } @@ -1211,7 +1223,9 @@ void Method::unlink_code() { void Method::unlink_method() { assert(CDSConfig::is_dumping_archive(), "sanity"); _code = nullptr; - _adapter = nullptr; + if (!CDSConfig::is_dumping_adapters() || AdapterHandlerLibrary::is_abstract_method_adapter(_adapter)) { + _adapter = nullptr; + } _i2i_entry = nullptr; _from_compiled_entry = nullptr; _from_interpreted_entry = nullptr; @@ -1256,17 +1270,21 @@ void Method::link_method(const methodHandle& h_method, TRAPS) { ClassLoader::perf_ik_link_methods_count()->inc(); } + // TODO: how to identify code cache full situation now that the adapter() can be + // non-null if AOT cache is in use +#if 0 // If the code cache is full, we may reenter this function for the // leftover methods that weren't linked. if (adapter() != nullptr) { return; } +#endif assert( _code == nullptr, "nothing compiled yet" ); // Setup interpreter entrypoint assert(this == h_method(), "wrong h_method()" ); - assert(adapter() == nullptr, "init'd to null"); + assert(adapter() == nullptr || adapter()->is_linked(), "init'd to null or restored from cache"); address entry = Interpreter::entry_for_method(h_method); assert(entry != nullptr, "interpreter entry must be non-null"); // Sets both _i2i_entry and _from_interpreted_entry @@ -1287,7 +1305,9 @@ void Method::link_method(const methodHandle& h_method, TRAPS) { // called from the vtable. We need adapters on such methods that get loaded // later. Ditto for mega-morphic itable calls. If this proves to be a // problem we'll make these lazily later. - (void) make_adapters(h_method, CHECK); + if (_adapter == nullptr) { + (void) make_adapters(h_method, CHECK); + } // ONLY USE the h_method now as make_adapter may have blocked @@ -1568,6 +1588,10 @@ methodHandle Method::make_method_handle_intrinsic(vmIntrinsics::ID iid, #if INCLUDE_CDS void Method::restore_archived_method_handle_intrinsic(methodHandle m, TRAPS) { + if (m->adapter() != nullptr) { + m->adapter()->restore_unshareable_info(CHECK); + m->set_from_compiled_entry(m->adapter()->get_c2i_entry()); + } m->link_method(m, CHECK); if (m->intrinsic_id() == vmIntrinsics::_linkToNative) { diff --git a/src/hotspot/share/runtime/init.cpp b/src/hotspot/share/runtime/init.cpp index 704440bf16d..58b3942ef85 100644 --- a/src/hotspot/share/runtime/init.cpp +++ b/src/hotspot/share/runtime/init.cpp @@ -167,6 +167,7 @@ jint init_globals() { VMRegImpl::set_regName(); // need this before generate_stubs (for printing oop maps). SharedRuntime::generate_stubs(); SCCache::init_shared_blobs_table(); // need this after generate_stubs + SharedRuntime::init_adapter_library(); // do this after SCCache::init_shared_blobs_table return JNI_OK; } diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 14297b99ec4..65d917210eb 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -23,11 +23,14 @@ */ #include "precompiled.hpp" +#include "cds/archiveBuilder.hpp" +#include "cds/archiveUtils.inline.hpp" #include "classfile/classLoader.hpp" #include "classfile/javaClasses.inline.hpp" #include "classfile/stringTable.hpp" #include "classfile/vmClasses.hpp" #include "classfile/vmSymbols.hpp" +#include "code/SCCache.hpp" #include "code/codeCache.hpp" #include "code/compiledIC.hpp" #include "code/nmethod.inline.hpp" @@ -156,8 +159,6 @@ void SharedRuntime::generate_stubs() { generate_throw_exception(SharedStubId::throw_NullPointerException_at_call_id, CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call)); - AdapterHandlerLibrary::initialize(); - #if COMPILER2_OR_JVMCI // Vectors are generated only by C2 and JVMCI. bool support_wide = is_wide_vector(MaxVectorSize); @@ -189,6 +190,10 @@ void SharedRuntime::generate_stubs() { } } +void SharedRuntime::init_adapter_library() { + AdapterHandlerLibrary::initialize(); +} + void SharedRuntime::print_counters_on(outputStream* st) { st->print_cr("SharedRuntime:"); if (UsePerfData) { @@ -2263,7 +2268,7 @@ static int _compact; // number of equals calls with compact signature // A simple wrapper class around the calling convention information // that allows sharing of adapters for the same calling convention. -class AdapterFingerPrint : public CHeapObj { +class AdapterFingerPrint : public MetaspaceObj { private: enum { _basic_type_bits = 4, @@ -2274,12 +2279,29 @@ class AdapterFingerPrint : public CHeapObj { // TO DO: Consider integrating this with a more global scheme for compressing signatures. // For now, 4 bits per components (plus T_VOID gaps after double/long) is not excessive. - union { - int _compact[_compact_int_count]; - int* _fingerprint; - } _value; - int _length; // A negative length indicates the fingerprint is in the compact form, - // Otherwise _value._fingerprint is the array. + int _length; + int _value[_compact_int_count]; + + // Private construtor. Use allocate() to get an instance. + AdapterFingerPrint(int total_args_passed, BasicType* sig_bt) { + // Pack the BasicTypes with 8 per int + _length = (total_args_passed + (_basic_types_per_int-1)) / _basic_types_per_int; + int sig_index = 0; + for (int index = 0; index < _length; index++) { + int value = 0; + for (int byte = 0; sig_index < total_args_passed && byte < _basic_types_per_int; byte++) { + int bt = adapter_encoding(sig_bt[sig_index++]); + assert((bt & _basic_type_mask) == bt, "must fit in 4 bits"); + value = (value << _basic_type_bits) | bt; + } + _value[index] = value; + } + } + + // Call deallocate instead + ~AdapterFingerPrint() { + FreeHeap(this); + } // Remap BasicTypes that are handled equivalently by the adapters. // These are correct for the current system but someday it might be @@ -2316,57 +2338,39 @@ class AdapterFingerPrint : public CHeapObj { } } + void* operator new(size_t size, size_t fp_size) throw() { + assert(fp_size >= size, "sanity check"); + void* p = AllocateHeap(fp_size, mtCode); + memset(p, 0, fp_size); + return p; + } + public: - AdapterFingerPrint(int total_args_passed, BasicType* sig_bt) { - // The fingerprint is based on the BasicType signature encoded - // into an array of ints with eight entries per int. - int* ptr; + static int allocation_size(int total_args_passed, BasicType* sig_bt) { int len = (total_args_passed + (_basic_types_per_int-1)) / _basic_types_per_int; - if (len <= _compact_int_count) { - assert(_compact_int_count == 3, "else change next line"); - _value._compact[0] = _value._compact[1] = _value._compact[2] = 0; - // Storing the signature encoded as signed chars hits about 98% - // of the time. - _length = -len; - ptr = _value._compact; - } else { - _length = len; - _value._fingerprint = NEW_C_HEAP_ARRAY(int, _length, mtCode); - ptr = _value._fingerprint; - } + return sizeof(AdapterFingerPrint) + (len > _compact_int_count ? (len - _compact_int_count) * sizeof(int) : 0); + } - // Now pack the BasicTypes with 8 per int - int sig_index = 0; - for (int index = 0; index < len; index++) { - int value = 0; - for (int byte = 0; sig_index < total_args_passed && byte < _basic_types_per_int; byte++) { - int bt = adapter_encoding(sig_bt[sig_index++]); - assert((bt & _basic_type_mask) == bt, "must fit in 4 bits"); - value = (value << _basic_type_bits) | bt; - } - ptr[index] = value; - } + static AdapterFingerPrint* allocate(int total_args_passed, BasicType* sig_bt) { + int size_in_bytes = allocation_size(total_args_passed, sig_bt); + return new (size_in_bytes) AdapterFingerPrint(total_args_passed, sig_bt); } - ~AdapterFingerPrint() { - if (_length > 0) { - FREE_C_HEAP_ARRAY(int, _value._fingerprint); - } + static void deallocate(AdapterFingerPrint* fp) { + fp->~AdapterFingerPrint(); } int value(int index) { - if (_length < 0) { - return _value._compact[index]; - } - return _value._fingerprint[index]; + return _value[index]; } + int length() { if (_length < 0) return -_length; return _length; } bool is_compact() { - return _length <= 0; + return _length <= _compact_int_count; } unsigned int compute_hash() { @@ -2426,18 +2430,68 @@ class AdapterFingerPrint : public CHeapObj { return st.as_string(); } + BasicType* as_basic_type(int& nargs) { + nargs = 0; + GrowableArray btarray; + bool long_prev = false; + for (int i = 0; i < length(); i++) { + unsigned val = (unsigned)value(i); + // args are packed so that first/lower arguments are in the highest + // bits of each int value, so iterate from highest to the lowest + for (int j = 32 - _basic_type_bits; j >= 0; j -= _basic_type_bits) { + unsigned v = (val >> j) & _basic_type_mask; + if (v == 0) continue; + if (long_prev) { + long_prev = false; + if (v == T_VOID) { + btarray.append(T_LONG); + } else { + btarray.append(T_OBJECT); // it could be T_ARRAY; it shouldn't matter + } + } + switch (v) { + case T_INT: // fallthrough + case T_FLOAT: // fallthrough + case T_DOUBLE: + case T_VOID: + btarray.append((BasicType)v); + break; + case T_LONG: + long_prev = true; + break; + default: ShouldNotReachHere(); + } + } + } + if (long_prev) { + btarray.append(T_OBJECT); + } + + nargs = btarray.length(); + BasicType* sig_bt = NEW_RESOURCE_ARRAY(BasicType, nargs); + int index = 0; + GrowableArrayIterator iter = btarray.begin(); + while (iter != btarray.end()) { + sig_bt[index++] = *iter; + ++iter; + } + assert(index == btarray.length(), "sanity check"); +#ifdef ASSERT + { + AdapterFingerPrint* compare_fp = AdapterFingerPrint::allocate(nargs, sig_bt); + assert(this->equals(compare_fp), "sanity check"); + AdapterFingerPrint::deallocate(compare_fp); + } +#endif + return sig_bt; + } + bool equals(AdapterFingerPrint* other) { if (other->_length != _length) { return false; - } - if (_length < 0) { - assert(_compact_int_count == 3, "else change next line"); - return _value._compact[0] == other->_value._compact[0] && - _value._compact[1] == other->_value._compact[1] && - _value._compact[2] == other->_value._compact[2]; } else { for (int i = 0; i < _length; i++) { - if (_value._fingerprint[i] != other->_value._fingerprint[i]) { + if (_value[i] != other->_value[i]) { return false; } } @@ -2445,6 +2499,11 @@ class AdapterFingerPrint : public CHeapObj { return true; } + // methods required by virtue of being a MetaspaceObj + void metaspace_pointers_do(MetaspaceClosure* it) { return; /* nothing to do here */ } + int size() const { return heap_word_size(sizeof(AdapterFingerPrint) + (_length > _compact_int_count ? (_length - _compact_int_count) * sizeof(int) : 0)); } + MetaspaceObj::Type type() const { return AdapterFingerPrintType; } + static bool equals(AdapterFingerPrint* const& fp1, AdapterFingerPrint* const& fp2) { NOT_PRODUCT(_equals++); return fp1->equals(fp2); @@ -2461,16 +2520,16 @@ using AdapterHandlerTable = ResourceHashtable; static AdapterHandlerTable* _adapter_handler_table; +static GrowableArray* _adapter_handler_list = nullptr; // Find a entry with the same fingerprint if it exists -static AdapterHandlerEntry* lookup(int total_args_passed, BasicType* sig_bt) { +static AdapterHandlerEntry* lookup(AdapterFingerPrint* fp) { NOT_PRODUCT(_lookups++); assert_lock_strong(AdapterHandlerLibrary_lock); - AdapterFingerPrint fp(total_args_passed, sig_bt); - AdapterHandlerEntry** entry = _adapter_handler_table->get(&fp); + AdapterHandlerEntry** entry = _adapter_handler_table->get(fp); if (entry != nullptr) { #ifndef PRODUCT - if (fp.is_compact()) _compact++; + if (fp->is_compact()) _compact++; _hits++; #endif return *entry; @@ -2500,6 +2559,8 @@ AdapterHandlerEntry* AdapterHandlerLibrary::_int_arg_handler = nullptr; AdapterHandlerEntry* AdapterHandlerLibrary::_obj_arg_handler = nullptr; AdapterHandlerEntry* AdapterHandlerLibrary::_obj_int_arg_handler = nullptr; AdapterHandlerEntry* AdapterHandlerLibrary::_obj_obj_arg_handler = nullptr; +Array* AdapterHandlerLibrary::_archived_adapter_handler_list = nullptr; + const int AdapterHandlerLibrary_size = 16*K; BufferBlob* AdapterHandlerLibrary::_buffer = nullptr; @@ -2535,6 +2596,7 @@ void AdapterHandlerLibrary::initialize() { AdapterBlob* obj_obj_arg_blob = nullptr; { _adapter_handler_table = new (mtCode) AdapterHandlerTable(); + populate_adapter_handler_table(); MutexLocker mu(AdapterHandlerLibrary_lock); // Create a special handler for abstract methods. Abstract methods @@ -2543,30 +2605,30 @@ void AdapterHandlerLibrary::initialize() { // Pass wrong_method_abstract for the c2i transitions to return // AbstractMethodError for invalid invocations. address wrong_method_abstract = SharedRuntime::get_handle_wrong_method_abstract_stub(); - _abstract_method_handler = AdapterHandlerLibrary::new_entry(new AdapterFingerPrint(0, nullptr), + _abstract_method_handler = AdapterHandlerLibrary::new_entry(AdapterFingerPrint::allocate(0, nullptr), SharedRuntime::throw_AbstractMethodError_entry(), wrong_method_abstract, wrong_method_abstract); _buffer = BufferBlob::create("adapters", AdapterHandlerLibrary_size); - _no_arg_handler = create_adapter(no_arg_blob, 0, nullptr, true); + _no_arg_handler = create_simple_adapter(no_arg_blob, 0, nullptr); BasicType obj_args[] = { T_OBJECT }; - _obj_arg_handler = create_adapter(obj_arg_blob, 1, obj_args, true); + _obj_arg_handler = create_simple_adapter(obj_arg_blob, 1, obj_args); BasicType int_args[] = { T_INT }; - _int_arg_handler = create_adapter(int_arg_blob, 1, int_args, true); + _int_arg_handler = create_simple_adapter(int_arg_blob, 1, int_args); BasicType obj_int_args[] = { T_OBJECT, T_INT }; - _obj_int_arg_handler = create_adapter(obj_int_arg_blob, 2, obj_int_args, true); + _obj_int_arg_handler = create_simple_adapter(obj_int_arg_blob, 2, obj_int_args); BasicType obj_obj_args[] = { T_OBJECT, T_OBJECT }; - _obj_obj_arg_handler = create_adapter(obj_obj_arg_blob, 2, obj_obj_args, true); + _obj_obj_arg_handler = create_simple_adapter(obj_obj_arg_blob, 2, obj_obj_args); assert(no_arg_blob != nullptr && - obj_arg_blob != nullptr && - int_arg_blob != nullptr && - obj_int_arg_blob != nullptr && - obj_obj_arg_blob != nullptr, "Initial adapters must be properly created"); + obj_arg_blob != nullptr && + int_arg_blob != nullptr && + obj_int_arg_blob != nullptr && + obj_obj_arg_blob != nullptr, "Initial adapters must be properly created"); } // Outside of the lock @@ -2577,14 +2639,28 @@ void AdapterHandlerLibrary::initialize() { post_adapter_creation(obj_obj_arg_blob, _obj_obj_arg_handler); } +AdapterHandlerEntry* AdapterHandlerLibrary::create_simple_adapter(AdapterBlob*& new_adapter, + int total_args_passed, + BasicType* sig_bt) { + AdapterFingerPrint* fp = AdapterFingerPrint::allocate(total_args_passed, sig_bt); + // We may find the adapter in the table if it is loaded from the AOT cache + AdapterHandlerEntry* entry = lookup(fp); + assert(entry == nullptr || (entry->is_shared() && !entry->is_linked()), "Non null AdapterHandlerEntry should be in the AOT cache in unlinked state"); + entry = create_adapter(new_adapter, fp, total_args_passed, sig_bt, true, entry); + if (entry->is_shared()) { + AdapterFingerPrint::deallocate(fp); + } + return entry; +} + AdapterHandlerEntry* AdapterHandlerLibrary::new_entry(AdapterFingerPrint* fingerprint, address i2c_entry, address c2i_entry, address c2i_unverified_entry, address c2i_no_clinit_check_entry) { // Insert an entry into the table - return new AdapterHandlerEntry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, - c2i_no_clinit_check_entry); + return AdapterHandlerEntry::allocate(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, + c2i_no_clinit_check_entry); } AdapterHandlerEntry* AdapterHandlerLibrary::get_simple_adapter(const methodHandle& method) { @@ -2694,23 +2770,27 @@ AdapterHandlerEntry* AdapterHandlerLibrary::get_adapter(const methodHandle& meth MutexLocker mu(AdapterHandlerLibrary_lock); // Lookup method signature's fingerprint - entry = lookup(total_args_passed, sig_bt); + AdapterFingerPrint *fp = AdapterFingerPrint::allocate(total_args_passed, sig_bt); + entry = lookup(fp); if (entry != nullptr) { #ifdef ASSERT if (VerifyAdapterSharing) { AdapterBlob* comparison_blob = nullptr; - AdapterHandlerEntry* comparison_entry = create_adapter(comparison_blob, total_args_passed, sig_bt, false); + AdapterFingerPrint* comparison_fp = AdapterFingerPrint::allocate(total_args_passed, sig_bt); + AdapterHandlerEntry* comparison_entry = create_adapter(comparison_blob, comparison_fp, total_args_passed, sig_bt, false); assert(comparison_blob == nullptr, "no blob should be created when creating an adapter for comparison"); assert(comparison_entry->compare_code(entry), "code must match"); + AdapterFingerPrint::deallocate(comparison_fp); // Release the one just created and return the original delete comparison_entry; } #endif + AdapterFingerPrint::deallocate(fp); return entry; } - entry = create_adapter(new_adapter, total_args_passed, sig_bt, /* allocate_code_blob */ true); + entry = create_adapter(new_adapter, fp, total_args_passed, sig_bt, /* allocate_code_blob */ true); } // Outside of the lock @@ -2721,19 +2801,15 @@ AdapterHandlerEntry* AdapterHandlerLibrary::get_adapter(const methodHandle& meth } AdapterHandlerEntry* AdapterHandlerLibrary::create_adapter(AdapterBlob*& new_adapter, + AdapterFingerPrint* fingerprint, int total_args_passed, BasicType* sig_bt, - bool allocate_code_blob) { + bool allocate_code_blob, + AdapterHandlerEntry* cached_entry) { if (log_is_enabled(Info, perf, class, link)) { ClassLoader::perf_method_adapters_count()->inc(); } - // StubRoutines::_final_stubs_code is initialized after this function can be called. As a result, - // VerifyAdapterCalls and VerifyAdapterSharing can fail if we re-use code that generated prior - // to all StubRoutines::_final_stubs_code being set. Checks refer to runtime range checks generated - // in an I2C stub that ensure that an I2C stub is called from an interpreter frame or stubs. - bool contains_all_checks = StubRoutines::final_stubs_code() != nullptr; - VMRegPair stack_regs[16]; VMRegPair* regs = (total_args_passed <= 16) ? stack_regs : NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed); @@ -2745,15 +2821,15 @@ AdapterHandlerEntry* AdapterHandlerLibrary::create_adapter(AdapterBlob*& new_ada buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs, sizeof(buffer_locs)/sizeof(relocInfo)); - // Make a C heap allocated version of the fingerprint to store in the adapter - AdapterFingerPrint* fingerprint = new AdapterFingerPrint(total_args_passed, sig_bt); MacroAssembler _masm(&buffer); - AdapterHandlerEntry* entry = SharedRuntime::generate_i2c2i_adapters(&_masm, - total_args_passed, - comp_args_on_stack, - sig_bt, - regs, - fingerprint); + AdapterHandlerEntry* entry = (cached_entry != nullptr) ? cached_entry : AdapterHandlerLibrary::new_entry(fingerprint); + ResourceMark rm; //for AdapterFingerPrint::as_basic_args_string() + SharedRuntime::generate_i2c2i_adapters(&_masm, + total_args_passed, + comp_args_on_stack, + sig_bt, + regs, + entry); #ifdef ASSERT if (VerifyAdapterSharing) { @@ -2793,9 +2869,7 @@ AdapterHandlerEntry* AdapterHandlerLibrary::create_adapter(AdapterBlob*& new_ada } #endif - // Add the entry only if the entry contains all required checks (see sharedRuntime_xxx.cpp) - // The checks are inserted only if -XX:+VerifyAdapterCalls is specified. - if (contains_all_checks || !VerifyAdapterCalls) { + if (cached_entry == nullptr) { assert_lock_strong(AdapterHandlerLibrary_lock); _adapter_handler_table->put(fingerprint, entry); } @@ -2826,9 +2900,50 @@ void AdapterHandlerEntry::relocate(address new_base) { assert(base_address() == new_base, ""); } +void AdapterHandlerEntry::metaspace_pointers_do(MetaspaceClosure* it) { + LogStreamHandle(Trace, cds) lsh; + if (lsh.is_enabled()) { + lsh.print("Iter(AdapterHandlerEntry): %p(%s)", this, _fingerprint->as_basic_args_string()); + lsh.cr(); + } + it->push(&_fingerprint); +} + +void AdapterHandlerEntry::remove_unshareable_info() { + _i2c_entry = nullptr; + _c2i_entry = nullptr; + _c2i_unverified_entry = nullptr; + _c2i_no_clinit_check_entry = nullptr; + _linked = false; +} + +void AdapterHandlerEntry::restore_unshareable_info(TRAPS) { + PerfTraceElapsedTime timer(ClassLoader::perf_method_adapters_time()); + if (is_linked()) { + return; + } + AdapterBlob* blob = nullptr; + { + MutexLocker mu(AdapterHandlerLibrary_lock); + assert(_fingerprint != nullptr, "_fingerprint must not be null"); +#ifdef ASSERT + AdapterHandlerEntry** entry = _adapter_handler_table->get(_fingerprint); + assert(entry != nullptr, "AdapterHandlerEntry not found in the table"); + assert(*entry == this, "sanity check"); +#endif + ResourceMark rm; + int nargs; + BasicType* bt = _fingerprint->as_basic_type(nargs); + AdapterHandlerLibrary::create_adapter(blob, _fingerprint, nargs, bt, true, this); + } + // Outside of the lock + if (blob != nullptr) { + post_adapter_creation(blob, this); + } + assert(_linked, "AdapterHandlerEntry must now be linked"); +} AdapterHandlerEntry::~AdapterHandlerEntry() { - delete _fingerprint; #ifdef ASSERT FREE_C_HEAP_ARRAY(unsigned char, _saved_code); #endif @@ -3187,6 +3302,55 @@ void AdapterHandlerEntry::print_adapter_on(outputStream* st) const { st->cr(); } +bool AdapterHandlerLibrary::is_abstract_method_adapter(AdapterHandlerEntry* entry) { + if (entry == _abstract_method_handler) { + return true; + } + return false; +} + +void AdapterHandlerLibrary::dump_adapter_handler_list() { + GrowableArray buffered_handlers; + auto handler_collector = [&] (AdapterFingerPrint* fp, AdapterHandlerEntry* entry) { + LogStreamHandle(Trace, cds) lsh; + if (ArchiveBuilder::current()->has_been_archived((address)entry)) { + AdapterHandlerEntry* buffered = ArchiveBuilder::current()->get_buffered_addr(entry); + assert(buffered != nullptr,"sanity check"); + buffered_handlers.append(buffered); + if (lsh.is_enabled()) { + address runtime_addr = (address)buffered + ArchiveBuilder::current()->buffer_to_requested_delta(); + log_trace(cds)("Added adapter handler %p (buffered=%p, runtime=%p)", entry, buffered, runtime_addr); + } + } else { + if (lsh.is_enabled()) { + log_trace(cds)("Skipping adapter handler %p as it is not archived", entry); + } + } + }; + _adapter_handler_table->iterate_all(handler_collector); + _archived_adapter_handler_list = ArchiveUtils::archive_array(&buffered_handlers); + log_info(cds)("Dumped adapter handlers list (size=%d)", _adapter_handler_table->number_of_entries()); +} + +void AdapterHandlerLibrary::serialize(SerializeClosure* sc) { + sc->do_ptr((void**)&_archived_adapter_handler_list); +} + +void AdapterHandlerLibrary::populate_adapter_handler_table() { + PerfTraceElapsedTime timer(ClassLoader::perf_method_adapters_time()); + assert(_adapter_handler_table != nullptr, "_adapter_handler_table is not initialized"); + if (_archived_adapter_handler_list != nullptr) { + log_info(cds)("Found adapter handlers list (size=%d) in the AOT cache", _archived_adapter_handler_list->length()); + log_trace(cds)("Populating _adapter_handler_table"); + for (int i =0; i < _archived_adapter_handler_list->length(); i++) { + AdapterHandlerEntry* entry = _archived_adapter_handler_list->at(i); + assert(entry->fingerprint() != nullptr, "Fingerpring must not be null"); + _adapter_handler_table->put(entry->fingerprint(), entry); + log_trace(cds)("Added entry=%p, fp=%p", entry, entry->fingerprint()); + } + } +} + JRT_LEAF(void, SharedRuntime::enable_stack_reserved_zone(JavaThread* current)) assert(current == JavaThread::current(), "pre-condition"); StackOverflow* overflow_state = current->stack_overflow_state(); diff --git a/src/hotspot/share/runtime/sharedRuntime.hpp b/src/hotspot/share/runtime/sharedRuntime.hpp index 6ec01114eea..d52ec7971a8 100644 --- a/src/hotspot/share/runtime/sharedRuntime.hpp +++ b/src/hotspot/share/runtime/sharedRuntime.hpp @@ -29,6 +29,7 @@ #include "code/vmreg.hpp" #include "interpreter/linkResolver.hpp" #include "memory/allStatic.hpp" +#include "memory/metaspaceClosure.hpp" #include "memory/resourceArea.hpp" #include "runtime/stubDeclarations.hpp" #include "utilities/macros.hpp" @@ -116,6 +117,7 @@ class SharedRuntime: AllStatic { // For c2: call to runtime to return a buffer lease. static RuntimeStub* generate_jfr_return_lease(); #endif + static void init_adapter_library(); static const char *stub_name(SharedStubId id) { assert(id > SharedStubId::NO_STUBID && id < SharedStubId::NUM_STUBIDS, "stub id out of range"); @@ -466,12 +468,12 @@ class SharedRuntime: AllStatic { // pointer as needed. This means the i2c adapter code doesn't need any special // handshaking path with compiled code to keep the stack walking correct. - static AdapterHandlerEntry* generate_i2c2i_adapters(MacroAssembler *_masm, - int total_args_passed, - int max_arg, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint); + static void generate_i2c2i_adapters(MacroAssembler *_masm, + int total_args_passed, + int max_arg, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* entry); static void gen_i2c_adapter(MacroAssembler *_masm, int total_args_passed, @@ -674,7 +676,7 @@ class SharedRuntime: AllStatic { // used by the adapters. The code generation happens here because it's very // similar to what the adapters have to do. -class AdapterHandlerEntry : public CHeapObj { +class AdapterHandlerEntry : public MetaspaceObj { friend class AdapterHandlerLibrary; private: @@ -683,6 +685,7 @@ class AdapterHandlerEntry : public CHeapObj { address _c2i_entry; address _c2i_unverified_entry; address _c2i_no_clinit_check_entry; + bool _linked; #ifdef ASSERT // Captures code and signature used to generate this adapter when @@ -698,7 +701,8 @@ class AdapterHandlerEntry : public CHeapObj { _i2c_entry(i2c_entry), _c2i_entry(c2i_entry), _c2i_unverified_entry(c2i_unverified_entry), - _c2i_no_clinit_check_entry(c2i_no_clinit_check_entry) + _c2i_no_clinit_check_entry(c2i_no_clinit_check_entry), + _linked(false) #ifdef ASSERT , _saved_code_length(0) #endif @@ -707,11 +711,28 @@ class AdapterHandlerEntry : public CHeapObj { ~AdapterHandlerEntry(); public: + static AdapterHandlerEntry* allocate(AdapterFingerPrint* fingerprint, + address i2c_entry, + address c2i_entry, + address c2i_unverified_entry, + address c2i_no_clinit_check_entry) + { + return new (mtCode) AdapterHandlerEntry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + } + void set_entry_points(address i2c_entry, address c2i_entry, address c2i_unverified_entry, address c2i_no_clinit_check_entry) { + _i2c_entry = i2c_entry; + _c2i_entry = c2i_entry; + _c2i_unverified_entry = c2i_unverified_entry; + _c2i_no_clinit_check_entry = c2i_no_clinit_check_entry; + _linked = true; + } + address get_i2c_entry() const { return _i2c_entry; } address get_c2i_entry() const { return _c2i_entry; } address get_c2i_unverified_entry() const { return _c2i_unverified_entry; } address get_c2i_no_clinit_check_entry() const { return _c2i_no_clinit_check_entry; } + bool is_linked() const { return _linked; } address base_address(); void relocate(address new_base); @@ -725,6 +746,13 @@ class AdapterHandlerEntry : public CHeapObj { //virtual void print_on(outputStream* st) const; DO NOT USE void print_adapter_on(outputStream* st) const; + + void metaspace_pointers_do(MetaspaceClosure* it); + int size() const { return heap_word_size(sizeof(AdapterHandlerEntry)); } + MetaspaceObj::Type type() const { return AdapterHandlerEntryType; } + + void remove_unshareable_info(); + void restore_unshareable_info(TRAPS); }; class AdapterHandlerLibrary: public AllStatic { @@ -737,23 +765,29 @@ class AdapterHandlerLibrary: public AllStatic { static AdapterHandlerEntry* _obj_arg_handler; static AdapterHandlerEntry* _obj_int_arg_handler; static AdapterHandlerEntry* _obj_obj_arg_handler; + static Array* _archived_adapter_handler_list; static BufferBlob* buffer_blob(); static void initialize(); - static AdapterHandlerEntry* create_adapter(AdapterBlob*& new_adapter, - int total_args_passed, - BasicType* sig_bt, - bool allocate_code_blob); static AdapterHandlerEntry* get_simple_adapter(const methodHandle& method); public: static AdapterHandlerEntry* new_entry(AdapterFingerPrint* fingerprint, - address i2c_entry, - address c2i_entry, - address c2i_unverified_entry, + address i2c_entry = nullptr, + address c2i_entry = nullptr, + address c2i_unverified_entry = nullptr, address c2i_no_clinit_check_entry = nullptr); static void create_native_wrapper(const methodHandle& method); static AdapterHandlerEntry* get_adapter(const methodHandle& method); + static AdapterHandlerEntry* create_simple_adapter(AdapterBlob*& new_adapter, + int total_args_passed, + BasicType* sig_bt); + static AdapterHandlerEntry* create_adapter(AdapterBlob*& new_adapter, + AdapterFingerPrint* fingerprint, + int total_args_passed, + BasicType* sig_bt, + bool allocate_code_blob, + AdapterHandlerEntry* cached_entry = nullptr); static void print_handler(const CodeBlob* b) { print_handler_on(tty, b); } static void print_handler_on(outputStream* st, const CodeBlob* b); @@ -764,6 +798,10 @@ class AdapterHandlerLibrary: public AllStatic { static void print_statistics_on(outputStream* st); #endif // PRODUCT + static bool is_abstract_method_adapter(AdapterHandlerEntry* adapter); + static void dump_adapter_handler_list(); + static void populate_adapter_handler_table(); + static void serialize(SerializeClosure* sc); }; #endif // SHARE_RUNTIME_SHAREDRUNTIME_HPP From 17de7eae1d9e39a1e93d64fbb31a950995036424 Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Tue, 15 Oct 2024 12:31:15 -0400 Subject: [PATCH 05/32] Cleanup stale pointers in TrainingData Signed-off-by: Ashutosh Mehra --- src/hotspot/share/oops/trainingData.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hotspot/share/oops/trainingData.cpp b/src/hotspot/share/oops/trainingData.cpp index 0b34c91ded8..db8969015c5 100644 --- a/src/hotspot/share/oops/trainingData.cpp +++ b/src/hotspot/share/oops/trainingData.cpp @@ -589,6 +589,8 @@ void MethodTrainingData::cleanup(Visitor& visitor) { if (_final_profile != nullptr && _final_profile->method() != _holder) { log_warning(cds)("Stale MDO for %s::%s", name()->as_klass_external_name(), signature()->as_utf8()); } + _final_profile = nullptr; + _final_counters = nullptr; _holder = nullptr; key()->make_empty(); } From 091626aa9e3d92ae60ff2674a9a60d416f3bdf1b Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Tue, 15 Oct 2024 16:00:49 -0400 Subject: [PATCH 06/32] Refactor code to lookup adapter code in AOT cache Signed-off-by: Ashutosh Mehra --- .../cpu/aarch64/sharedRuntime_aarch64.cpp | 42 +++------------ src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp | 29 ---------- src/hotspot/share/runtime/sharedRuntime.cpp | 54 ++++++++++++++----- src/hotspot/share/runtime/sharedRuntime.hpp | 7 +-- 4 files changed, 52 insertions(+), 80 deletions(-) diff --git a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp index 2c2ecd139cb..c1c05e60f37 100644 --- a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp @@ -712,30 +712,14 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, } // --------------------------------------------------------------- -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint) { +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* entry) { address i2c_entry = __ pc(); - const char* name = AdapterHandlerLibrary::name(fingerprint); - const uint32_t id = AdapterHandlerLibrary::id(fingerprint); - // we need to avoid restoring code that might refer to stub routines - // until we know they are available likewise we don't want to save - // early generated adapters that try to refernce them - bool attempt_save_restore = (StubRoutines::final_stubs_code() != nullptr); - if (attempt_save_restore) { - // see if we can load the code rather than generate it - CodeBuffer* buffer = masm->code(); - uint32_t offsets[4]; - if (SCCache::load_adapter(buffer, id, name, offsets)) { - assert(offsets[0] == 0, "sanity check"); - return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, i2c_entry + offsets[1], i2c_entry + offsets[2], i2c_entry + offsets[3]); - } - } - gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs); address c2i_unverified_entry = __ pc(); @@ -794,18 +778,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - if (attempt_save_restore) { - // try to save generated code - CodeBuffer* buffer = masm->code(); - uint32_t offsets[4]; - offsets[0] = 0; - offsets[1] = c2i_entry - i2c_entry; - offsets[2] = c2i_unverified_entry - i2c_entry; - offsets[3] = c2i_no_clinit_check_entry - i2c_entry; - SCCache::store_adapter(buffer, id, name, offsets); - } - - return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + entry->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + return; } static int c_calling_convention_priv(const BasicType *sig_bt, diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp index 52d7c6f7bcc..aa8040c5608 100644 --- a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp +++ b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp @@ -28,8 +28,6 @@ #endif #include "asm/macroAssembler.hpp" #include "asm/macroAssembler.inline.hpp" -#include "cds/cdsConfig.hpp" -#include "code/SCCache.hpp" #include "code/compiledIC.hpp" #include "code/debugInfoRec.hpp" #include "code/nativeInst.hpp" @@ -1012,22 +1010,6 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, AdapterHandlerEntry* entry) { address i2c_entry = __ pc(); - const char* name = AdapterHandlerLibrary::name(entry->fingerprint()); - const uint32_t id = AdapterHandlerLibrary::id(entry->fingerprint()); - // we need to avoid restoring code that might refer to stub routines - // until we know they are available likewise we don't want to save - // early generated adapters that try to refernce them - { - // see if we can load the code rather than generate it - CodeBuffer* buffer = masm->code(); - uint32_t offsets[4]; - if (SCCache::load_adapter(buffer, id, name, offsets)) { - assert(offsets[0] == 0, "sanity check"); - entry->set_entry_points(i2c_entry, i2c_entry + offsets[1], i2c_entry + offsets[2], i2c_entry + offsets[3]); - return; - } - } - gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs); // ------------------------------------------------------------------------- @@ -1087,17 +1069,6 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - if (CDSConfig::is_dumping_adapters()) { - // try to save generated code - CodeBuffer* buffer = masm->code(); - uint32_t offsets[4]; - offsets[0] = 0; - offsets[1] = c2i_entry - i2c_entry; - offsets[2] = c2i_unverified_entry - i2c_entry; - offsets[3] = c2i_no_clinit_check_entry - i2c_entry; - SCCache::store_adapter(buffer, id, name, offsets); - } - entry->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); return; } diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 65d917210eb..5cfc6314126 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -25,6 +25,7 @@ #include "precompiled.hpp" #include "cds/archiveBuilder.hpp" #include "cds/archiveUtils.inline.hpp" +#include "cds/cdsConfig.hpp" #include "classfile/classLoader.hpp" #include "classfile/javaClasses.inline.hpp" #include "classfile/stringTable.hpp" @@ -2800,6 +2801,20 @@ AdapterHandlerEntry* AdapterHandlerLibrary::get_adapter(const methodHandle& meth return entry; } +bool AdapterHandlerLibrary::lookup_aot_cache(AdapterHandlerEntry* entry, MacroAssembler* masm) { + const char* name = AdapterHandlerLibrary::name(entry->fingerprint()); + const uint32_t id = AdapterHandlerLibrary::id(entry->fingerprint()); + CodeBuffer* buffer = masm->code(); + uint32_t offsets[4]; + if (SCCache::load_adapter(buffer, id, name, offsets)) { + address i2c_entry = masm->pc(); + assert(offsets[0] == 0, "sanity check"); + entry->set_entry_points(i2c_entry, i2c_entry + offsets[1], i2c_entry + offsets[2], i2c_entry + offsets[3]); + return true; + } + return false; +} + AdapterHandlerEntry* AdapterHandlerLibrary::create_adapter(AdapterBlob*& new_adapter, AdapterFingerPrint* fingerprint, int total_args_passed, @@ -2810,27 +2825,38 @@ AdapterHandlerEntry* AdapterHandlerLibrary::create_adapter(AdapterBlob*& new_ada ClassLoader::perf_method_adapters_count()->inc(); } - VMRegPair stack_regs[16]; - VMRegPair* regs = (total_args_passed <= 16) ? stack_regs : NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed); - - // Get a description of the compiled java calling convention and the largest used (VMReg) stack slot usage - int comp_args_on_stack = SharedRuntime::java_calling_convention(sig_bt, regs, total_args_passed); BufferBlob* buf = buffer_blob(); // the temporary code buffer in CodeCache CodeBuffer buffer(buf); short buffer_locs[20]; buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs, sizeof(buffer_locs)/sizeof(relocInfo)); - MacroAssembler _masm(&buffer); + MacroAssembler masm(&buffer); AdapterHandlerEntry* entry = (cached_entry != nullptr) ? cached_entry : AdapterHandlerLibrary::new_entry(fingerprint); - ResourceMark rm; //for AdapterFingerPrint::as_basic_args_string() - SharedRuntime::generate_i2c2i_adapters(&_masm, - total_args_passed, - comp_args_on_stack, - sig_bt, - regs, - entry); - + if (!lookup_aot_cache(entry, &masm)) { + VMRegPair stack_regs[16]; + VMRegPair* regs = (total_args_passed <= 16) ? stack_regs : NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed); + + // Get a description of the compiled java calling convention and the largest used (VMReg) stack slot usage + int comp_args_on_stack = SharedRuntime::java_calling_convention(sig_bt, regs, total_args_passed); + SharedRuntime::generate_i2c2i_adapters(&masm, + total_args_passed, + comp_args_on_stack, + sig_bt, + regs, + entry); + if (CDSConfig::is_dumping_adapters()) { + // try to save generated code + const char* name = AdapterHandlerLibrary::name(entry->fingerprint()); + const uint32_t id = AdapterHandlerLibrary::id(entry->fingerprint()); + uint32_t offsets[4]; + offsets[0] = 0; + offsets[1] = entry->get_c2i_entry() - entry->get_i2c_entry(); + offsets[2] = entry->get_c2i_unverified_entry() - entry->get_i2c_entry(); + offsets[3] = entry->get_c2i_no_clinit_check_entry() - entry->get_i2c_entry(); + SCCache::store_adapter(&buffer, id, name, offsets); + } + } #ifdef ASSERT if (VerifyAdapterSharing) { entry->save_code(buf->code_begin(), buffer.insts_size()); diff --git a/src/hotspot/share/runtime/sharedRuntime.hpp b/src/hotspot/share/runtime/sharedRuntime.hpp index d52ec7971a8..7c0e3e77414 100644 --- a/src/hotspot/share/runtime/sharedRuntime.hpp +++ b/src/hotspot/share/runtime/sharedRuntime.hpp @@ -769,7 +769,11 @@ class AdapterHandlerLibrary: public AllStatic { static BufferBlob* buffer_blob(); static void initialize(); + static AdapterHandlerEntry* create_simple_adapter(AdapterBlob*& new_adapter, + int total_args_passed, + BasicType* sig_bt); static AdapterHandlerEntry* get_simple_adapter(const methodHandle& method); + static bool lookup_aot_cache(AdapterHandlerEntry* entry, MacroAssembler* masm); public: static AdapterHandlerEntry* new_entry(AdapterFingerPrint* fingerprint, @@ -779,9 +783,6 @@ class AdapterHandlerLibrary: public AllStatic { address c2i_no_clinit_check_entry = nullptr); static void create_native_wrapper(const methodHandle& method); static AdapterHandlerEntry* get_adapter(const methodHandle& method); - static AdapterHandlerEntry* create_simple_adapter(AdapterBlob*& new_adapter, - int total_args_passed, - BasicType* sig_bt); static AdapterHandlerEntry* create_adapter(AdapterBlob*& new_adapter, AdapterFingerPrint* fingerprint, int total_args_passed, From b05860adbc43b7699caf7863e0d04ff6403e1c1e Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Tue, 15 Oct 2024 16:03:06 -0400 Subject: [PATCH 07/32] Add PerfCounter for tracking time to load adapter table Signed-off-by: Ashutosh Mehra --- src/hotspot/share/classfile/classLoader.cpp | 10 +++++++--- src/hotspot/share/classfile/classLoader.hpp | 2 ++ src/hotspot/share/runtime/sharedRuntime.cpp | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/classfile/classLoader.cpp b/src/hotspot/share/classfile/classLoader.cpp index e9208498a98..6233b731272 100644 --- a/src/hotspot/share/classfile/classLoader.cpp +++ b/src/hotspot/share/classfile/classLoader.cpp @@ -130,6 +130,7 @@ PerfCounter* ClassLoader::_perf_preinit_time = nullptr; PerfCounter* ClassLoader::_perf_preresolve_time = nullptr; PerfCounter* ClassLoader::_perf_ik_link_methods_time = nullptr; PerfCounter* ClassLoader::_perf_method_adapters_time = nullptr; +PerfCounter* ClassLoader::_perf_load_adapter_table_time = nullptr; PerfCounter* ClassLoader::_perf_ik_link_methods_count = nullptr; PerfCounter* ClassLoader::_perf_method_adapters_count = nullptr; PerfCounter* ClassLoader::_unsafe_defineClassCallCounter = nullptr; @@ -150,12 +151,14 @@ void ClassLoader::print_counters(outputStream *st) { // we print to the passed in outputStream as requested. if (log_is_enabled(Info, perf, class, link)) { st->print_cr("ClassLoader:"); - st->print_cr( " clinit: " JLONG_FORMAT_W(6) "us / " JLONG_FORMAT " events", + st->print_cr(" clinit: " JLONG_FORMAT_W(6) "us / " JLONG_FORMAT " events", ClassLoader::class_init_time_ms(), ClassLoader::class_init_count()); - st->print_cr(" link methods: " JLONG_FORMAT_W(6) "us / " JLONG_FORMAT " events", + st->print_cr(" link methods: " JLONG_FORMAT_W(6) "us / " JLONG_FORMAT " events", Management::ticks_to_us(_perf_ik_link_methods_time->get_value()) , _perf_ik_link_methods_count->get_value()); - st->print_cr(" method adapters: " JLONG_FORMAT_W(6) "us / " JLONG_FORMAT " events", + st->print_cr(" method adapters: " JLONG_FORMAT_W(6) "us / " JLONG_FORMAT " events", Management::ticks_to_us(_perf_method_adapters_time->get_value()) , _perf_method_adapters_count->get_value()); + st->print_cr(" load method adapters table: " JLONG_FORMAT_W(6) "us", + Management::ticks_to_us(_perf_load_adapter_table_time->get_value())); if (CountBytecodes || CountBytecodesPerThread) { st->print_cr("; executed " JLONG_FORMAT " bytecodes", ClassLoader::class_init_bytecodes_count()); } @@ -1496,6 +1499,7 @@ void ClassLoader::initialize(TRAPS) { NEWPERFTICKCOUNTER(_perf_ik_link_methods_time, SUN_CLS, "linkMethodsTime"); NEWPERFTICKCOUNTER(_perf_method_adapters_time, SUN_CLS, "makeAdaptersTime"); + NEWPERFTICKCOUNTER(_perf_load_adapter_table_time, SUN_CLS, "loadAdapterTableTime"); NEWPERFEVENTCOUNTER(_perf_ik_link_methods_count, SUN_CLS, "linkMethodsCount"); NEWPERFEVENTCOUNTER(_perf_method_adapters_count, SUN_CLS, "makeAdaptersCount"); diff --git a/src/hotspot/share/classfile/classLoader.hpp b/src/hotspot/share/classfile/classLoader.hpp index 0c44df8912a..666f7569173 100644 --- a/src/hotspot/share/classfile/classLoader.hpp +++ b/src/hotspot/share/classfile/classLoader.hpp @@ -177,6 +177,7 @@ class ClassLoader: AllStatic { static PerfCounter* _perf_preresolve_time; static PerfCounter* _perf_ik_link_methods_time; static PerfCounter* _perf_method_adapters_time; + static PerfCounter* _perf_load_adapter_table_time; static PerfCounter* _perf_ik_link_methods_count; static PerfCounter* _perf_method_adapters_count; @@ -315,6 +316,7 @@ class ClassLoader: AllStatic { static PerfCounter* perf_preresolve_time() { return _perf_preresolve_time; } static PerfCounter* perf_ik_link_methods_time() { return _perf_ik_link_methods_time; } static PerfCounter* perf_method_adapters_time() { return _perf_method_adapters_time; } + static PerfCounter* perf_load_adapter_table_time() { return _perf_load_adapter_table_time; } static PerfCounter* perf_ik_link_methods_count() { return _perf_ik_link_methods_count; } static PerfCounter* perf_method_adapters_count() { return _perf_method_adapters_count; } diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 5cfc6314126..9abe38bfa70 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -3363,7 +3363,7 @@ void AdapterHandlerLibrary::serialize(SerializeClosure* sc) { } void AdapterHandlerLibrary::populate_adapter_handler_table() { - PerfTraceElapsedTime timer(ClassLoader::perf_method_adapters_time()); + PerfTraceElapsedTime timer(ClassLoader::perf_load_adapter_table_time()); assert(_adapter_handler_table != nullptr, "_adapter_handler_table is not initialized"); if (_archived_adapter_handler_list != nullptr) { log_info(cds)("Found adapter handlers list (size=%d) in the AOT cache", _archived_adapter_handler_list->length()); From 7fe50ae7387935ac9f766db840127e8470a0cf78 Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Thu, 17 Oct 2024 10:06:04 -0400 Subject: [PATCH 08/32] Add assertion that ArchiveAdapters is set in SCCache::store_adapter Signed-off-by: Ashutosh Mehra --- src/hotspot/share/code/SCCache.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hotspot/share/code/SCCache.cpp b/src/hotspot/share/code/SCCache.cpp index d0d895cdb19..924b5a01468 100644 --- a/src/hotspot/share/code/SCCache.cpp +++ b/src/hotspot/share/code/SCCache.cpp @@ -2353,6 +2353,7 @@ bool SCCache::write_relocations(CodeBuffer* buffer, uint& all_reloc_size) { } bool SCCache::store_adapter(CodeBuffer* buffer, uint32_t id, const char* name, uint32_t offsets[4]) { + assert(CDSConfig::is_dumping_adapters(), "must be"); SCCache* cache = open_for_write(); if (cache == nullptr) { return false; From fb5b88ca5da6606bbfebf75f685331743f529756 Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Thu, 17 Oct 2024 18:15:07 -0400 Subject: [PATCH 09/32] Archive adapter_handler_table to the AOT cache Signed-off-by: Ashutosh Mehra --- src/hotspot/share/cds/archiveBuilder.cpp | 3 +- src/hotspot/share/cds/metaspaceShared.cpp | 3 +- src/hotspot/share/runtime/sharedRuntime.cpp | 89 ++++++++++++++++++--- src/hotspot/share/runtime/sharedRuntime.hpp | 16 ++++ 4 files changed, 97 insertions(+), 14 deletions(-) diff --git a/src/hotspot/share/cds/archiveBuilder.cpp b/src/hotspot/share/cds/archiveBuilder.cpp index 9a124d7c6b9..2394a01683b 100644 --- a/src/hotspot/share/cds/archiveBuilder.cpp +++ b/src/hotspot/share/cds/archiveBuilder.cpp @@ -359,7 +359,8 @@ size_t ArchiveBuilder::estimate_archive_size() { size_t symbol_table_est = SymbolTable::estimate_size_for_archive(); size_t dictionary_est = SystemDictionaryShared::estimate_size_for_archive(); size_t training_data_est = TrainingData::estimate_size_for_archive(); - _estimated_hashtable_bytes = symbol_table_est + dictionary_est + training_data_est; + size_t adapter_data_est = AdapterHandlerLibrary::estimate_size_for_archive(); + _estimated_hashtable_bytes = symbol_table_est + dictionary_est + training_data_est + adapter_data_est; if (CDSConfig::is_dumping_aot_linked_classes()) { _estimated_hashtable_bytes += _klasses->length() * 16 * sizeof(Klass*); diff --git a/src/hotspot/share/cds/metaspaceShared.cpp b/src/hotspot/share/cds/metaspaceShared.cpp index d550807f33a..f96af1bc8c1 100644 --- a/src/hotspot/share/cds/metaspaceShared.cpp +++ b/src/hotspot/share/cds/metaspaceShared.cpp @@ -448,9 +448,9 @@ void MetaspaceShared::serialize(SerializeClosure* soc) { CDS_JAVA_HEAP_ONLY(ClassLoaderDataShared::serialize(soc);) soc->do_ptr((void**)&_archived_method_handle_intrinsics); - LambdaFormInvokers::serialize(soc); AdapterHandlerLibrary::serialize(soc); + AdapterHandlerLibrary::serialize_shared_table_header(soc); soc->do_tag(666); } @@ -560,6 +560,7 @@ char* VM_PopulateDumpSharedSpace::dump_read_only_tables() { LambdaFormInvokers::dump_static_archive_invokers(); if (CDSConfig::is_dumping_adapters()) { AdapterHandlerLibrary::dump_adapter_handler_list(); + AdapterHandlerLibrary::archive_adapter_table(); } // Write module name into archive CDS_JAVA_HEAP_ONLY(Modules::dump_main_module_name();) diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 9abe38bfa70..32541111eff 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -2263,7 +2263,8 @@ void SharedRuntime::print_call_statistics_on(outputStream* st) { #ifndef PRODUCT static int _lookups; // number of calls to lookup static int _equals; // number of buckets checked with matching hash -static int _hits; // number of successful lookups +static int _archived_hits; // number of successful lookups in archived table +static int _runtime_hits; // number of successful lookups in runtime table static int _compact; // number of equals calls with compact signature #endif @@ -2524,16 +2525,27 @@ static AdapterHandlerTable* _adapter_handler_table; static GrowableArray* _adapter_handler_list = nullptr; // Find a entry with the same fingerprint if it exists -static AdapterHandlerEntry* lookup(AdapterFingerPrint* fp) { +AdapterHandlerEntry* AdapterHandlerLibrary::lookup(AdapterFingerPrint* fp) { NOT_PRODUCT(_lookups++); - assert_lock_strong(AdapterHandlerLibrary_lock); - AdapterHandlerEntry** entry = _adapter_handler_table->get(fp); + // Search archived table first. It is read-only table so can be searched without lock + AdapterHandlerEntry* entry = _archived_adapter_table.lookup(fp, fp->compute_hash(), 0 /* unused */); if (entry != nullptr) { +#ifndef PRODUCT + if (fp->is_compact()) { + _compact++; + } + _archived_hits++; +#endif + return entry; + } + assert_lock_strong(AdapterHandlerLibrary_lock); + AdapterHandlerEntry** entry_p = _adapter_handler_table->get(fp); + if (entry_p != nullptr) { #ifndef PRODUCT if (fp->is_compact()) _compact++; - _hits++; + _runtime_hits++; #endif - return *entry; + return *entry_p; } return nullptr; } @@ -2547,8 +2559,9 @@ void AdapterHandlerLibrary::print_statistics_on(outputStream* st) { ts.print(st, "AdapterHandlerTable"); st->print_cr("AdapterHandlerTable (table_size=%d, entries=%d)", _adapter_handler_table->table_size(), _adapter_handler_table->number_of_entries()); - st->print_cr("AdapterHandlerTable: lookups %d equals %d hits %d compact %d", - _lookups, _equals, _hits, _compact); + int total_hits = _archived_hits + _runtime_hits; + st->print_cr("AdapterHandlerTable: lookups %d equals %d hits %d (archived=%d+runtime=%d) compact %d", + _lookups, _equals, total_hits, _archived_hits, _runtime_hits, _compact); } #endif // !PRODUCT @@ -2561,6 +2574,7 @@ AdapterHandlerEntry* AdapterHandlerLibrary::_obj_arg_handler = nullptr; AdapterHandlerEntry* AdapterHandlerLibrary::_obj_int_arg_handler = nullptr; AdapterHandlerEntry* AdapterHandlerLibrary::_obj_obj_arg_handler = nullptr; Array* AdapterHandlerLibrary::_archived_adapter_handler_list = nullptr; +ArchivedAdapterTable AdapterHandlerLibrary::_archived_adapter_table; const int AdapterHandlerLibrary_size = 16*K; BufferBlob* AdapterHandlerLibrary::_buffer = nullptr; @@ -2597,7 +2611,7 @@ void AdapterHandlerLibrary::initialize() { AdapterBlob* obj_obj_arg_blob = nullptr; { _adapter_handler_table = new (mtCode) AdapterHandlerTable(); - populate_adapter_handler_table(); + //populate_adapter_handler_table(); MutexLocker mu(AdapterHandlerLibrary_lock); // Create a special handler for abstract methods. Abstract methods @@ -2953,9 +2967,9 @@ void AdapterHandlerEntry::restore_unshareable_info(TRAPS) { MutexLocker mu(AdapterHandlerLibrary_lock); assert(_fingerprint != nullptr, "_fingerprint must not be null"); #ifdef ASSERT - AdapterHandlerEntry** entry = _adapter_handler_table->get(_fingerprint); - assert(entry != nullptr, "AdapterHandlerEntry not found in the table"); - assert(*entry == this, "sanity check"); + AdapterHandlerEntry* entry = AdapterHandlerLibrary::lookup(_fingerprint); + //AdapterHandlerEntry** entry = _adapter_handler_table->get(_fingerprint); + assert(entry == this, "sanity check"); #endif ResourceMark rm; int nargs; @@ -3335,6 +3349,57 @@ bool AdapterHandlerLibrary::is_abstract_method_adapter(AdapterHandlerEntry* entr return false; } +class CopyAdapterTableToArchive : StackObj { +private: + CompactHashtableWriter* _writer; + ArchiveBuilder* _builder; +public: + CopyAdapterTableToArchive(CompactHashtableWriter* writer) : _writer(writer), + _builder(ArchiveBuilder::current()) + {} + + bool do_entry(AdapterFingerPrint* fp, AdapterHandlerEntry* entry) { + LogStreamHandle(Trace, cds) lsh; + if (ArchiveBuilder::current()->has_been_archived((address)entry)) { + assert(ArchiveBuilder::current()->has_been_archived((address)fp), "must be"); + AdapterFingerPrint* buffered_fp = ArchiveBuilder::current()->get_buffered_addr(fp); + assert(buffered_fp != nullptr,"sanity check"); + AdapterHandlerEntry* buffered_entry = ArchiveBuilder::current()->get_buffered_addr(entry); + assert(buffered_entry != nullptr,"sanity check"); + + uint hash = fp->compute_hash(); + u4 delta = _builder->buffer_to_offset_u4((address)buffered_entry); + _writer->add(hash, delta); + if (lsh.is_enabled()) { + address fp_runtime_addr = (address)buffered_fp + ArchiveBuilder::current()->buffer_to_requested_delta(); + address entry_runtime_addr = (address)buffered_entry + ArchiveBuilder::current()->buffer_to_requested_delta(); + log_trace(cds)("Added fp=%p, entry=%p to the archived adater table", buffered_fp, buffered_entry); + } + } else { + if (lsh.is_enabled()) { + log_trace(cds)("Skipping adapter handler %p as it is not archived", entry); + } + } + return true; + } +}; + +size_t AdapterHandlerLibrary::estimate_size_for_archive() { + return CompactHashtableWriter::estimate_size(_adapter_handler_table->number_of_entries()); +} + +void AdapterHandlerLibrary::archive_adapter_table() { + CompactHashtableStats stats; + CompactHashtableWriter writer(_adapter_handler_table->number_of_entries(), &stats); + CopyAdapterTableToArchive copy(&writer); + _adapter_handler_table->iterate(©); + writer.dump(&_archived_adapter_table, "archived adapter table"); +} + +void AdapterHandlerLibrary::serialize_shared_table_header(SerializeClosure* soc) { + _archived_adapter_table.serialize_header(soc); +} + void AdapterHandlerLibrary::dump_adapter_handler_list() { GrowableArray buffered_handlers; auto handler_collector = [&] (AdapterFingerPrint* fp, AdapterHandlerEntry* entry) { diff --git a/src/hotspot/share/runtime/sharedRuntime.hpp b/src/hotspot/share/runtime/sharedRuntime.hpp index 7c0e3e77414..adfe7c50a4b 100644 --- a/src/hotspot/share/runtime/sharedRuntime.hpp +++ b/src/hotspot/share/runtime/sharedRuntime.hpp @@ -25,6 +25,7 @@ #ifndef SHARE_RUNTIME_SHAREDRUNTIME_HPP #define SHARE_RUNTIME_SHAREDRUNTIME_HPP +#include "classfile/compactHashtable.hpp" #include "code/codeBlob.hpp" #include "code/vmreg.hpp" #include "interpreter/linkResolver.hpp" @@ -755,6 +756,8 @@ class AdapterHandlerEntry : public MetaspaceObj { void restore_unshareable_info(TRAPS); }; +class ArchivedAdapterTable; + class AdapterHandlerLibrary: public AllStatic { friend class SharedRuntime; private: @@ -766,6 +769,7 @@ class AdapterHandlerLibrary: public AllStatic { static AdapterHandlerEntry* _obj_int_arg_handler; static AdapterHandlerEntry* _obj_obj_arg_handler; static Array* _archived_adapter_handler_list; + static ArchivedAdapterTable _archived_adapter_table; static BufferBlob* buffer_blob(); static void initialize(); @@ -789,6 +793,7 @@ class AdapterHandlerLibrary: public AllStatic { BasicType* sig_bt, bool allocate_code_blob, AdapterHandlerEntry* cached_entry = nullptr); + static AdapterHandlerEntry* lookup(AdapterFingerPrint* fp); static void print_handler(const CodeBlob* b) { print_handler_on(tty, b); } static void print_handler_on(outputStream* st, const CodeBlob* b); @@ -800,9 +805,20 @@ class AdapterHandlerLibrary: public AllStatic { #endif // PRODUCT static bool is_abstract_method_adapter(AdapterHandlerEntry* adapter); + static size_t estimate_size_for_archive(); + static void archive_adapter_table(); + static void serialize_shared_table_header(SerializeClosure* soc); static void dump_adapter_handler_list(); static void populate_adapter_handler_table(); static void serialize(SerializeClosure* sc); + static bool EQUALS(AdapterHandlerEntry* entry, AdapterFingerPrint* fp, int len_unused) { + return entry->fingerprint() == fp; + } }; +class ArchivedAdapterTable : public OffsetCompactHashtable< + AdapterFingerPrint*, + AdapterHandlerEntry*, + AdapterHandlerLibrary::EQUALS> {}; + #endif // SHARE_RUNTIME_SHAREDRUNTIME_HPP From f6a79e37285c33b923422d58ea1c8efae925e5b6 Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Fri, 18 Oct 2024 11:03:58 -0400 Subject: [PATCH 10/32] Remove unused code Signed-off-by: Ashutosh Mehra --- .../cpu/aarch64/sharedRuntime_aarch64.cpp | 34 -------------- src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp | 45 ------------------- src/hotspot/share/cds/metaspaceShared.cpp | 2 - src/hotspot/share/classfile/classLoader.cpp | 4 -- src/hotspot/share/classfile/classLoader.hpp | 2 - src/hotspot/share/runtime/sharedRuntime.cpp | 43 ------------------ src/hotspot/share/runtime/sharedRuntime.hpp | 3 -- 7 files changed, 133 deletions(-) diff --git a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp index c1c05e60f37..b42a5a991f1 100644 --- a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp @@ -558,40 +558,6 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, // If this happens, control eventually transfers back to the compiled // caller, but with an uncorrected stack, causing delayed havoc. - if (VerifyAdapterCalls && - (Interpreter::code() != nullptr || StubRoutines::final_stubs_code() != nullptr)) { -#if 0 - // So, let's test for cascading c2i/i2c adapters right now. - // assert(Interpreter::contains($return_addr) || - // StubRoutines::contains($return_addr), - // "i2c adapter must return to an interpreter frame"); - __ block_comment("verify_i2c { "); - Label L_ok; - if (Interpreter::code() != nullptr) { - range_check(masm, rax, r11, - Interpreter::code()->code_start(), Interpreter::code()->code_end(), - L_ok); - } - if (StubRoutines::initial_stubs_code() != nullptr) { - range_check(masm, rax, r11, - StubRoutines::initial_stubs_code()->code_begin(), - StubRoutines::initial_stubs_code()->code_end(), - L_ok); - } - if (StubRoutines::final_stubs_code() != nullptr) { - range_check(masm, rax, r11, - StubRoutines::final_stubs_code()->code_begin(), - StubRoutines::final_stubs_code()->code_end(), - L_ok); - } - const char* msg = "i2c adapter must return to an interpreter frame"; - __ block_comment(msg); - __ stop(msg); - __ bind(L_ok); - __ block_comment("} verify_i2ce "); -#endif - } - // Cut-out for having no stack args. int comp_words_on_stack = align_up(comp_args_on_stack*VMRegImpl::stack_slot_size, wordSize)>>LogBytesPerWord; if (comp_args_on_stack) { diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp index c6ab00cc9ee..0abb35fda65 100644 --- a/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp +++ b/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp @@ -707,19 +707,6 @@ static void move_i2c_double(MacroAssembler *masm, XMMRegister r, Register saved_ __ movdbl(r, Address(saved_sp, next_val_off)); } -static void range_check(MacroAssembler* masm, Register pc_reg, Register temp_reg, - address code_start, address code_end, - Label& L_ok) { - Label L_fail; - __ lea(temp_reg, AddressLiteral(code_start, relocInfo::none)); - __ cmpptr(pc_reg, temp_reg); - __ jcc(Assembler::belowEqual, L_fail); - __ lea(temp_reg, AddressLiteral(code_end, relocInfo::none)); - __ cmpptr(pc_reg, temp_reg); - __ jcc(Assembler::below, L_ok); - __ bind(L_fail); -} - void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, int total_args_passed, int comp_args_on_stack, @@ -749,38 +736,6 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, // Pick up the return address __ movptr(rax, Address(rsp, 0)); - if (VerifyAdapterCalls && - (Interpreter::code() != nullptr || StubRoutines::final_stubs_code() != nullptr)) { - // So, let's test for cascading c2i/i2c adapters right now. - // assert(Interpreter::contains($return_addr) || - // StubRoutines::contains($return_addr), - // "i2c adapter must return to an interpreter frame"); - __ block_comment("verify_i2c { "); - Label L_ok; - if (Interpreter::code() != nullptr) { - range_check(masm, rax, rdi, - Interpreter::code()->code_start(), Interpreter::code()->code_end(), - L_ok); - } - if (StubRoutines::initial_stubs_code() != nullptr) { - range_check(masm, rax, rdi, - StubRoutines::initial_stubs_code()->code_begin(), - StubRoutines::initial_stubs_code()->code_end(), - L_ok); - } - if (StubRoutines::final_stubs_code() != nullptr) { - range_check(masm, rax, rdi, - StubRoutines::final_stubs_code()->code_begin(), - StubRoutines::final_stubs_code()->code_end(), - L_ok); - } - const char* msg = "i2c adapter must return to an interpreter frame"; - __ block_comment(msg); - __ stop(msg); - __ bind(L_ok); - __ block_comment("} verify_i2ce "); - } - // Must preserve original SP for loading incoming arguments because // we need to align the outgoing SP for compiled code. __ movptr(rdi, rsp); diff --git a/src/hotspot/share/cds/metaspaceShared.cpp b/src/hotspot/share/cds/metaspaceShared.cpp index f96af1bc8c1..cf938ee703c 100644 --- a/src/hotspot/share/cds/metaspaceShared.cpp +++ b/src/hotspot/share/cds/metaspaceShared.cpp @@ -449,7 +449,6 @@ void MetaspaceShared::serialize(SerializeClosure* soc) { soc->do_ptr((void**)&_archived_method_handle_intrinsics); LambdaFormInvokers::serialize(soc); - AdapterHandlerLibrary::serialize(soc); AdapterHandlerLibrary::serialize_shared_table_header(soc); soc->do_tag(666); @@ -559,7 +558,6 @@ char* VM_PopulateDumpSharedSpace::dump_read_only_tables() { // Write lambform lines into archive LambdaFormInvokers::dump_static_archive_invokers(); if (CDSConfig::is_dumping_adapters()) { - AdapterHandlerLibrary::dump_adapter_handler_list(); AdapterHandlerLibrary::archive_adapter_table(); } // Write module name into archive diff --git a/src/hotspot/share/classfile/classLoader.cpp b/src/hotspot/share/classfile/classLoader.cpp index 6233b731272..93b88faf6b5 100644 --- a/src/hotspot/share/classfile/classLoader.cpp +++ b/src/hotspot/share/classfile/classLoader.cpp @@ -130,7 +130,6 @@ PerfCounter* ClassLoader::_perf_preinit_time = nullptr; PerfCounter* ClassLoader::_perf_preresolve_time = nullptr; PerfCounter* ClassLoader::_perf_ik_link_methods_time = nullptr; PerfCounter* ClassLoader::_perf_method_adapters_time = nullptr; -PerfCounter* ClassLoader::_perf_load_adapter_table_time = nullptr; PerfCounter* ClassLoader::_perf_ik_link_methods_count = nullptr; PerfCounter* ClassLoader::_perf_method_adapters_count = nullptr; PerfCounter* ClassLoader::_unsafe_defineClassCallCounter = nullptr; @@ -157,8 +156,6 @@ void ClassLoader::print_counters(outputStream *st) { Management::ticks_to_us(_perf_ik_link_methods_time->get_value()) , _perf_ik_link_methods_count->get_value()); st->print_cr(" method adapters: " JLONG_FORMAT_W(6) "us / " JLONG_FORMAT " events", Management::ticks_to_us(_perf_method_adapters_time->get_value()) , _perf_method_adapters_count->get_value()); - st->print_cr(" load method adapters table: " JLONG_FORMAT_W(6) "us", - Management::ticks_to_us(_perf_load_adapter_table_time->get_value())); if (CountBytecodes || CountBytecodesPerThread) { st->print_cr("; executed " JLONG_FORMAT " bytecodes", ClassLoader::class_init_bytecodes_count()); } @@ -1499,7 +1496,6 @@ void ClassLoader::initialize(TRAPS) { NEWPERFTICKCOUNTER(_perf_ik_link_methods_time, SUN_CLS, "linkMethodsTime"); NEWPERFTICKCOUNTER(_perf_method_adapters_time, SUN_CLS, "makeAdaptersTime"); - NEWPERFTICKCOUNTER(_perf_load_adapter_table_time, SUN_CLS, "loadAdapterTableTime"); NEWPERFEVENTCOUNTER(_perf_ik_link_methods_count, SUN_CLS, "linkMethodsCount"); NEWPERFEVENTCOUNTER(_perf_method_adapters_count, SUN_CLS, "makeAdaptersCount"); diff --git a/src/hotspot/share/classfile/classLoader.hpp b/src/hotspot/share/classfile/classLoader.hpp index 666f7569173..0c44df8912a 100644 --- a/src/hotspot/share/classfile/classLoader.hpp +++ b/src/hotspot/share/classfile/classLoader.hpp @@ -177,7 +177,6 @@ class ClassLoader: AllStatic { static PerfCounter* _perf_preresolve_time; static PerfCounter* _perf_ik_link_methods_time; static PerfCounter* _perf_method_adapters_time; - static PerfCounter* _perf_load_adapter_table_time; static PerfCounter* _perf_ik_link_methods_count; static PerfCounter* _perf_method_adapters_count; @@ -316,7 +315,6 @@ class ClassLoader: AllStatic { static PerfCounter* perf_preresolve_time() { return _perf_preresolve_time; } static PerfCounter* perf_ik_link_methods_time() { return _perf_ik_link_methods_time; } static PerfCounter* perf_method_adapters_time() { return _perf_method_adapters_time; } - static PerfCounter* perf_load_adapter_table_time() { return _perf_load_adapter_table_time; } static PerfCounter* perf_ik_link_methods_count() { return _perf_ik_link_methods_count; } static PerfCounter* perf_method_adapters_count() { return _perf_method_adapters_count; } diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 32541111eff..42a4e360d4f 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -2611,7 +2611,6 @@ void AdapterHandlerLibrary::initialize() { AdapterBlob* obj_obj_arg_blob = nullptr; { _adapter_handler_table = new (mtCode) AdapterHandlerTable(); - //populate_adapter_handler_table(); MutexLocker mu(AdapterHandlerLibrary_lock); // Create a special handler for abstract methods. Abstract methods @@ -3400,48 +3399,6 @@ void AdapterHandlerLibrary::serialize_shared_table_header(SerializeClosure* soc) _archived_adapter_table.serialize_header(soc); } -void AdapterHandlerLibrary::dump_adapter_handler_list() { - GrowableArray buffered_handlers; - auto handler_collector = [&] (AdapterFingerPrint* fp, AdapterHandlerEntry* entry) { - LogStreamHandle(Trace, cds) lsh; - if (ArchiveBuilder::current()->has_been_archived((address)entry)) { - AdapterHandlerEntry* buffered = ArchiveBuilder::current()->get_buffered_addr(entry); - assert(buffered != nullptr,"sanity check"); - buffered_handlers.append(buffered); - if (lsh.is_enabled()) { - address runtime_addr = (address)buffered + ArchiveBuilder::current()->buffer_to_requested_delta(); - log_trace(cds)("Added adapter handler %p (buffered=%p, runtime=%p)", entry, buffered, runtime_addr); - } - } else { - if (lsh.is_enabled()) { - log_trace(cds)("Skipping adapter handler %p as it is not archived", entry); - } - } - }; - _adapter_handler_table->iterate_all(handler_collector); - _archived_adapter_handler_list = ArchiveUtils::archive_array(&buffered_handlers); - log_info(cds)("Dumped adapter handlers list (size=%d)", _adapter_handler_table->number_of_entries()); -} - -void AdapterHandlerLibrary::serialize(SerializeClosure* sc) { - sc->do_ptr((void**)&_archived_adapter_handler_list); -} - -void AdapterHandlerLibrary::populate_adapter_handler_table() { - PerfTraceElapsedTime timer(ClassLoader::perf_load_adapter_table_time()); - assert(_adapter_handler_table != nullptr, "_adapter_handler_table is not initialized"); - if (_archived_adapter_handler_list != nullptr) { - log_info(cds)("Found adapter handlers list (size=%d) in the AOT cache", _archived_adapter_handler_list->length()); - log_trace(cds)("Populating _adapter_handler_table"); - for (int i =0; i < _archived_adapter_handler_list->length(); i++) { - AdapterHandlerEntry* entry = _archived_adapter_handler_list->at(i); - assert(entry->fingerprint() != nullptr, "Fingerpring must not be null"); - _adapter_handler_table->put(entry->fingerprint(), entry); - log_trace(cds)("Added entry=%p, fp=%p", entry, entry->fingerprint()); - } - } -} - JRT_LEAF(void, SharedRuntime::enable_stack_reserved_zone(JavaThread* current)) assert(current == JavaThread::current(), "pre-condition"); StackOverflow* overflow_state = current->stack_overflow_state(); diff --git a/src/hotspot/share/runtime/sharedRuntime.hpp b/src/hotspot/share/runtime/sharedRuntime.hpp index adfe7c50a4b..aa05669d397 100644 --- a/src/hotspot/share/runtime/sharedRuntime.hpp +++ b/src/hotspot/share/runtime/sharedRuntime.hpp @@ -808,9 +808,6 @@ class AdapterHandlerLibrary: public AllStatic { static size_t estimate_size_for_archive(); static void archive_adapter_table(); static void serialize_shared_table_header(SerializeClosure* soc); - static void dump_adapter_handler_list(); - static void populate_adapter_handler_table(); - static void serialize(SerializeClosure* sc); static bool EQUALS(AdapterHandlerEntry* entry, AdapterFingerPrint* fp, int len_unused) { return entry->fingerprint() == fp; } From 0f7b7bab8dbf46b9b95e639ca7e6fa5e98cbc892 Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Mon, 21 Oct 2024 16:53:35 -0400 Subject: [PATCH 11/32] Misc changes Signed-off-by: Ashutosh Mehra --- src/hotspot/share/cds/cdsConfig.cpp | 5 +++++ src/hotspot/share/cds/cds_globals.hpp | 3 ++- src/hotspot/share/runtime/sharedRuntime.cpp | 25 ++++++++------------- src/hotspot/share/runtime/sharedRuntime.hpp | 9 ++++---- 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/src/hotspot/share/cds/cdsConfig.cpp b/src/hotspot/share/cds/cdsConfig.cpp index c296b4d29b1..9443f292653 100644 --- a/src/hotspot/share/cds/cdsConfig.cpp +++ b/src/hotspot/share/cds/cdsConfig.cpp @@ -32,6 +32,7 @@ #include "classfile/classLoaderDataShared.hpp" #include "classfile/moduleEntry.hpp" #include "classfile/systemDictionaryShared.hpp" +#include "code/SCCache.hpp" #include "include/jvm_io.h" #include "logging/log.hpp" #include "prims/jvmtiExport.hpp" @@ -542,6 +543,10 @@ bool CDSConfig::check_vm_args_consistency(bool patch_mod_javabase, bool mode_fla // Cannot dump cached code until metadata and heap are dumped. disable_dumping_cached_code(); } + if (StoreCachedCode) { + log_info(cds)("ArchiveAdapters is enabled"); + FLAG_SET_ERGO_IF_DEFAULT(ArchiveAdapters, true); + } } } else { // Old workflow diff --git a/src/hotspot/share/cds/cds_globals.hpp b/src/hotspot/share/cds/cds_globals.hpp index ced6b1a2612..921305799c6 100644 --- a/src/hotspot/share/cds/cds_globals.hpp +++ b/src/hotspot/share/cds/cds_globals.hpp @@ -170,7 +170,8 @@ "Skip verification of CDS archive heap") \ \ product(bool, ArchiveAdapters, false, \ - "Archive AdapterFingerPrint and AdapterHandlerEntry") \ + "Archive AdapterFingerPrint and AdapterHandlerEntry." \ + "Requires AOT code cache") \ // end of CDS_FLAGS diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 42a4e360d4f..cbff4d8feee 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -2528,7 +2528,7 @@ static GrowableArray* _adapter_handler_list = nullptr; AdapterHandlerEntry* AdapterHandlerLibrary::lookup(AdapterFingerPrint* fp) { NOT_PRODUCT(_lookups++); // Search archived table first. It is read-only table so can be searched without lock - AdapterHandlerEntry* entry = _archived_adapter_table.lookup(fp, fp->compute_hash(), 0 /* unused */); + AdapterHandlerEntry* entry = _archived_adapter_handler_table.lookup(fp, fp->compute_hash(), 0 /* unused */); if (entry != nullptr) { #ifndef PRODUCT if (fp->is_compact()) { @@ -2573,8 +2573,7 @@ AdapterHandlerEntry* AdapterHandlerLibrary::_int_arg_handler = nullptr; AdapterHandlerEntry* AdapterHandlerLibrary::_obj_arg_handler = nullptr; AdapterHandlerEntry* AdapterHandlerLibrary::_obj_int_arg_handler = nullptr; AdapterHandlerEntry* AdapterHandlerLibrary::_obj_obj_arg_handler = nullptr; -Array* AdapterHandlerLibrary::_archived_adapter_handler_list = nullptr; -ArchivedAdapterTable AdapterHandlerLibrary::_archived_adapter_table; +ArchivedAdapterTable AdapterHandlerLibrary::_archived_adapter_handler_table; const int AdapterHandlerLibrary_size = 16*K; BufferBlob* AdapterHandlerLibrary::_buffer = nullptr; @@ -2814,13 +2813,12 @@ AdapterHandlerEntry* AdapterHandlerLibrary::get_adapter(const methodHandle& meth return entry; } -bool AdapterHandlerLibrary::lookup_aot_cache(AdapterHandlerEntry* entry, MacroAssembler* masm) { +bool AdapterHandlerLibrary::lookup_aot_cache(AdapterHandlerEntry* entry, CodeBuffer* buffer) { const char* name = AdapterHandlerLibrary::name(entry->fingerprint()); const uint32_t id = AdapterHandlerLibrary::id(entry->fingerprint()); - CodeBuffer* buffer = masm->code(); uint32_t offsets[4]; if (SCCache::load_adapter(buffer, id, name, offsets)) { - address i2c_entry = masm->pc(); + address i2c_entry = buffer->insts_begin(); assert(offsets[0] == 0, "sanity check"); entry->set_entry_points(i2c_entry, i2c_entry + offsets[1], i2c_entry + offsets[2], i2c_entry + offsets[3]); return true; @@ -2844,9 +2842,9 @@ AdapterHandlerEntry* AdapterHandlerLibrary::create_adapter(AdapterBlob*& new_ada buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs, sizeof(buffer_locs)/sizeof(relocInfo)); - MacroAssembler masm(&buffer); AdapterHandlerEntry* entry = (cached_entry != nullptr) ? cached_entry : AdapterHandlerLibrary::new_entry(fingerprint); - if (!lookup_aot_cache(entry, &masm)) { + if (!lookup_aot_cache(entry, &buffer)) { + MacroAssembler masm(&buffer); VMRegPair stack_regs[16]; VMRegPair* regs = (total_args_passed <= 16) ? stack_regs : NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed); @@ -2949,11 +2947,7 @@ void AdapterHandlerEntry::metaspace_pointers_do(MetaspaceClosure* it) { } void AdapterHandlerEntry::remove_unshareable_info() { - _i2c_entry = nullptr; - _c2i_entry = nullptr; - _c2i_unverified_entry = nullptr; - _c2i_no_clinit_check_entry = nullptr; - _linked = false; + set_entry_points(nullptr, nullptr, nullptr, nullptr, false); } void AdapterHandlerEntry::restore_unshareable_info(TRAPS) { @@ -2967,7 +2961,6 @@ void AdapterHandlerEntry::restore_unshareable_info(TRAPS) { assert(_fingerprint != nullptr, "_fingerprint must not be null"); #ifdef ASSERT AdapterHandlerEntry* entry = AdapterHandlerLibrary::lookup(_fingerprint); - //AdapterHandlerEntry** entry = _adapter_handler_table->get(_fingerprint); assert(entry == this, "sanity check"); #endif ResourceMark rm; @@ -3392,11 +3385,11 @@ void AdapterHandlerLibrary::archive_adapter_table() { CompactHashtableWriter writer(_adapter_handler_table->number_of_entries(), &stats); CopyAdapterTableToArchive copy(&writer); _adapter_handler_table->iterate(©); - writer.dump(&_archived_adapter_table, "archived adapter table"); + writer.dump(&_archived_adapter_handler_table, "archived adapter table"); } void AdapterHandlerLibrary::serialize_shared_table_header(SerializeClosure* soc) { - _archived_adapter_table.serialize_header(soc); + _archived_adapter_handler_table.serialize_header(soc); } JRT_LEAF(void, SharedRuntime::enable_stack_reserved_zone(JavaThread* current)) diff --git a/src/hotspot/share/runtime/sharedRuntime.hpp b/src/hotspot/share/runtime/sharedRuntime.hpp index aa05669d397..6edb4afe9c7 100644 --- a/src/hotspot/share/runtime/sharedRuntime.hpp +++ b/src/hotspot/share/runtime/sharedRuntime.hpp @@ -720,12 +720,12 @@ class AdapterHandlerEntry : public MetaspaceObj { { return new (mtCode) AdapterHandlerEntry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); } - void set_entry_points(address i2c_entry, address c2i_entry, address c2i_unverified_entry, address c2i_no_clinit_check_entry) { + void set_entry_points(address i2c_entry, address c2i_entry, address c2i_unverified_entry, address c2i_no_clinit_check_entry, bool linked = true) { _i2c_entry = i2c_entry; _c2i_entry = c2i_entry; _c2i_unverified_entry = c2i_unverified_entry; _c2i_no_clinit_check_entry = c2i_no_clinit_check_entry; - _linked = true; + _linked = linked; } address get_i2c_entry() const { return _i2c_entry; } @@ -768,8 +768,7 @@ class AdapterHandlerLibrary: public AllStatic { static AdapterHandlerEntry* _obj_arg_handler; static AdapterHandlerEntry* _obj_int_arg_handler; static AdapterHandlerEntry* _obj_obj_arg_handler; - static Array* _archived_adapter_handler_list; - static ArchivedAdapterTable _archived_adapter_table; + static ArchivedAdapterTable _archived_adapter_handler_table; static BufferBlob* buffer_blob(); static void initialize(); @@ -777,7 +776,7 @@ class AdapterHandlerLibrary: public AllStatic { int total_args_passed, BasicType* sig_bt); static AdapterHandlerEntry* get_simple_adapter(const methodHandle& method); - static bool lookup_aot_cache(AdapterHandlerEntry* entry, MacroAssembler* masm); + static bool lookup_aot_cache(AdapterHandlerEntry* entry, CodeBuffer* buffer); public: static AdapterHandlerEntry* new_entry(AdapterFingerPrint* fingerprint, From ccae838c0bc84852cba2b324579487d47008797e Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Mon, 21 Oct 2024 16:54:46 -0400 Subject: [PATCH 12/32] Improve hashing algorithm for AdapterFingerPrint Signed-off-by: Ashutosh Mehra --- src/hotspot/share/runtime/sharedRuntime.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index cbff4d8feee..075e8179305 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -2379,7 +2379,8 @@ class AdapterFingerPrint : public MetaspaceObj { int hash = 0; for (int i = 0; i < length(); i++) { int v = value(i); - hash = (hash << 8) ^ v ^ (hash >> 5); + //Add arithmetic operation to the hash, like +3 to improve hashing + hash = ((hash << 8) ^ v ^ (hash >> 5)) + 3; } return (unsigned int)hash; } From 7db3ea0eca63d7d67618b4c56b5b327f4603cb21 Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Mon, 21 Oct 2024 17:48:12 -0400 Subject: [PATCH 13/32] Add a helper method to iterate over AdapterFingerPrint Signed-off-by: Ashutosh Mehra --- src/hotspot/share/runtime/sharedRuntime.cpp | 111 ++++++++++---------- 1 file changed, 55 insertions(+), 56 deletions(-) diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 075e8179305..aab293262b0 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -2347,6 +2347,22 @@ class AdapterFingerPrint : public MetaspaceObj { return p; } + template + void iterate_args(Function function) { + for (int i = 0; i < length(); i++) { + unsigned val = (unsigned)value(i); + // args are packed so that first/lower arguments are in the highest + // bits of each int value, so iterate from highest to the lowest + for (int j = 32 - _basic_type_bits; j >= 0; j -= _basic_type_bits) { + unsigned v = (val >> j) & _basic_type_mask; + if (v == 0) { + continue; + } + function(v); + } + } + } + public: static int allocation_size(int total_args_passed, BasicType* sig_bt) { int len = (total_args_passed + (_basic_types_per_int-1)) / _basic_types_per_int; @@ -2394,39 +2410,27 @@ class AdapterFingerPrint : public MetaspaceObj { return st.as_string(); } - // Reconstitutes the basic type arguments from the fingerprint, - // producing strings like LIJDF const char* as_basic_args_string() { stringStream st; bool long_prev = false; - for (int i = 0; i < length(); i++) { - unsigned val = (unsigned)value(i); - // args are packed so that first/lower arguments are in the highest - // bits of each int value, so iterate from highest to the lowest - for (int j = 32 - _basic_type_bits; j >= 0; j -= _basic_type_bits) { - unsigned v = (val >> j) & _basic_type_mask; - if (v == 0) { - assert(i == length() - 1, "Only expect zeroes in the last word"); - continue; - } - if (long_prev) { - long_prev = false; - if (v == T_VOID) { - st.print("J"); - } else { - st.print("L"); - } - } - switch (v) { - case T_INT: st.print("I"); break; - case T_LONG: long_prev = true; break; - case T_FLOAT: st.print("F"); break; - case T_DOUBLE: st.print("D"); break; - case T_VOID: break; - default: ShouldNotReachHere(); - } + iterate_args([&] (int arg) { + if (long_prev) { + long_prev = false; + if (arg == T_VOID) { + st.print("J"); + } else { + st.print("L"); + } } - } + switch (arg) { + case T_INT: st.print("I"); break; + case T_LONG: long_prev = true; break; + case T_FLOAT: st.print("F"); break; + case T_DOUBLE: st.print("D"); break; + case T_VOID: break; + default: ShouldNotReachHere(); + } + }); if (long_prev) { st.print("L"); } @@ -2437,35 +2441,30 @@ class AdapterFingerPrint : public MetaspaceObj { nargs = 0; GrowableArray btarray; bool long_prev = false; - for (int i = 0; i < length(); i++) { - unsigned val = (unsigned)value(i); - // args are packed so that first/lower arguments are in the highest - // bits of each int value, so iterate from highest to the lowest - for (int j = 32 - _basic_type_bits; j >= 0; j -= _basic_type_bits) { - unsigned v = (val >> j) & _basic_type_mask; - if (v == 0) continue; - if (long_prev) { - long_prev = false; - if (v == T_VOID) { - btarray.append(T_LONG); - } else { - btarray.append(T_OBJECT); // it could be T_ARRAY; it shouldn't matter - } + + iterate_args([&] (int arg) { + if (long_prev) { + long_prev = false; + if (arg == T_VOID) { + btarray.append(T_LONG); + } else { + btarray.append(T_OBJECT); // it could be T_ARRAY; it shouldn't matter } - switch (v) { - case T_INT: // fallthrough - case T_FLOAT: // fallthrough - case T_DOUBLE: - case T_VOID: - btarray.append((BasicType)v); - break; - case T_LONG: - long_prev = true; - break; - default: ShouldNotReachHere(); - } } - } + switch (arg) { + case T_INT: // fallthrough + case T_FLOAT: // fallthrough + case T_DOUBLE: + case T_VOID: + btarray.append((BasicType)arg); + break; + case T_LONG: + long_prev = true; + break; + default: ShouldNotReachHere(); + } + }); + if (long_prev) { btarray.append(T_OBJECT); } From 16bd0b57aaa4c62f3afe3e5863820acec9c0ed55 Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Tue, 22 Oct 2024 18:13:52 -0400 Subject: [PATCH 14/32] Refactor code to avoid calling AdapterFingerPrint::as_basic_type() when loading from AOT cache Signed-off-by: Ashutosh Mehra --- src/hotspot/share/runtime/globals.hpp | 3 + src/hotspot/share/runtime/sharedRuntime.cpp | 208 +++++++++++++------- src/hotspot/share/runtime/sharedRuntime.hpp | 22 ++- 3 files changed, 153 insertions(+), 80 deletions(-) diff --git a/src/hotspot/share/runtime/globals.hpp b/src/hotspot/share/runtime/globals.hpp index 119c9984955..fb40ec24d79 100644 --- a/src/hotspot/share/runtime/globals.hpp +++ b/src/hotspot/share/runtime/globals.hpp @@ -2021,6 +2021,9 @@ const int ObjectAlignmentInBytes = 8; \ product(bool, StressSecondarySupers, false, DIAGNOSTIC, \ "Use a terrible hash function in order to generate many collisions.") \ + \ + develop(bool, TestAdapterLinkFailure, false, \ + "Test failure of adapter linking when loading from AOT cache.") \ // end of RUNTIME_FLAGS diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index aab293262b0..2140ebf5dd1 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -2652,16 +2652,23 @@ void AdapterHandlerLibrary::initialize() { post_adapter_creation(obj_obj_arg_blob, _obj_obj_arg_handler); } -AdapterHandlerEntry* AdapterHandlerLibrary::create_simple_adapter(AdapterBlob*& new_adapter, +AdapterHandlerEntry* AdapterHandlerLibrary::create_simple_adapter(AdapterBlob*& adapter_blob, int total_args_passed, BasicType* sig_bt) { AdapterFingerPrint* fp = AdapterFingerPrint::allocate(total_args_passed, sig_bt); // We may find the adapter in the table if it is loaded from the AOT cache AdapterHandlerEntry* entry = lookup(fp); - assert(entry == nullptr || (entry->is_shared() && !entry->is_linked()), "Non null AdapterHandlerEntry should be in the AOT cache in unlinked state"); - entry = create_adapter(new_adapter, fp, total_args_passed, sig_bt, true, entry); - if (entry->is_shared()) { + if (entry != nullptr) { + assert(entry->is_shared() && !entry->is_linked(), "Non null AdapterHandlerEntry should be in the AOT cache in unlinked state"); + if (!link_adapter_handler(entry, adapter_blob)) { + if (generate_adapter_code(adapter_blob, entry, total_args_passed, sig_bt, true)) { + return nullptr; + } + } + // AdapterFingerPrint is already in the cache, no need to keep this one AdapterFingerPrint::deallocate(fp); + } else { + entry = create_adapter(adapter_blob, fp, total_args_passed, sig_bt, true); } return entry; } @@ -2813,25 +2820,45 @@ AdapterHandlerEntry* AdapterHandlerLibrary::get_adapter(const methodHandle& meth return entry; } -bool AdapterHandlerLibrary::lookup_aot_cache(AdapterHandlerEntry* entry, CodeBuffer* buffer) { - const char* name = AdapterHandlerLibrary::name(entry->fingerprint()); - const uint32_t id = AdapterHandlerLibrary::id(entry->fingerprint()); +bool AdapterHandlerLibrary::lookup_aot_cache(AdapterHandlerEntry* handler, CodeBuffer* buffer) { + ResourceMark rm; + const char* name = AdapterHandlerLibrary::name(handler->fingerprint()); + const uint32_t id = AdapterHandlerLibrary::id(handler->fingerprint()); uint32_t offsets[4]; if (SCCache::load_adapter(buffer, id, name, offsets)) { address i2c_entry = buffer->insts_begin(); assert(offsets[0] == 0, "sanity check"); - entry->set_entry_points(i2c_entry, i2c_entry + offsets[1], i2c_entry + offsets[2], i2c_entry + offsets[3]); + handler->set_entry_points(i2c_entry, i2c_entry + offsets[1], i2c_entry + offsets[2], i2c_entry + offsets[3]); return true; } return false; } -AdapterHandlerEntry* AdapterHandlerLibrary::create_adapter(AdapterBlob*& new_adapter, - AdapterFingerPrint* fingerprint, - int total_args_passed, - BasicType* sig_bt, - bool allocate_code_blob, - AdapterHandlerEntry* cached_entry) { +#ifndef PRODUCT +void AdapterHandlerLibrary::print_adapter_handler_info(AdapterHandlerEntry* handler, AdapterBlob* adapter_blob) { + ttyLocker ttyl; + ResourceMark rm; + int insts_size = adapter_blob->code_size(); + handler->print_adapter_on(tty); + tty->print_cr("i2c argument handler for: %s %s (%d bytes generated)", + handler->fingerprint()->as_basic_args_string(), + handler->fingerprint()->as_string(), insts_size); + tty->print_cr("c2i argument handler starts at " INTPTR_FORMAT, p2i(handler->get_c2i_entry())); + if (Verbose || PrintStubCode) { + address first_pc = handler->base_address(); + if (first_pc != nullptr) { + Disassembler::decode(first_pc, first_pc + insts_size, tty, &adapter_blob->asm_remarks()); + tty->cr(); + } + } +} +#endif // PRODUCT + +bool AdapterHandlerLibrary::generate_adapter_code(AdapterBlob*& adapter_blob, + AdapterHandlerEntry* handler, + int total_args_passed, + BasicType* sig_bt, + bool allocate_code_blob) { if (log_is_enabled(Info, perf, class, link)) { ClassLoader::perf_method_adapters_count()->inc(); } @@ -2840,77 +2867,100 @@ AdapterHandlerEntry* AdapterHandlerLibrary::create_adapter(AdapterBlob*& new_ada CodeBuffer buffer(buf); short buffer_locs[20]; buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs, - sizeof(buffer_locs)/sizeof(relocInfo)); - - AdapterHandlerEntry* entry = (cached_entry != nullptr) ? cached_entry : AdapterHandlerLibrary::new_entry(fingerprint); - if (!lookup_aot_cache(entry, &buffer)) { - MacroAssembler masm(&buffer); - VMRegPair stack_regs[16]; - VMRegPair* regs = (total_args_passed <= 16) ? stack_regs : NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed); - - // Get a description of the compiled java calling convention and the largest used (VMReg) stack slot usage - int comp_args_on_stack = SharedRuntime::java_calling_convention(sig_bt, regs, total_args_passed); - SharedRuntime::generate_i2c2i_adapters(&masm, - total_args_passed, - comp_args_on_stack, - sig_bt, - regs, - entry); - if (CDSConfig::is_dumping_adapters()) { - // try to save generated code - const char* name = AdapterHandlerLibrary::name(entry->fingerprint()); - const uint32_t id = AdapterHandlerLibrary::id(entry->fingerprint()); - uint32_t offsets[4]; - offsets[0] = 0; - offsets[1] = entry->get_c2i_entry() - entry->get_i2c_entry(); - offsets[2] = entry->get_c2i_unverified_entry() - entry->get_i2c_entry(); - offsets[3] = entry->get_c2i_no_clinit_check_entry() - entry->get_i2c_entry(); - SCCache::store_adapter(&buffer, id, name, offsets); - } + sizeof(buffer_locs)/sizeof(relocInfo)); + MacroAssembler masm(&buffer); + VMRegPair stack_regs[16]; + VMRegPair* regs = (total_args_passed <= 16) ? stack_regs : NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed); + + // Get a description of the compiled java calling convention and the largest used (VMReg) stack slot usage + int comp_args_on_stack = SharedRuntime::java_calling_convention(sig_bt, regs, total_args_passed); + SharedRuntime::generate_i2c2i_adapters(&masm, + total_args_passed, + comp_args_on_stack, + sig_bt, + regs, + handler); + if (CDSConfig::is_dumping_adapters()) { + // try to save generated code + const char* name = AdapterHandlerLibrary::name(handler->fingerprint()); + const uint32_t id = AdapterHandlerLibrary::id(handler->fingerprint()); + uint32_t offsets[4]; + offsets[0] = 0; + offsets[1] = handler->get_c2i_entry() - handler->get_i2c_entry(); + offsets[2] = handler->get_c2i_unverified_entry() - handler->get_i2c_entry(); + offsets[3] = handler->get_c2i_no_clinit_check_entry() - handler->get_i2c_entry(); + SCCache::store_adapter(&buffer, id, name, offsets); } #ifdef ASSERT if (VerifyAdapterSharing) { - entry->save_code(buf->code_begin(), buffer.insts_size()); + handler->save_code(buf->code_begin(), buffer.insts_size()); if (!allocate_code_blob) { - return entry; + return true; } } #endif - new_adapter = AdapterBlob::create(&buffer); - NOT_PRODUCT(int insts_size = buffer.insts_size()); - if (new_adapter == nullptr) { + adapter_blob = AdapterBlob::create(&buffer); + if (adapter_blob == nullptr) { // CodeCache is full, disable compilation // Ought to log this but compile log is only per compile thread // and we're some non descript Java thread. - return nullptr; + return true; } - entry->relocate(new_adapter->content_begin()); + handler->relocate(adapter_blob->content_begin()); #ifndef PRODUCT // debugging support if (PrintAdapterHandlers || PrintStubCode) { - ttyLocker ttyl; - entry->print_adapter_on(tty); - tty->print_cr("i2c argument handler #%d for: %s %s (%d bytes generated)", - _adapter_handler_table->number_of_entries(), fingerprint->as_basic_args_string(), - fingerprint->as_string(), insts_size); - tty->print_cr("c2i argument handler starts at " INTPTR_FORMAT, p2i(entry->get_c2i_entry())); - if (Verbose || PrintStubCode) { - address first_pc = entry->base_address(); - if (first_pc != nullptr) { - Disassembler::decode(first_pc, first_pc + insts_size, tty - NOT_PRODUCT(COMMA &new_adapter->asm_remarks())); - tty->cr(); - } - } + print_adapter_handler_info(handler, adapter_blob); } #endif + return true; +} - if (cached_entry == nullptr) { - assert_lock_strong(AdapterHandlerLibrary_lock); - _adapter_handler_table->put(fingerprint, entry); +AdapterHandlerEntry* AdapterHandlerLibrary::create_adapter(AdapterBlob*& adapter_blob, + AdapterFingerPrint* fingerprint, + int total_args_passed, + BasicType* sig_bt, + bool allocate_code_blob) { + AdapterHandlerEntry* handler = AdapterHandlerLibrary::new_entry(fingerprint); + if (!generate_adapter_code(adapter_blob, handler, total_args_passed, sig_bt, allocate_code_blob)) { + return nullptr; } - return entry; + assert_lock_strong(AdapterHandlerLibrary_lock); + _adapter_handler_table->put(fingerprint, handler); + return handler; +} + +bool AdapterHandlerLibrary::link_adapter_handler(AdapterHandlerEntry* handler, AdapterBlob*& adapter_blob) { +#ifndef PRODUCT + if (TestAdapterLinkFailure) { + return false; + } +#endif + BufferBlob* buf = buffer_blob(); // the temporary code buffer in CodeCache + CodeBuffer buffer(buf); + short buffer_locs[20]; + buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs, + sizeof(buffer_locs)/sizeof(relocInfo)); + + if (!lookup_aot_cache(handler, &buffer)) { + return false; + } + adapter_blob = AdapterBlob::create(&buffer); + if (adapter_blob == nullptr) { + // CodeCache is full, disable compilation + // Ought to log this but compile log is only per compile thread + // and we're some non descript Java thread. + return false; + } + handler->relocate(adapter_blob->content_begin()); +#ifndef PRODUCT + // debugging support + if (PrintAdapterHandlers || PrintStubCode) { + print_adapter_handler_info(handler, adapter_blob); + } +#endif + return true; } address AdapterHandlerEntry::base_address() { @@ -2955,7 +3005,7 @@ void AdapterHandlerEntry::restore_unshareable_info(TRAPS) { if (is_linked()) { return; } - AdapterBlob* blob = nullptr; + AdapterBlob* adapter_blob = nullptr; { MutexLocker mu(AdapterHandlerLibrary_lock); assert(_fingerprint != nullptr, "_fingerprint must not be null"); @@ -2963,14 +3013,26 @@ void AdapterHandlerEntry::restore_unshareable_info(TRAPS) { AdapterHandlerEntry* entry = AdapterHandlerLibrary::lookup(_fingerprint); assert(entry == this, "sanity check"); #endif - ResourceMark rm; - int nargs; - BasicType* bt = _fingerprint->as_basic_type(nargs); - AdapterHandlerLibrary::create_adapter(blob, _fingerprint, nargs, bt, true, this); + if (!AdapterHandlerLibrary::link_adapter_handler(this, adapter_blob)) { + ResourceMark rm; + log_warning(cds)("Failed to link AdapterHandlerEntry to its code in the AOT code cache"); + int nargs; + BasicType* bt = _fingerprint->as_basic_type(nargs); + if (!AdapterHandlerLibrary::generate_adapter_code(adapter_blob, this, nargs, bt, true)) { + if (!is_init_completed()) { + // Don't throw exceptions during VM initialization because java.lang.* classes + // might not have been initialized, causing problems when constructing the + // Java exception object. + vm_exit_during_initialization("Out of space in CodeCache for adapters"); + } else { + THROW_MSG(vmSymbols::java_lang_OutOfMemoryError(), "Out of space in CodeCache for adapters"); + } + } + } } // Outside of the lock - if (blob != nullptr) { - post_adapter_creation(blob, this); + if (adapter_blob != nullptr) { + post_adapter_creation(adapter_blob, this); } assert(_linked, "AdapterHandlerEntry must now be linked"); } diff --git a/src/hotspot/share/runtime/sharedRuntime.hpp b/src/hotspot/share/runtime/sharedRuntime.hpp index 6edb4afe9c7..3d90d557cc2 100644 --- a/src/hotspot/share/runtime/sharedRuntime.hpp +++ b/src/hotspot/share/runtime/sharedRuntime.hpp @@ -776,7 +776,15 @@ class AdapterHandlerLibrary: public AllStatic { int total_args_passed, BasicType* sig_bt); static AdapterHandlerEntry* get_simple_adapter(const methodHandle& method); - static bool lookup_aot_cache(AdapterHandlerEntry* entry, CodeBuffer* buffer); + static bool lookup_aot_cache(AdapterHandlerEntry* handler, CodeBuffer* buffer); + static AdapterHandlerEntry* create_adapter(AdapterBlob*& new_adapter, + AdapterFingerPrint* fingerprint, + int total_args_passed, + BasicType* sig_bt, + bool allocate_code_blob); +#ifndef PRODUCT + static void print_adapter_handler_info(AdapterHandlerEntry* handler, AdapterBlob* adapter_blob); +#endif // PRODUCT public: static AdapterHandlerEntry* new_entry(AdapterFingerPrint* fingerprint, @@ -786,13 +794,13 @@ class AdapterHandlerLibrary: public AllStatic { address c2i_no_clinit_check_entry = nullptr); static void create_native_wrapper(const methodHandle& method); static AdapterHandlerEntry* get_adapter(const methodHandle& method); - static AdapterHandlerEntry* create_adapter(AdapterBlob*& new_adapter, - AdapterFingerPrint* fingerprint, - int total_args_passed, - BasicType* sig_bt, - bool allocate_code_blob, - AdapterHandlerEntry* cached_entry = nullptr); static AdapterHandlerEntry* lookup(AdapterFingerPrint* fp); + static bool link_adapter_handler(AdapterHandlerEntry* handler, AdapterBlob*& adapter_blob); + static bool generate_adapter_code(AdapterBlob*& adapter_blob, + AdapterHandlerEntry* handler, + int total_args_passed, + BasicType* sig_bt, + bool allocate_code_blob); static void print_handler(const CodeBlob* b) { print_handler_on(tty, b); } static void print_handler_on(outputStream* st, const CodeBlob* b); From 3393d68e96a6d5defff29db17172e328be41d4be Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Wed, 23 Oct 2024 09:52:54 -0400 Subject: [PATCH 15/32] Fix a bug and add comment in AdapterHandlerEntry::restore_unshareable_info() Signed-off-by: Ashutosh Mehra --- src/hotspot/share/runtime/sharedRuntime.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 2140ebf5dd1..6e64b60431f 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -2905,7 +2905,7 @@ bool AdapterHandlerLibrary::generate_adapter_code(AdapterBlob*& adapter_blob, // CodeCache is full, disable compilation // Ought to log this but compile log is only per compile thread // and we're some non descript Java thread. - return true; + return false; } handler->relocate(adapter_blob->content_begin()); #ifndef PRODUCT @@ -3002,6 +3002,9 @@ void AdapterHandlerEntry::remove_unshareable_info() { void AdapterHandlerEntry::restore_unshareable_info(TRAPS) { PerfTraceElapsedTime timer(ClassLoader::perf_method_adapters_time()); + // A fixed set of simple adapters are eagerly linked during JVM initialization + // in AdapterHandlerTable::initialize(). + // Others may already have been linked because they are shared by other methods. if (is_linked()) { return; } From 1af4a1e3b655b1d3e792acc4dba590f5b67dfa4a Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Wed, 23 Oct 2024 12:59:20 -0400 Subject: [PATCH 16/32] Fix compile failures Signed-off-by: Ashutosh Mehra --- .../cpu/aarch64/sharedRuntime_aarch64.cpp | 4 ++-- src/hotspot/cpu/arm/sharedRuntime_arm.cpp | 15 ++++++------ src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp | 16 ++++++------- src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp | 16 +++++++------ src/hotspot/cpu/s390/sharedRuntime_s390.cpp | 15 ++++++------ src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp | 15 ++++++------ src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp | 4 ++-- src/hotspot/cpu/zero/sharedRuntime_zero.cpp | 23 +++++++++---------- src/hotspot/share/runtime/sharedRuntime.cpp | 2 +- src/hotspot/share/runtime/sharedRuntime.hpp | 4 ++-- 10 files changed, 59 insertions(+), 55 deletions(-) diff --git a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp index b42a5a991f1..9336617d0cc 100644 --- a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp @@ -683,7 +683,7 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, int comp_args_on_stack, const BasicType *sig_bt, const VMRegPair *regs, - AdapterHandlerEntry* entry) { + AdapterHandlerEntry* handler) { address i2c_entry = __ pc(); gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs); @@ -744,7 +744,7 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - entry->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); return; } diff --git a/src/hotspot/cpu/arm/sharedRuntime_arm.cpp b/src/hotspot/cpu/arm/sharedRuntime_arm.cpp index 7c1f3aafe7d..967e1660fe4 100644 --- a/src/hotspot/cpu/arm/sharedRuntime_arm.cpp +++ b/src/hotspot/cpu/arm/sharedRuntime_arm.cpp @@ -613,12 +613,12 @@ static void gen_c2i_adapter(MacroAssembler *masm, } -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint) { +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler) { address i2c_entry = __ pc(); gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs); @@ -638,7 +638,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm address c2i_entry = __ pc(); gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry); + handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, nullptr); + return; } diff --git a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp index aa8ae6070b6..455cf3a9f26 100644 --- a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp +++ b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp @@ -1144,12 +1144,12 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, __ bctr(); } -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint) { +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler) { address i2c_entry; address c2i_unverified_entry; address c2i_entry; @@ -1224,8 +1224,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, call_interpreter, ientry); - return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, - c2i_no_clinit_check_entry); + handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + return; } // An oop arg. Must pass a handle not the oop itself. diff --git a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp index 2b629fcfcb2..b7479eba411 100644 --- a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp +++ b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp @@ -597,12 +597,13 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, } // --------------------------------------------------------------- -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint) { + +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler) { address i2c_entry = __ pc(); gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs); @@ -660,7 +661,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + return; } int SharedRuntime::vector_calling_convention(VMRegPair *regs, diff --git a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp index 468610b588e..a570386e19f 100644 --- a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp +++ b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp @@ -2303,12 +2303,12 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, __ z_br(Z_R1_scratch); } -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint) { +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler) { __ align(CodeEntryAlignment); address i2c_entry = __ pc(); gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs); @@ -2362,7 +2362,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + return; } // This function returns the adjust size (in number of words) to a c2i adapter diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp index 0abb35fda65..96ad59b83a2 100644 --- a/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp +++ b/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp @@ -883,12 +883,12 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, } // --------------------------------------------------------------- -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint) { +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler) { address i2c_entry = __ pc(); gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs); @@ -926,7 +926,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry); + handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, nullptr); + return; } int SharedRuntime::c_calling_convention(const BasicType *sig_bt, diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp index aa8040c5608..c3bb782ab74 100644 --- a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp +++ b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp @@ -1007,7 +1007,7 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, int comp_args_on_stack, const BasicType *sig_bt, const VMRegPair *regs, - AdapterHandlerEntry* entry) { + AdapterHandlerEntry* handler) { address i2c_entry = __ pc(); gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs); @@ -1069,7 +1069,7 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - entry->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); return; } diff --git a/src/hotspot/cpu/zero/sharedRuntime_zero.cpp b/src/hotspot/cpu/zero/sharedRuntime_zero.cpp index 18ceb9514d3..2204b9e967f 100644 --- a/src/hotspot/cpu/zero/sharedRuntime_zero.cpp +++ b/src/hotspot/cpu/zero/sharedRuntime_zero.cpp @@ -51,18 +51,17 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt, return 0; } -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters( - MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint *fingerprint) { - return AdapterHandlerLibrary::new_entry( - fingerprint, - CAST_FROM_FN_PTR(address,zero_null_code_stub), - CAST_FROM_FN_PTR(address,zero_null_code_stub), - CAST_FROM_FN_PTR(address,zero_null_code_stub)); +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler) { + handler->set_entry_points(CAST_FROM_FN_PTR(address,zero_null_code_stub), + CAST_FROM_FN_PTR(address,zero_null_code_stub), + CAST_FROM_FN_PTR(address,zero_null_code_stub), + nullptr); + return; } nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm, diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 6e64b60431f..ffee6f3fc17 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -2503,7 +2503,7 @@ class AdapterFingerPrint : public MetaspaceObj { // methods required by virtue of being a MetaspaceObj void metaspace_pointers_do(MetaspaceClosure* it) { return; /* nothing to do here */ } - int size() const { return heap_word_size(sizeof(AdapterFingerPrint) + (_length > _compact_int_count ? (_length - _compact_int_count) * sizeof(int) : 0)); } + int size() const { return (int)heap_word_size(sizeof(AdapterFingerPrint) + (_length > _compact_int_count ? (_length - _compact_int_count) * sizeof(int) : 0)); } MetaspaceObj::Type type() const { return AdapterFingerPrintType; } static bool equals(AdapterFingerPrint* const& fp1, AdapterFingerPrint* const& fp2) { diff --git a/src/hotspot/share/runtime/sharedRuntime.hpp b/src/hotspot/share/runtime/sharedRuntime.hpp index 3d90d557cc2..7a52d2681cf 100644 --- a/src/hotspot/share/runtime/sharedRuntime.hpp +++ b/src/hotspot/share/runtime/sharedRuntime.hpp @@ -474,7 +474,7 @@ class SharedRuntime: AllStatic { int max_arg, const BasicType *sig_bt, const VMRegPair *regs, - AdapterHandlerEntry* entry); + AdapterHandlerEntry* handler); static void gen_i2c_adapter(MacroAssembler *_masm, int total_args_passed, @@ -749,7 +749,7 @@ class AdapterHandlerEntry : public MetaspaceObj { void print_adapter_on(outputStream* st) const; void metaspace_pointers_do(MetaspaceClosure* it); - int size() const { return heap_word_size(sizeof(AdapterHandlerEntry)); } + int size() const { return (int)heap_word_size(sizeof(AdapterHandlerEntry)); } MetaspaceObj::Type type() const { return AdapterHandlerEntryType; } void remove_unshareable_info(); From 268b945cfa3d0c9a8d0b7f3b70dcc216f51d8e5a Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Wed, 23 Oct 2024 14:36:54 -0400 Subject: [PATCH 17/32] Fix whitespace issues Signed-off-by: Ashutosh Mehra --- src/hotspot/share/cds/cdsConfig.cpp | 4 +- src/hotspot/share/code/SCCache.cpp | 42 ++++++++-------- src/hotspot/share/runtime/sharedRuntime.cpp | 56 ++++++++++----------- 3 files changed, 51 insertions(+), 51 deletions(-) diff --git a/src/hotspot/share/cds/cdsConfig.cpp b/src/hotspot/share/cds/cdsConfig.cpp index 9443f292653..329440e21db 100644 --- a/src/hotspot/share/cds/cdsConfig.cpp +++ b/src/hotspot/share/cds/cdsConfig.cpp @@ -544,8 +544,8 @@ bool CDSConfig::check_vm_args_consistency(bool patch_mod_javabase, bool mode_fla disable_dumping_cached_code(); } if (StoreCachedCode) { - log_info(cds)("ArchiveAdapters is enabled"); - FLAG_SET_ERGO_IF_DEFAULT(ArchiveAdapters, true); + log_info(cds)("ArchiveAdapters is enabled"); + FLAG_SET_ERGO_IF_DEFAULT(ArchiveAdapters, true); } } } else { diff --git a/src/hotspot/share/code/SCCache.cpp b/src/hotspot/share/code/SCCache.cpp index 924b5a01468..2f91f80c958 100644 --- a/src/hotspot/share/code/SCCache.cpp +++ b/src/hotspot/share/code/SCCache.cpp @@ -1164,20 +1164,20 @@ bool SCCache::finish_write() { } else if (_load_entries[i].for_preload() && _load_entries[i].method() != nullptr) { // record entrant first version code for pre-loading preload_entries[preload_entries_cnt++] = entries_count; - } else if (entries_address[i].kind() == SCCEntry::Adapter) { - adapters_count++; - } else if (entries_address[i].kind() == SCCEntry::Stub) { - stubs_count++; - } else if (entries_address[i].kind() == SCCEntry::Blob) { - total_blobs_count++; - if (entries_address[i].comp_level() == CompLevel_none) { - shared_blobs_count++; - } else if (entries_address[i].comp_level() == CompLevel_simple) { - c1_blobs_count++; - } else { - assert(entries_address[i].comp_level() == CompLevel_full_optimization, "must be!"); - opto_blobs_count++; - } + } else if (entries_address[i].kind() == SCCEntry::Adapter) { + adapters_count++; + } else if (entries_address[i].kind() == SCCEntry::Stub) { + stubs_count++; + } else if (entries_address[i].kind() == SCCEntry::Blob) { + total_blobs_count++; + if (entries_address[i].comp_level() == CompLevel_none) { + shared_blobs_count++; + } else if (entries_address[i].comp_level() == CompLevel_simple) { + c1_blobs_count++; + } else { + assert(entries_address[i].comp_level() == CompLevel_full_optimization, "must be!"); + opto_blobs_count++; + } } { uint size = align_up(_load_entries[i].size(), DATA_ALIGNMENT); @@ -4362,7 +4362,7 @@ address SCAddressTable::address_for_id(int idx) { if (id > (_c_str_base + _c_str_max)) { return (address)os::init + idx; } - if (idx < 0) { + if (idx < 0) { fatal("Incorrect id %d for SCA table", id); } // no need to compare unsigned id against 0 @@ -4421,14 +4421,14 @@ int SCAddressTable::id_for_address(address addr, RelocIterator reloc, CodeBuffer // Search in code blobs id = search_address(addr, _blobs_addr, _blobs_length); if (id == -1) { - id_base = _C1_blobs_base; - // search C1 blobs - id = search_address(addr, _C1_blobs_addr, _C1_blobs_length); + id_base = _C1_blobs_base; + // search C1 blobs + id = search_address(addr, _C1_blobs_addr, _C1_blobs_length); } if (id == -1) { - id_base = _C2_blobs_base; - // search C2 blobs - id = search_address(addr, _C2_blobs_addr, _C2_blobs_length); + id_base = _C2_blobs_base; + // search C2 blobs + id = search_address(addr, _C2_blobs_addr, _C2_blobs_length); } if (id < 0) { fatal("Address " INTPTR_FORMAT " for Blob:%s is missing in SCA table", p2i(addr), cb->name()); diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index ffee6f3fc17..23b700acdea 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -2415,20 +2415,20 @@ class AdapterFingerPrint : public MetaspaceObj { bool long_prev = false; iterate_args([&] (int arg) { if (long_prev) { - long_prev = false; - if (arg == T_VOID) { - st.print("J"); - } else { - st.print("L"); - } + long_prev = false; + if (arg == T_VOID) { + st.print("J"); + } else { + st.print("L"); + } } switch (arg) { - case T_INT: st.print("I"); break; - case T_LONG: long_prev = true; break; - case T_FLOAT: st.print("F"); break; - case T_DOUBLE: st.print("D"); break; - case T_VOID: break; - default: ShouldNotReachHere(); + case T_INT: st.print("I"); break; + case T_LONG: long_prev = true; break; + case T_FLOAT: st.print("F"); break; + case T_DOUBLE: st.print("D"); break; + case T_VOID: break; + default: ShouldNotReachHere(); } }); if (long_prev) { @@ -2444,24 +2444,24 @@ class AdapterFingerPrint : public MetaspaceObj { iterate_args([&] (int arg) { if (long_prev) { - long_prev = false; - if (arg == T_VOID) { - btarray.append(T_LONG); - } else { - btarray.append(T_OBJECT); // it could be T_ARRAY; it shouldn't matter - } + long_prev = false; + if (arg == T_VOID) { + btarray.append(T_LONG); + } else { + btarray.append(T_OBJECT); // it could be T_ARRAY; it shouldn't matter + } } switch (arg) { - case T_INT: // fallthrough - case T_FLOAT: // fallthrough - case T_DOUBLE: - case T_VOID: - btarray.append((BasicType)arg); - break; - case T_LONG: - long_prev = true; - break; - default: ShouldNotReachHere(); + case T_INT: // fallthrough + case T_FLOAT: // fallthrough + case T_DOUBLE: + case T_VOID: + btarray.append((BasicType)arg); + break; + case T_LONG: + long_prev = true; + break; + default: ShouldNotReachHere(); } }); From 98b0da703e36b97945ccc99b7b54521c0b845bb2 Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Wed, 23 Oct 2024 17:27:11 -0400 Subject: [PATCH 18/32] Surround code with INCLUDE_CDS where appropriate Signed-off-by: Ashutosh Mehra --- src/hotspot/share/oops/method.cpp | 2 + src/hotspot/share/runtime/sharedRuntime.cpp | 111 +++++++++++--------- src/hotspot/share/runtime/sharedRuntime.hpp | 26 +++-- 3 files changed, 77 insertions(+), 62 deletions(-) diff --git a/src/hotspot/share/oops/method.cpp b/src/hotspot/share/oops/method.cpp index b3d4156a94a..b1ce9ce74a3 100644 --- a/src/hotspot/share/oops/method.cpp +++ b/src/hotspot/share/oops/method.cpp @@ -399,11 +399,13 @@ void Method::metaspace_pointers_do(MetaspaceClosure* it) { } else { it->push(&_constMethod); } +#if INCLUDE_CDS if (CDSConfig::is_dumping_adapters()) { if (!AdapterHandlerLibrary::is_abstract_method_adapter(_adapter)) { it->push(&_adapter); } } +#endif it->push(&_method_data); it->push(&_method_counters); NOT_PRODUCT(it->push(&_name);) diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 23b700acdea..cc22dec2501 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -2527,6 +2527,7 @@ static GrowableArray* _adapter_handler_list = nullptr; // Find a entry with the same fingerprint if it exists AdapterHandlerEntry* AdapterHandlerLibrary::lookup(AdapterFingerPrint* fp) { NOT_PRODUCT(_lookups++); +#if INCLUDE_CDS // Search archived table first. It is read-only table so can be searched without lock AdapterHandlerEntry* entry = _archived_adapter_handler_table.lookup(fp, fp->compute_hash(), 0 /* unused */); if (entry != nullptr) { @@ -2538,6 +2539,7 @@ AdapterHandlerEntry* AdapterHandlerLibrary::lookup(AdapterFingerPrint* fp) { #endif return entry; } +#endif // INCLUDE_CDS assert_lock_strong(AdapterHandlerLibrary_lock); AdapterHandlerEntry** entry_p = _adapter_handler_table->get(fp); if (entry_p != nullptr) { @@ -2573,8 +2575,9 @@ AdapterHandlerEntry* AdapterHandlerLibrary::_int_arg_handler = nullptr; AdapterHandlerEntry* AdapterHandlerLibrary::_obj_arg_handler = nullptr; AdapterHandlerEntry* AdapterHandlerLibrary::_obj_int_arg_handler = nullptr; AdapterHandlerEntry* AdapterHandlerLibrary::_obj_obj_arg_handler = nullptr; +#if INCLUDE_CDS ArchivedAdapterTable AdapterHandlerLibrary::_archived_adapter_handler_table; - +#endif // INCLUDE_CDS const int AdapterHandlerLibrary_size = 16*K; BufferBlob* AdapterHandlerLibrary::_buffer = nullptr; @@ -2931,6 +2934,7 @@ AdapterHandlerEntry* AdapterHandlerLibrary::create_adapter(AdapterBlob*& adapter return handler; } +#if INCLUDE_CDS bool AdapterHandlerLibrary::link_adapter_handler(AdapterHandlerEntry* handler, AdapterBlob*& adapter_blob) { #ifndef PRODUCT if (TestAdapterLinkFailure) { @@ -2963,6 +2967,58 @@ bool AdapterHandlerLibrary::link_adapter_handler(AdapterHandlerEntry* handler, A return true; } +class CopyAdapterTableToArchive : StackObj { +private: + CompactHashtableWriter* _writer; + ArchiveBuilder* _builder; +public: + CopyAdapterTableToArchive(CompactHashtableWriter* writer) : _writer(writer), + _builder(ArchiveBuilder::current()) + {} + + bool do_entry(AdapterFingerPrint* fp, AdapterHandlerEntry* entry) { + LogStreamHandle(Trace, cds) lsh; + if (ArchiveBuilder::current()->has_been_archived((address)entry)) { + assert(ArchiveBuilder::current()->has_been_archived((address)fp), "must be"); + AdapterFingerPrint* buffered_fp = ArchiveBuilder::current()->get_buffered_addr(fp); + assert(buffered_fp != nullptr,"sanity check"); + AdapterHandlerEntry* buffered_entry = ArchiveBuilder::current()->get_buffered_addr(entry); + assert(buffered_entry != nullptr,"sanity check"); + + uint hash = fp->compute_hash(); + u4 delta = _builder->buffer_to_offset_u4((address)buffered_entry); + _writer->add(hash, delta); + if (lsh.is_enabled()) { + address fp_runtime_addr = (address)buffered_fp + ArchiveBuilder::current()->buffer_to_requested_delta(); + address entry_runtime_addr = (address)buffered_entry + ArchiveBuilder::current()->buffer_to_requested_delta(); + log_trace(cds)("Added fp=%p, entry=%p to the archived adater table", buffered_fp, buffered_entry); + } + } else { + if (lsh.is_enabled()) { + log_trace(cds)("Skipping adapter handler %p as it is not archived", entry); + } + } + return true; + } +}; + +size_t AdapterHandlerLibrary::estimate_size_for_archive() { + return CompactHashtableWriter::estimate_size(_adapter_handler_table->number_of_entries()); +} + +void AdapterHandlerLibrary::archive_adapter_table() { + CompactHashtableStats stats; + CompactHashtableWriter writer(_adapter_handler_table->number_of_entries(), &stats); + CopyAdapterTableToArchive copy(&writer); + _adapter_handler_table->iterate(©); + writer.dump(&_archived_adapter_handler_table, "archived adapter table"); +} + +void AdapterHandlerLibrary::serialize_shared_table_header(SerializeClosure* soc) { + _archived_adapter_handler_table.serialize_header(soc); +} +#endif // INCLUDE_CDS + address AdapterHandlerEntry::base_address() { address base = _i2c_entry; if (base == nullptr) base = _c2i_entry; @@ -2996,6 +3052,7 @@ void AdapterHandlerEntry::metaspace_pointers_do(MetaspaceClosure* it) { it->push(&_fingerprint); } +#if INCLUDE_CDS void AdapterHandlerEntry::remove_unshareable_info() { set_entry_points(nullptr, nullptr, nullptr, nullptr, false); } @@ -3039,6 +3096,7 @@ void AdapterHandlerEntry::restore_unshareable_info(TRAPS) { } assert(_linked, "AdapterHandlerEntry must now be linked"); } +#endif // INCLUDE_CDS AdapterHandlerEntry::~AdapterHandlerEntry() { #ifdef ASSERT @@ -3406,57 +3464,6 @@ bool AdapterHandlerLibrary::is_abstract_method_adapter(AdapterHandlerEntry* entr return false; } -class CopyAdapterTableToArchive : StackObj { -private: - CompactHashtableWriter* _writer; - ArchiveBuilder* _builder; -public: - CopyAdapterTableToArchive(CompactHashtableWriter* writer) : _writer(writer), - _builder(ArchiveBuilder::current()) - {} - - bool do_entry(AdapterFingerPrint* fp, AdapterHandlerEntry* entry) { - LogStreamHandle(Trace, cds) lsh; - if (ArchiveBuilder::current()->has_been_archived((address)entry)) { - assert(ArchiveBuilder::current()->has_been_archived((address)fp), "must be"); - AdapterFingerPrint* buffered_fp = ArchiveBuilder::current()->get_buffered_addr(fp); - assert(buffered_fp != nullptr,"sanity check"); - AdapterHandlerEntry* buffered_entry = ArchiveBuilder::current()->get_buffered_addr(entry); - assert(buffered_entry != nullptr,"sanity check"); - - uint hash = fp->compute_hash(); - u4 delta = _builder->buffer_to_offset_u4((address)buffered_entry); - _writer->add(hash, delta); - if (lsh.is_enabled()) { - address fp_runtime_addr = (address)buffered_fp + ArchiveBuilder::current()->buffer_to_requested_delta(); - address entry_runtime_addr = (address)buffered_entry + ArchiveBuilder::current()->buffer_to_requested_delta(); - log_trace(cds)("Added fp=%p, entry=%p to the archived adater table", buffered_fp, buffered_entry); - } - } else { - if (lsh.is_enabled()) { - log_trace(cds)("Skipping adapter handler %p as it is not archived", entry); - } - } - return true; - } -}; - -size_t AdapterHandlerLibrary::estimate_size_for_archive() { - return CompactHashtableWriter::estimate_size(_adapter_handler_table->number_of_entries()); -} - -void AdapterHandlerLibrary::archive_adapter_table() { - CompactHashtableStats stats; - CompactHashtableWriter writer(_adapter_handler_table->number_of_entries(), &stats); - CopyAdapterTableToArchive copy(&writer); - _adapter_handler_table->iterate(©); - writer.dump(&_archived_adapter_handler_table, "archived adapter table"); -} - -void AdapterHandlerLibrary::serialize_shared_table_header(SerializeClosure* soc) { - _archived_adapter_handler_table.serialize_header(soc); -} - JRT_LEAF(void, SharedRuntime::enable_stack_reserved_zone(JavaThread* current)) assert(current == JavaThread::current(), "pre-condition"); StackOverflow* overflow_state = current->stack_overflow_state(); diff --git a/src/hotspot/share/runtime/sharedRuntime.hpp b/src/hotspot/share/runtime/sharedRuntime.hpp index 7a52d2681cf..9ca4f287c70 100644 --- a/src/hotspot/share/runtime/sharedRuntime.hpp +++ b/src/hotspot/share/runtime/sharedRuntime.hpp @@ -749,14 +749,16 @@ class AdapterHandlerEntry : public MetaspaceObj { void print_adapter_on(outputStream* st) const; void metaspace_pointers_do(MetaspaceClosure* it); - int size() const { return (int)heap_word_size(sizeof(AdapterHandlerEntry)); } + int size() const {return (int)heap_word_size(sizeof(AdapterHandlerEntry)); } MetaspaceObj::Type type() const { return AdapterHandlerEntryType; } - void remove_unshareable_info(); - void restore_unshareable_info(TRAPS); + void remove_unshareable_info() NOT_CDS_RETURN; + void restore_unshareable_info(TRAPS) NOT_CDS_RETURN; }; +#if INCLUDE_CDS class ArchivedAdapterTable; +#endif // INCLUDE_CDS class AdapterHandlerLibrary: public AllStatic { friend class SharedRuntime; @@ -768,8 +770,9 @@ class AdapterHandlerLibrary: public AllStatic { static AdapterHandlerEntry* _obj_arg_handler; static AdapterHandlerEntry* _obj_int_arg_handler; static AdapterHandlerEntry* _obj_obj_arg_handler; +#if INCLUDE_CDS static ArchivedAdapterTable _archived_adapter_handler_table; - +#endif // INCLUDE_CDS static BufferBlob* buffer_blob(); static void initialize(); static AdapterHandlerEntry* create_simple_adapter(AdapterBlob*& new_adapter, @@ -795,12 +798,11 @@ class AdapterHandlerLibrary: public AllStatic { static void create_native_wrapper(const methodHandle& method); static AdapterHandlerEntry* get_adapter(const methodHandle& method); static AdapterHandlerEntry* lookup(AdapterFingerPrint* fp); - static bool link_adapter_handler(AdapterHandlerEntry* handler, AdapterBlob*& adapter_blob); static bool generate_adapter_code(AdapterBlob*& adapter_blob, AdapterHandlerEntry* handler, int total_args_passed, - BasicType* sig_bt, - bool allocate_code_blob); + BasicType* sig_bt, + bool allocate_code_blob); static void print_handler(const CodeBlob* b) { print_handler_on(tty, b); } static void print_handler_on(outputStream* st, const CodeBlob* b); @@ -812,17 +814,21 @@ class AdapterHandlerLibrary: public AllStatic { #endif // PRODUCT static bool is_abstract_method_adapter(AdapterHandlerEntry* adapter); - static size_t estimate_size_for_archive(); - static void archive_adapter_table(); - static void serialize_shared_table_header(SerializeClosure* soc); + + static bool link_adapter_handler(AdapterHandlerEntry* handler, AdapterBlob*& adapter_blob) NOT_CDS_RETURN_(false); + static size_t estimate_size_for_archive() NOT_CDS_RETURN_(0); + static void archive_adapter_table() NOT_CDS_RETURN; + static void serialize_shared_table_header(SerializeClosure* soc) NOT_CDS_RETURN; static bool EQUALS(AdapterHandlerEntry* entry, AdapterFingerPrint* fp, int len_unused) { return entry->fingerprint() == fp; } }; +#if INCLUDE_CDS class ArchivedAdapterTable : public OffsetCompactHashtable< AdapterFingerPrint*, AdapterHandlerEntry*, AdapterHandlerLibrary::EQUALS> {}; +#endif // INCLUDE_CDS #endif // SHARE_RUNTIME_SHAREDRUNTIME_HPP From ea3d208708b6cf17ae34205b75abdfa989e52644 Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Wed, 23 Oct 2024 17:33:09 -0400 Subject: [PATCH 19/32] Move the check for is_abstract_method_adapter() to correct place Signed-off-by: Ashutosh Mehra --- src/hotspot/share/cds/archiveBuilder.cpp | 7 +++++++ src/hotspot/share/oops/method.cpp | 8 +------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/hotspot/share/cds/archiveBuilder.cpp b/src/hotspot/share/cds/archiveBuilder.cpp index 2394a01683b..1ec9ffe048f 100644 --- a/src/hotspot/share/cds/archiveBuilder.cpp +++ b/src/hotspot/share/cds/archiveBuilder.cpp @@ -615,6 +615,13 @@ ArchiveBuilder::FollowMode ArchiveBuilder::get_follow_mode(MetaspaceClosure::Ref ref->msotype() == MetaspaceObj::MethodTrainingDataType || ref->msotype() == MetaspaceObj::CompileTrainingDataType) { return TrainingData::need_data() ? make_a_copy : set_to_null; + } else if (ref->msotype() == MetaspaceObj::AdapterHandlerEntryType) { + if (CDSConfig::is_dumping_adapters()) { + AdapterHandlerEntry* entry = (AdapterHandlerEntry*)ref->obj(); + return AdapterHandlerLibrary::is_abstract_method_adapter(entry) ? set_to_null : make_a_copy; + } else { + return set_to_null; + } } else { if (ref->msotype() == MetaspaceObj::ClassType) { Klass* klass = (Klass*)ref->obj(); diff --git a/src/hotspot/share/oops/method.cpp b/src/hotspot/share/oops/method.cpp index b1ce9ce74a3..8054f10d8b3 100644 --- a/src/hotspot/share/oops/method.cpp +++ b/src/hotspot/share/oops/method.cpp @@ -399,13 +399,7 @@ void Method::metaspace_pointers_do(MetaspaceClosure* it) { } else { it->push(&_constMethod); } -#if INCLUDE_CDS - if (CDSConfig::is_dumping_adapters()) { - if (!AdapterHandlerLibrary::is_abstract_method_adapter(_adapter)) { - it->push(&_adapter); - } - } -#endif + it->push(&_adapter); it->push(&_method_data); it->push(&_method_counters); NOT_PRODUCT(it->push(&_name);) From 04fb256b45225f9f298695500b21efeb75132ee7 Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Wed, 23 Oct 2024 21:51:05 -0400 Subject: [PATCH 20/32] More whitespace fixes Signed-off-by: Ashutosh Mehra --- src/hotspot/share/runtime/sharedRuntime.cpp | 42 ++++++++++----------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index cc22dec2501..b9babd1e0e7 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -2292,9 +2292,9 @@ class AdapterFingerPrint : public MetaspaceObj { for (int index = 0; index < _length; index++) { int value = 0; for (int byte = 0; sig_index < total_args_passed && byte < _basic_types_per_int; byte++) { - int bt = adapter_encoding(sig_bt[sig_index++]); - assert((bt & _basic_type_mask) == bt, "must fit in 4 bits"); - value = (value << _basic_type_bits) | bt; + int bt = adapter_encoding(sig_bt[sig_index++]); + assert((bt & _basic_type_mask) == bt, "must fit in 4 bits"); + value = (value << _basic_type_bits) | bt; } _value[index] = value; } @@ -2844,8 +2844,8 @@ void AdapterHandlerLibrary::print_adapter_handler_info(AdapterHandlerEntry* hand int insts_size = adapter_blob->code_size(); handler->print_adapter_on(tty); tty->print_cr("i2c argument handler for: %s %s (%d bytes generated)", - handler->fingerprint()->as_basic_args_string(), - handler->fingerprint()->as_string(), insts_size); + handler->fingerprint()->as_basic_args_string(), + handler->fingerprint()->as_string(), insts_size); tty->print_cr("c2i argument handler starts at " INTPTR_FORMAT, p2i(handler->get_c2i_entry())); if (Verbose || PrintStubCode) { address first_pc = handler->base_address(); @@ -2878,11 +2878,11 @@ bool AdapterHandlerLibrary::generate_adapter_code(AdapterBlob*& adapter_blob, // Get a description of the compiled java calling convention and the largest used (VMReg) stack slot usage int comp_args_on_stack = SharedRuntime::java_calling_convention(sig_bt, regs, total_args_passed); SharedRuntime::generate_i2c2i_adapters(&masm, - total_args_passed, - comp_args_on_stack, - sig_bt, - regs, - handler); + total_args_passed, + comp_args_on_stack, + sig_bt, + regs, + handler); if (CDSConfig::is_dumping_adapters()) { // try to save generated code const char* name = AdapterHandlerLibrary::name(handler->fingerprint()); @@ -2989,9 +2989,9 @@ class CopyAdapterTableToArchive : StackObj { u4 delta = _builder->buffer_to_offset_u4((address)buffered_entry); _writer->add(hash, delta); if (lsh.is_enabled()) { - address fp_runtime_addr = (address)buffered_fp + ArchiveBuilder::current()->buffer_to_requested_delta(); - address entry_runtime_addr = (address)buffered_entry + ArchiveBuilder::current()->buffer_to_requested_delta(); - log_trace(cds)("Added fp=%p, entry=%p to the archived adater table", buffered_fp, buffered_entry); + address fp_runtime_addr = (address)buffered_fp + ArchiveBuilder::current()->buffer_to_requested_delta(); + address entry_runtime_addr = (address)buffered_entry + ArchiveBuilder::current()->buffer_to_requested_delta(); + log_trace(cds)("Added fp=%p, entry=%p to the archived adater table", buffered_fp, buffered_entry); } } else { if (lsh.is_enabled()) { @@ -3079,14 +3079,14 @@ void AdapterHandlerEntry::restore_unshareable_info(TRAPS) { int nargs; BasicType* bt = _fingerprint->as_basic_type(nargs); if (!AdapterHandlerLibrary::generate_adapter_code(adapter_blob, this, nargs, bt, true)) { - if (!is_init_completed()) { - // Don't throw exceptions during VM initialization because java.lang.* classes - // might not have been initialized, causing problems when constructing the - // Java exception object. - vm_exit_during_initialization("Out of space in CodeCache for adapters"); - } else { - THROW_MSG(vmSymbols::java_lang_OutOfMemoryError(), "Out of space in CodeCache for adapters"); - } + if (!is_init_completed()) { + // Don't throw exceptions during VM initialization because java.lang.* classes + // might not have been initialized, causing problems when constructing the + // Java exception object. + vm_exit_during_initialization("Out of space in CodeCache for adapters"); + } else { + THROW_MSG(vmSymbols::java_lang_OutOfMemoryError(), "Out of space in CodeCache for adapters"); + } } } } From b04ba15414b1800bd02972ac891dbedb00c0d0da Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Thu, 24 Oct 2024 11:55:10 -0400 Subject: [PATCH 21/32] Acquire Compile_lock when writing exception blobs to the SCCache Signed-off-by: Ashutosh Mehra --- src/hotspot/share/code/SCCache.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/code/SCCache.cpp b/src/hotspot/share/code/SCCache.cpp index 2f91f80c958..e9788a4863f 100644 --- a/src/hotspot/share/code/SCCache.cpp +++ b/src/hotspot/share/code/SCCache.cpp @@ -2367,10 +2367,8 @@ bool SCCache::store_adapter(CodeBuffer* buffer, uint32_t id, const char* name, u buffer->decode(); } #endif - // we need to take a lock to stop C1 and C2 compiler threads racing to + // we need to take a lock to stop main thread racing with C1 and C2 compiler threads to // write blobs in parallel with each other or with later nmethods - // TODO - maybe move this up to selected callers so we only lock - // when saving a c1 or opto blob MutexLocker ml(Compile_lock); if (!cache->align_write()) { return false; @@ -2483,6 +2481,11 @@ bool SCCache::store_exception_blob(CodeBuffer* buffer, int pc_offset) { buffer->decode(); } #endif + // we need to take a lock to stop C1 and C2 compiler threads racing to + // write blobs in parallel with each other or with later nmethods + // TODO - maybe move this up to selected callers so we only lock + // when saving a c1 or opto blob + MutexLocker ml(Compile_lock); if (!cache->align_write()) { return false; } From d99c24c7bccb802ef2010d32f4dc07d1bb86f22c Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Tue, 12 Nov 2024 13:32:27 -0500 Subject: [PATCH 22/32] Address reivew comments by vnkozlov Signed-off-by: Ashutosh Mehra --- src/hotspot/share/code/SCCache.cpp | 44 ++++++++++++++---------------- src/hotspot/share/code/SCCache.hpp | 4 +++ 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/hotspot/share/code/SCCache.cpp b/src/hotspot/share/code/SCCache.cpp index e9788a4863f..9340185b144 100644 --- a/src/hotspot/share/code/SCCache.cpp +++ b/src/hotspot/share/code/SCCache.cpp @@ -590,41 +590,41 @@ SCCache::SCCache(const char* cache_path, int fd, uint load_size) { } void SCCache::init_extrs_table() { - SCCache* cache = SCCache::cache(); - if (cache != nullptr && cache->_table != nullptr) { - cache->_table->init_extrs(); + SCAddressTable* table = addr_table(); + if (table != nullptr) { + table->init_extrs(); } } void SCCache::init_early_stubs_table() { - SCCache* cache = SCCache::cache(); - if (cache != nullptr && cache->_table != nullptr) { - cache->_table->init_early_stubs(); + SCAddressTable* table = addr_table(); + if (table != nullptr) { + table->init_early_stubs(); } } void SCCache::init_shared_blobs_table() { - SCCache* cache = SCCache::cache(); - if (cache != nullptr && cache->_table != nullptr) { - cache->_table->init_shared_blobs(); + SCAddressTable* table = addr_table(); + if (table != nullptr) { + table->init_shared_blobs(); } } void SCCache::init_stubs_table() { - SCCache* cache = SCCache::cache(); - if (cache != nullptr && cache->_table != nullptr) { - cache->_table->init_stubs(); + SCAddressTable* table = addr_table(); + if (table != nullptr) { + table->init_stubs(); } } void SCCache::init_opto_table() { - SCCache* cache = SCCache::cache(); - if (cache != nullptr && cache->_table != nullptr) { - cache->_table->init_opto(); + SCAddressTable* table = addr_table(); + if (table != nullptr) { + table->init_opto(); } } void SCCache::init_c1_table() { - SCCache* cache = SCCache::cache(); - if (cache != nullptr && cache->_table != nullptr) { - cache->_table->init_c1(); + SCAddressTable* table = addr_table(); + if (table != nullptr) { + table->init_c1(); } } @@ -1172,7 +1172,8 @@ bool SCCache::finish_write() { total_blobs_count++; if (entries_address[i].comp_level() == CompLevel_none) { shared_blobs_count++; - } else if (entries_address[i].comp_level() == CompLevel_simple) { + } else if ((entries_address[i].comp_level() == CompLevel_simple) || + (entries_address[i].comp_level() == CompLevel_limited_profile)) { c1_blobs_count++; } else { assert(entries_address[i].comp_level() == CompLevel_full_optimization, "must be!"); @@ -2481,11 +2482,6 @@ bool SCCache::store_exception_blob(CodeBuffer* buffer, int pc_offset) { buffer->decode(); } #endif - // we need to take a lock to stop C1 and C2 compiler threads racing to - // write blobs in parallel with each other or with later nmethods - // TODO - maybe move this up to selected callers so we only lock - // when saving a c1 or opto blob - MutexLocker ml(Compile_lock); if (!cache->align_write()) { return false; } diff --git a/src/hotspot/share/code/SCCache.hpp b/src/hotspot/share/code/SCCache.hpp index a1ca68a9e45..ea4b4840ab2 100644 --- a/src/hotspot/share/code/SCCache.hpp +++ b/src/hotspot/share/code/SCCache.hpp @@ -421,6 +421,10 @@ class SCCache : public CHeapObj { uint write_bytes(const void* buffer, uint nbytes); const char* addr(uint offset) const { return _load_buffer + offset; } + static SCAddressTable* addr_table() { + return is_on() && (cache()->_table != nullptr) ? cache()->_table : nullptr; + } + bool _lookup_failed; // Failed to lookup for info (skip only this code load) void set_lookup_failed() { _lookup_failed = true; } void clear_lookup_failed() { _lookup_failed = false; } From 59b65ef17ccd05ad3f4e1e48e213fa51786d117d Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Tue, 12 Nov 2024 13:33:28 -0500 Subject: [PATCH 23/32] Address review comments by iklam Signed-off-by: Ashutosh Mehra --- src/hotspot/share/oops/method.cpp | 9 +++++++-- src/hotspot/share/oops/method.hpp | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/oops/method.cpp b/src/hotspot/share/oops/method.cpp index 8054f10d8b3..891373c333c 100644 --- a/src/hotspot/share/oops/method.cpp +++ b/src/hotspot/share/oops/method.cpp @@ -425,12 +425,16 @@ void Method::remove_unshareable_info() { JFR_ONLY(REMOVE_METHOD_ID(this);) } -void Method::restore_unshareable_info(TRAPS) { - assert(is_method() && is_valid_method(this), "ensure C++ vtable is restored"); +void Method::restore_adapter(TRAPS) { if (_adapter != nullptr) { _adapter->restore_unshareable_info(CHECK); _from_compiled_entry = _adapter->get_c2i_entry(); } +} + +void Method::restore_unshareable_info(TRAPS) { + assert(is_method() && is_valid_method(this), "ensure C++ vtable is restored"); + restore_adapter(CHECK); if (method_data() != nullptr) { method_data()->restore_unshareable_info(CHECK); } @@ -1584,6 +1588,7 @@ methodHandle Method::make_method_handle_intrinsic(vmIntrinsics::ID iid, #if INCLUDE_CDS void Method::restore_archived_method_handle_intrinsic(methodHandle m, TRAPS) { + m->restore_adapter(CHECK); if (m->adapter() != nullptr) { m->adapter()->restore_unshareable_info(CHECK); m->set_from_compiled_entry(m->adapter()->get_c2i_entry()); diff --git a/src/hotspot/share/oops/method.hpp b/src/hotspot/share/oops/method.hpp index 8431a93f210..ae1aed62cde 100644 --- a/src/hotspot/share/oops/method.hpp +++ b/src/hotspot/share/oops/method.hpp @@ -126,6 +126,7 @@ class Method : public Metadata { #if INCLUDE_CDS void remove_unshareable_info(); + void restore_adapter(TRAPS); void restore_unshareable_info(TRAPS); static void restore_archived_method_handle_intrinsic(methodHandle m, TRAPS); #endif From 9102e3eb22647db5e9858d8d1c8240bba23779df Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Tue, 10 Dec 2024 22:16:40 -0500 Subject: [PATCH 24/32] Fix the condition to check for code cache full in Method::link_method Signed-off-by: Ashutosh Mehra --- src/hotspot/share/oops/method.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/hotspot/share/oops/method.cpp b/src/hotspot/share/oops/method.cpp index 891373c333c..210786916f6 100644 --- a/src/hotspot/share/oops/method.cpp +++ b/src/hotspot/share/oops/method.cpp @@ -1270,15 +1270,11 @@ void Method::link_method(const methodHandle& h_method, TRAPS) { ClassLoader::perf_ik_link_methods_count()->inc(); } - // TODO: how to identify code cache full situation now that the adapter() can be - // non-null if AOT cache is in use -#if 0 // If the code cache is full, we may reenter this function for the // leftover methods that weren't linked. - if (adapter() != nullptr) { + if (adapter() != nullptr && !adapter()->is_shared()) { return; } -#endif assert( _code == nullptr, "nothing compiled yet" ); // Setup interpreter entrypoint From a5085d81b5f4bc72d63d951b30797255b46c5dd6 Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Thu, 2 Jan 2025 22:43:37 -0500 Subject: [PATCH 25/32] Fix merge conflict Signed-off-by: Ashutosh Mehra --- src/hotspot/share/classfile/classLoader.cpp | 33 --------------------- 1 file changed, 33 deletions(-) diff --git a/src/hotspot/share/classfile/classLoader.cpp b/src/hotspot/share/classfile/classLoader.cpp index 36222715954..266ce24de80 100644 --- a/src/hotspot/share/classfile/classLoader.cpp +++ b/src/hotspot/share/classfile/classLoader.cpp @@ -147,38 +147,6 @@ PerfCounter* ClassLoader::_perf_resolve_mh_count = nullptr; PerfCounter* ClassLoader::_perf_resolve_mt_count = nullptr; void ClassLoader::print_counters(outputStream *st) { -<<<<<<< HEAD - // The counters are only active if the logging is enabled, but - // we print to the passed in outputStream as requested. - if (log_is_enabled(Info, perf, class, link)) { - st->print_cr("ClassLoader:"); - st->print_cr(" clinit: " JLONG_FORMAT_W(6) "us / " JLONG_FORMAT " events", - ClassLoader::class_init_time_ms(), ClassLoader::class_init_count()); - st->print_cr(" link methods: " JLONG_FORMAT_W(6) "us / " JLONG_FORMAT " events", - Management::ticks_to_us(_perf_ik_link_methods_time->get_value()) , _perf_ik_link_methods_count->get_value()); - st->print_cr(" method adapters: " JLONG_FORMAT_W(6) "us / " JLONG_FORMAT " events", - Management::ticks_to_us(_perf_method_adapters_time->get_value()) , _perf_method_adapters_count->get_value()); - if (CountBytecodes || CountBytecodesPerThread) { - st->print_cr("; executed " JLONG_FORMAT " bytecodes", ClassLoader::class_init_bytecodes_count()); - } - st->print_cr(" resolve..."); - st->print_cr(" invokedynamic: " JLONG_FORMAT_W(6) "us (elapsed) " JLONG_FORMAT_W(6) "us (thread) / " JLONG_FORMAT_W(5) " events", - _perf_resolve_indy_time->elapsed_counter_value_us(), - _perf_resolve_indy_time->thread_counter_value_us(), - _perf_resolve_indy_count->get_value()); - st->print_cr(" invokehandle: " JLONG_FORMAT_W(6) "us (elapsed) " JLONG_FORMAT_W(6) "us (thread) / " JLONG_FORMAT_W(5) " events", - _perf_resolve_invokehandle_time->elapsed_counter_value_us(), - _perf_resolve_invokehandle_time->thread_counter_value_us(), - _perf_resolve_invokehandle_count->get_value()); - st->print_cr(" CP_MethodHandle: " JLONG_FORMAT_W(6) "us (elapsed) " JLONG_FORMAT_W(6) "us (thread) / " JLONG_FORMAT_W(5) " events", - _perf_resolve_mh_time->elapsed_counter_value_us(), - _perf_resolve_mh_time->thread_counter_value_us(), - _perf_resolve_mh_count->get_value()); - st->print_cr(" CP_MethodType: " JLONG_FORMAT_W(6) "us (elapsed) " JLONG_FORMAT_W(6) "us (thread) / " JLONG_FORMAT_W(5) " events", - _perf_resolve_mt_time->elapsed_counter_value_us(), - _perf_resolve_mt_time->thread_counter_value_us(), - _perf_resolve_mt_count->get_value()); -======= st->print_cr("ClassLoader:"); st->print_cr( " clinit: " JLONG_FORMAT_W(6) "us / " JLONG_FORMAT " events", ClassLoader::class_init_time_ms(), ClassLoader::class_init_count()); @@ -188,7 +156,6 @@ void ClassLoader::print_counters(outputStream *st) { Management::ticks_to_us(_perf_method_adapters_time->get_value()) , _perf_method_adapters_count->get_value()); if (CountBytecodes || CountBytecodesPerThread) { st->print_cr("; executed " JLONG_FORMAT " bytecodes", ClassLoader::class_init_bytecodes_count()); ->>>>>>> premain } st->print_cr(" resolve..."); st->print_cr(" invokedynamic: " JLONG_FORMAT_W(6) "us (elapsed) " JLONG_FORMAT_W(6) "us (thread) / " JLONG_FORMAT_W(5) " events", From 2c1c65ccd278de00b7dd43f62f086f7bfa2d0e5f Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Thu, 9 Jan 2025 21:52:48 -0500 Subject: [PATCH 26/32] Avoid adding AdapterHandlerEntry created for comparison to the handler table. Do not call delete on AdapterHandlerEntry. Instead call the destructor explicitly through a deallocate() method. Signed-off-by: Ashutosh Mehra --- src/hotspot/share/runtime/sharedRuntime.cpp | 33 +++++++++++++-------- src/hotspot/share/runtime/sharedRuntime.hpp | 9 ++++-- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 21a54e4852b..4c18c83c8b1 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -2542,11 +2542,15 @@ AdapterHandlerEntry* AdapterHandlerLibrary::lookup(AdapterFingerPrint* fp) { assert_lock_strong(AdapterHandlerLibrary_lock); AdapterHandlerEntry** entry_p = _adapter_handler_table->get(fp); if (entry_p != nullptr) { + entry = *entry_p; + assert(entry->fingerprint()->equals(fp), "fingerprint mismatch key fp %s %s (hash=%d) != found fp %s %s (hash=%d)", + entry->fingerprint()->as_basic_args_string(), entry->fingerprint()->as_string(), entry->fingerprint()->compute_hash(), + fp->as_basic_args_string(), fp->as_string(), fp->compute_hash()); #ifndef PRODUCT if (fp->is_compact()) _compact++; _runtime_hits++; #endif - return *entry_p; + return entry; } return nullptr; } @@ -2663,14 +2667,14 @@ AdapterHandlerEntry* AdapterHandlerLibrary::create_simple_adapter(AdapterBlob*& if (entry != nullptr) { assert(entry->is_shared() && !entry->is_linked(), "Non null AdapterHandlerEntry should be in the AOT cache in unlinked state"); if (!link_adapter_handler(entry, adapter_blob)) { - if (generate_adapter_code(adapter_blob, entry, total_args_passed, sig_bt, true)) { + if (generate_adapter_code(adapter_blob, entry, total_args_passed, sig_bt, /* is_transient */ true)) { return nullptr; } } // AdapterFingerPrint is already in the cache, no need to keep this one AdapterFingerPrint::deallocate(fp); } else { - entry = create_adapter(adapter_blob, fp, total_args_passed, sig_bt, true); + entry = create_adapter(adapter_blob, fp, total_args_passed, sig_bt, /* is_transient */ false); } return entry; } @@ -2800,19 +2804,19 @@ AdapterHandlerEntry* AdapterHandlerLibrary::get_adapter(const methodHandle& meth if (VerifyAdapterSharing) { AdapterBlob* comparison_blob = nullptr; AdapterFingerPrint* comparison_fp = AdapterFingerPrint::allocate(total_args_passed, sig_bt); - AdapterHandlerEntry* comparison_entry = create_adapter(comparison_blob, comparison_fp, total_args_passed, sig_bt, false); + AdapterHandlerEntry* comparison_entry = create_adapter(comparison_blob, comparison_fp, total_args_passed, sig_bt, true); assert(comparison_blob == nullptr, "no blob should be created when creating an adapter for comparison"); assert(comparison_entry->compare_code(entry), "code must match"); AdapterFingerPrint::deallocate(comparison_fp); // Release the one just created and return the original - delete comparison_entry; + AdapterHandlerEntry::deallocate(comparison_entry); } #endif AdapterFingerPrint::deallocate(fp); return entry; } - entry = create_adapter(new_adapter, fp, total_args_passed, sig_bt, /* allocate_code_blob */ true); + entry = create_adapter(new_adapter, fp, total_args_passed, sig_bt, /* is_transient */ false); } // Outside of the lock @@ -2860,7 +2864,7 @@ bool AdapterHandlerLibrary::generate_adapter_code(AdapterBlob*& adapter_blob, AdapterHandlerEntry* handler, int total_args_passed, BasicType* sig_bt, - bool allocate_code_blob) { + bool is_transient) { if (log_is_enabled(Info, perf, class, link)) { ClassLoader::perf_method_adapters_count()->inc(); } @@ -2896,7 +2900,7 @@ bool AdapterHandlerLibrary::generate_adapter_code(AdapterBlob*& adapter_blob, #ifdef ASSERT if (VerifyAdapterSharing) { handler->save_code(buf->code_begin(), buffer.insts_size()); - if (!allocate_code_blob) { + if (is_transient) { return true; } } @@ -2923,13 +2927,15 @@ AdapterHandlerEntry* AdapterHandlerLibrary::create_adapter(AdapterBlob*& adapter AdapterFingerPrint* fingerprint, int total_args_passed, BasicType* sig_bt, - bool allocate_code_blob) { + bool is_transient) { AdapterHandlerEntry* handler = AdapterHandlerLibrary::new_entry(fingerprint); - if (!generate_adapter_code(adapter_blob, handler, total_args_passed, sig_bt, allocate_code_blob)) { + if (!generate_adapter_code(adapter_blob, handler, total_args_passed, sig_bt, is_transient)) { return nullptr; } - assert_lock_strong(AdapterHandlerLibrary_lock); - _adapter_handler_table->put(fingerprint, handler); + if (!is_transient) { + assert_lock_strong(AdapterHandlerLibrary_lock); + _adapter_handler_table->put(fingerprint, handler); + } return handler; } @@ -3077,7 +3083,7 @@ void AdapterHandlerEntry::restore_unshareable_info(TRAPS) { log_warning(cds)("Failed to link AdapterHandlerEntry to its code in the AOT code cache"); int nargs; BasicType* bt = _fingerprint->as_basic_type(nargs); - if (!AdapterHandlerLibrary::generate_adapter_code(adapter_blob, this, nargs, bt, true)) { + if (!AdapterHandlerLibrary::generate_adapter_code(adapter_blob, this, nargs, bt, /* is_transient */ false)) { if (!is_init_completed()) { // Don't throw exceptions during VM initialization because java.lang.* classes // might not have been initialized, causing problems when constructing the @@ -3101,6 +3107,7 @@ AdapterHandlerEntry::~AdapterHandlerEntry() { #ifdef ASSERT FREE_C_HEAP_ARRAY(unsigned char, _saved_code); #endif + FreeHeap(this); } diff --git a/src/hotspot/share/runtime/sharedRuntime.hpp b/src/hotspot/share/runtime/sharedRuntime.hpp index 8fe29179ed9..11c7b215423 100644 --- a/src/hotspot/share/runtime/sharedRuntime.hpp +++ b/src/hotspot/share/runtime/sharedRuntime.hpp @@ -724,6 +724,11 @@ class AdapterHandlerEntry : public MetaspaceObj { { return new (mtCode) AdapterHandlerEntry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); } + + static void deallocate(AdapterHandlerEntry *handler) { + handler->~AdapterHandlerEntry(); + } + void set_entry_points(address i2c_entry, address c2i_entry, address c2i_unverified_entry, address c2i_no_clinit_check_entry, bool linked = true) { _i2c_entry = i2c_entry; _c2i_entry = c2i_entry; @@ -788,7 +793,7 @@ class AdapterHandlerLibrary: public AllStatic { AdapterFingerPrint* fingerprint, int total_args_passed, BasicType* sig_bt, - bool allocate_code_blob); + bool is_transient); #ifndef PRODUCT static void print_adapter_handler_info(AdapterHandlerEntry* handler, AdapterBlob* adapter_blob); #endif // PRODUCT @@ -806,7 +811,7 @@ class AdapterHandlerLibrary: public AllStatic { AdapterHandlerEntry* handler, int total_args_passed, BasicType* sig_bt, - bool allocate_code_blob); + bool is_transient); static void print_handler(const CodeBlob* b) { print_handler_on(tty, b); } static void print_handler_on(outputStream* st, const CodeBlob* b); From 28419f46bea5160b0e5b7d65d89f78fbaceca804 Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Thu, 9 Jan 2025 21:59:16 -0500 Subject: [PATCH 27/32] Avoid race condition between compiler thread generating exception blob and main thread generating adapters Signed-off-by: Ashutosh Mehra --- src/hotspot/share/code/SCCache.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hotspot/share/code/SCCache.cpp b/src/hotspot/share/code/SCCache.cpp index 0742df86837..cd683fa32bd 100644 --- a/src/hotspot/share/code/SCCache.cpp +++ b/src/hotspot/share/code/SCCache.cpp @@ -2489,6 +2489,8 @@ bool SCCache::store_exception_blob(CodeBuffer* buffer, int pc_offset) { buffer->decode(); } #endif + // we need to take a lock to prevent race between compiler thread generating blob and the main thread generating adapter + MutexLocker ml(Compile_lock); if (!cache->align_write()) { return false; } From 4d9025c219b4407eb5c16d27550fb6141b716011 Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Fri, 10 Jan 2025 11:05:02 -0500 Subject: [PATCH 28/32] Fix bool condition check Signed-off-by: Ashutosh Mehra --- src/hotspot/share/runtime/sharedRuntime.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 4c18c83c8b1..4cc53cf62d8 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -2667,7 +2667,7 @@ AdapterHandlerEntry* AdapterHandlerLibrary::create_simple_adapter(AdapterBlob*& if (entry != nullptr) { assert(entry->is_shared() && !entry->is_linked(), "Non null AdapterHandlerEntry should be in the AOT cache in unlinked state"); if (!link_adapter_handler(entry, adapter_blob)) { - if (generate_adapter_code(adapter_blob, entry, total_args_passed, sig_bt, /* is_transient */ true)) { + if (!generate_adapter_code(adapter_blob, entry, total_args_passed, sig_bt, /* is_transient */ true)) { return nullptr; } } From efc97745eec74b9f127e1a03298e48de4dad96c8 Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Fri, 10 Jan 2025 14:34:58 -0500 Subject: [PATCH 29/32] Fix bug in looking up AdapterFingerPrint in archived table Signed-off-by: Ashutosh Mehra --- src/hotspot/share/runtime/sharedRuntime.cpp | 17 ++++++++++++++--- src/hotspot/share/runtime/sharedRuntime.hpp | 10 ---------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 4cc53cf62d8..d6250bce5a4 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -2515,6 +2515,17 @@ class AdapterFingerPrint : public MetaspaceObj { } }; +#if INCLUDE_CDS +static inline bool adapter_fp_equals_compact_hashtable_entry(AdapterHandlerEntry* entry, AdapterFingerPrint* fp, int len_unused) { + return AdapterFingerPrint::equals(entry->fingerprint(), fp); +} + +class ArchivedAdapterTable : public OffsetCompactHashtable< + AdapterFingerPrint*, + AdapterHandlerEntry*, + adapter_fp_equals_compact_hashtable_entry> {}; +#endif // INCLUDE_CDS + // A hashtable mapping from AdapterFingerPrints to AdapterHandlerEntries using AdapterHandlerTable = ResourceHashtableis_shared() && !entry->is_linked(), "Non null AdapterHandlerEntry should be in the AOT cache in unlinked state"); if (!link_adapter_handler(entry, adapter_blob)) { - if (!generate_adapter_code(adapter_blob, entry, total_args_passed, sig_bt, /* is_transient */ true)) { + if (!generate_adapter_code(adapter_blob, entry, total_args_passed, sig_bt, /* is_transient */ false)) { return nullptr; } } @@ -2996,11 +3007,11 @@ class CopyAdapterTableToArchive : StackObj { if (lsh.is_enabled()) { address fp_runtime_addr = (address)buffered_fp + ArchiveBuilder::current()->buffer_to_requested_delta(); address entry_runtime_addr = (address)buffered_entry + ArchiveBuilder::current()->buffer_to_requested_delta(); - log_trace(cds)("Added fp=%p, entry=%p to the archived adater table", buffered_fp, buffered_entry); + log_trace(cds)("Added fp=%p (%s), entry=%p to the archived adater table", buffered_fp, buffered_fp->as_basic_args_string(), buffered_entry); } } else { if (lsh.is_enabled()) { - log_trace(cds)("Skipping adapter handler %p as it is not archived", entry); + log_trace(cds)("Skipping adapter handler %p (fp=%s) as it is not archived", entry, fp->as_basic_args_string()); } } return true; diff --git a/src/hotspot/share/runtime/sharedRuntime.hpp b/src/hotspot/share/runtime/sharedRuntime.hpp index 11c7b215423..a1760228f10 100644 --- a/src/hotspot/share/runtime/sharedRuntime.hpp +++ b/src/hotspot/share/runtime/sharedRuntime.hpp @@ -828,16 +828,6 @@ class AdapterHandlerLibrary: public AllStatic { static size_t estimate_size_for_archive() NOT_CDS_RETURN_(0); static void archive_adapter_table() NOT_CDS_RETURN; static void serialize_shared_table_header(SerializeClosure* soc) NOT_CDS_RETURN; - static bool EQUALS(AdapterHandlerEntry* entry, AdapterFingerPrint* fp, int len_unused) { - return entry->fingerprint() == fp; - } }; -#if INCLUDE_CDS -class ArchivedAdapterTable : public OffsetCompactHashtable< - AdapterFingerPrint*, - AdapterHandlerEntry*, - AdapterHandlerLibrary::EQUALS> {}; -#endif // INCLUDE_CDS - #endif // SHARE_RUNTIME_SHAREDRUNTIME_HPP From b631d117e6d5c1d9decc584b37442cb03a099aea Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Fri, 10 Jan 2025 17:25:22 -0500 Subject: [PATCH 30/32] Search archived handler table before runtime table when searching a CodeBlob Signed-off-by: Ashutosh Mehra --- src/hotspot/share/runtime/sharedRuntime.cpp | 38 ++++++++++++++++----- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index d6250bce5a4..6ef4d26bfee 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -3423,11 +3423,17 @@ JRT_END bool AdapterHandlerLibrary::contains(const CodeBlob* b) { bool found = false; - auto findblob = [&] (AdapterFingerPrint* key, AdapterHandlerEntry* a) { - return (found = (b == CodeCache::find_blob(a->get_i2c_entry()))); + auto findblob_archived_table = [&] (AdapterHandlerEntry* handler) { + return (found = (b == CodeCache::find_blob(handler->get_i2c_entry()))); }; - assert_locked_or_safepoint(AdapterHandlerLibrary_lock); - _adapter_handler_table->iterate(findblob); + _archived_adapter_handler_table.iterate(findblob_archived_table); + if (!found) { + auto findblob_runtime_table = [&] (AdapterFingerPrint* key, AdapterHandlerEntry* a) { + return (found = (b == CodeCache::find_blob(a->get_i2c_entry()))); + }; + assert_locked_or_safepoint(AdapterHandlerLibrary_lock); + _adapter_handler_table->iterate(findblob_runtime_table); + } return found; } @@ -3442,18 +3448,32 @@ uint32_t AdapterHandlerLibrary::id(AdapterFingerPrint* fingerprint) { void AdapterHandlerLibrary::print_handler_on(outputStream* st, const CodeBlob* b) { bool found = false; - auto findblob = [&] (AdapterFingerPrint* key, AdapterHandlerEntry* a) { - if (b == CodeCache::find_blob(a->get_i2c_entry())) { + auto findblob_archived_table = [&] (AdapterHandlerEntry* handler) { + if (b == CodeCache::find_blob(handler->get_i2c_entry())) { found = true; st->print("Adapter for signature: "); - a->print_adapter_on(st); + handler->print_adapter_on(st); return true; } else { return false; // keep looking + } }; - assert_locked_or_safepoint(AdapterHandlerLibrary_lock); - _adapter_handler_table->iterate(findblob); + _archived_adapter_handler_table.iterate(findblob_archived_table); + if (!found) { + auto findblob_runtime_table = [&] (AdapterFingerPrint* key, AdapterHandlerEntry* a) { + if (b == CodeCache::find_blob(a->get_i2c_entry())) { + found = true; + st->print("Adapter for signature: "); + a->print_adapter_on(st); + return true; + } else { + return false; // keep looking + } + }; + assert_locked_or_safepoint(AdapterHandlerLibrary_lock); + _adapter_handler_table->iterate(findblob_runtime_table); + } assert(found, "Should have found handler"); } From a11da423f416ecdb341896bdd857df95a0e994f5 Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Fri, 10 Jan 2025 17:35:13 -0500 Subject: [PATCH 31/32] Fix compile failure in minimal build config Signed-off-by: Ashutosh Mehra --- src/hotspot/share/runtime/sharedRuntime.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 6ef4d26bfee..94a7e10ec76 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -2537,9 +2537,10 @@ static GrowableArray* _adapter_handler_list = nullptr; // Find a entry with the same fingerprint if it exists AdapterHandlerEntry* AdapterHandlerLibrary::lookup(AdapterFingerPrint* fp) { NOT_PRODUCT(_lookups++); + AdapterHandlerEntry* entry = nullptr; #if INCLUDE_CDS // Search archived table first. It is read-only table so can be searched without lock - AdapterHandlerEntry* entry = _archived_adapter_handler_table.lookup(fp, fp->compute_hash(), 0 /* unused */); + entry = _archived_adapter_handler_table.lookup(fp, fp->compute_hash(), 0 /* unused */); if (entry != nullptr) { #ifndef PRODUCT if (fp->is_compact()) { From d4d8cf884b4b553e0a18f8e124a97c3c6806a184 Mon Sep 17 00:00:00 2001 From: Ashutosh Mehra Date: Fri, 10 Jan 2025 18:11:26 -0500 Subject: [PATCH 32/32] Fix whitespace errors Signed-off-by: Ashutosh Mehra --- src/hotspot/share/runtime/sharedRuntime.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 94a7e10ec76..7974324543c 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -3464,12 +3464,12 @@ void AdapterHandlerLibrary::print_handler_on(outputStream* st, const CodeBlob* b if (!found) { auto findblob_runtime_table = [&] (AdapterFingerPrint* key, AdapterHandlerEntry* a) { if (b == CodeCache::find_blob(a->get_i2c_entry())) { - found = true; - st->print("Adapter for signature: "); - a->print_adapter_on(st); - return true; + found = true; + st->print("Adapter for signature: "); + a->print_adapter_on(st); + return true; } else { - return false; // keep looking + return false; // keep looking } }; assert_locked_or_safepoint(AdapterHandlerLibrary_lock);