diff --git a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp index 94d73b6bc27..0b495f0c10f 100644 --- a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp @@ -558,40 +558,6 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, // If this happens, control eventually transfers back to the compiled // caller, but with an uncorrected stack, causing delayed havoc. - if (VerifyAdapterCalls && - (Interpreter::code() != nullptr || StubRoutines::final_stubs_code() != nullptr)) { -#if 0 - // So, let's test for cascading c2i/i2c adapters right now. - // assert(Interpreter::contains($return_addr) || - // StubRoutines::contains($return_addr), - // "i2c adapter must return to an interpreter frame"); - __ block_comment("verify_i2c { "); - Label L_ok; - if (Interpreter::code() != nullptr) { - range_check(masm, rax, r11, - Interpreter::code()->code_start(), Interpreter::code()->code_end(), - L_ok); - } - if (StubRoutines::initial_stubs_code() != nullptr) { - range_check(masm, rax, r11, - StubRoutines::initial_stubs_code()->code_begin(), - StubRoutines::initial_stubs_code()->code_end(), - L_ok); - } - if (StubRoutines::final_stubs_code() != nullptr) { - range_check(masm, rax, r11, - StubRoutines::final_stubs_code()->code_begin(), - StubRoutines::final_stubs_code()->code_end(), - L_ok); - } - const char* msg = "i2c adapter must return to an interpreter frame"; - __ block_comment(msg); - __ stop(msg); - __ bind(L_ok); - __ block_comment("} verify_i2ce "); -#endif - } - // Cut-out for having no stack args. int comp_words_on_stack = align_up(comp_args_on_stack*VMRegImpl::stack_slot_size, wordSize)>>LogBytesPerWord; if (comp_args_on_stack) { @@ -712,12 +678,12 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, } // --------------------------------------------------------------- -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint) { +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler) { address i2c_entry = __ pc(); gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs); @@ -778,7 +744,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + return; } static int c_calling_convention_priv(const BasicType *sig_bt, diff --git a/src/hotspot/cpu/arm/sharedRuntime_arm.cpp b/src/hotspot/cpu/arm/sharedRuntime_arm.cpp index a0cc2f44199..b41a73145c8 100644 --- a/src/hotspot/cpu/arm/sharedRuntime_arm.cpp +++ b/src/hotspot/cpu/arm/sharedRuntime_arm.cpp @@ -613,12 +613,12 @@ static void gen_c2i_adapter(MacroAssembler *masm, } -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint) { +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler) { address i2c_entry = __ pc(); gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs); @@ -638,7 +638,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm address c2i_entry = __ pc(); gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry); + handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, nullptr); + return; } diff --git a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp index 84102ef41e8..8d999c69f68 100644 --- a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp +++ b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp @@ -1144,12 +1144,12 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, __ bctr(); } -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint) { +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler) { address i2c_entry; address c2i_unverified_entry; address c2i_entry; @@ -1224,8 +1224,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, call_interpreter, ientry); - return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, - c2i_no_clinit_check_entry); + handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + return; } // An oop arg. Must pass a handle not the oop itself. diff --git a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp index 5ab2a9f00b9..83b745022f2 100644 --- a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp +++ b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp @@ -597,12 +597,13 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, } // --------------------------------------------------------------- -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint) { + +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler) { address i2c_entry = __ pc(); gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs); @@ -659,7 +660,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + return; } int SharedRuntime::vector_calling_convention(VMRegPair *regs, diff --git a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp index c60f6ef3295..3873b99f88f 100644 --- a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp +++ b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp @@ -2303,12 +2303,12 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, __ z_br(Z_R1_scratch); } -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint) { +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler) { __ align(CodeEntryAlignment); address i2c_entry = __ pc(); gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs); @@ -2362,7 +2362,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + return; } // This function returns the adjust size (in number of words) to a c2i adapter diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp index ee99f49321a..55b80b96437 100644 --- a/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp +++ b/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp @@ -707,19 +707,6 @@ static void move_i2c_double(MacroAssembler *masm, XMMRegister r, Register saved_ __ movdbl(r, Address(saved_sp, next_val_off)); } -static void range_check(MacroAssembler* masm, Register pc_reg, Register temp_reg, - address code_start, address code_end, - Label& L_ok) { - Label L_fail; - __ lea(temp_reg, AddressLiteral(code_start, relocInfo::none)); - __ cmpptr(pc_reg, temp_reg); - __ jcc(Assembler::belowEqual, L_fail); - __ lea(temp_reg, AddressLiteral(code_end, relocInfo::none)); - __ cmpptr(pc_reg, temp_reg); - __ jcc(Assembler::below, L_ok); - __ bind(L_fail); -} - void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, int total_args_passed, int comp_args_on_stack, @@ -749,38 +736,6 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, // Pick up the return address __ movptr(rax, Address(rsp, 0)); - if (VerifyAdapterCalls && - (Interpreter::code() != nullptr || StubRoutines::final_stubs_code() != nullptr)) { - // So, let's test for cascading c2i/i2c adapters right now. - // assert(Interpreter::contains($return_addr) || - // StubRoutines::contains($return_addr), - // "i2c adapter must return to an interpreter frame"); - __ block_comment("verify_i2c { "); - Label L_ok; - if (Interpreter::code() != nullptr) { - range_check(masm, rax, rdi, - Interpreter::code()->code_start(), Interpreter::code()->code_end(), - L_ok); - } - if (StubRoutines::initial_stubs_code() != nullptr) { - range_check(masm, rax, rdi, - StubRoutines::initial_stubs_code()->code_begin(), - StubRoutines::initial_stubs_code()->code_end(), - L_ok); - } - if (StubRoutines::final_stubs_code() != nullptr) { - range_check(masm, rax, rdi, - StubRoutines::final_stubs_code()->code_begin(), - StubRoutines::final_stubs_code()->code_end(), - L_ok); - } - const char* msg = "i2c adapter must return to an interpreter frame"; - __ block_comment(msg); - __ stop(msg); - __ bind(L_ok); - __ block_comment("} verify_i2ce "); - } - // Must preserve original SP for loading incoming arguments because // we need to align the outgoing SP for compiled code. __ movptr(rdi, rsp); @@ -928,12 +883,12 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, } // --------------------------------------------------------------- -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint) { +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler) { address i2c_entry = __ pc(); gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs); @@ -971,7 +926,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry); + handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, nullptr); + return; } int SharedRuntime::c_calling_convention(const BasicType *sig_bt, diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp index b98a17848df..e825b389e52 100644 --- a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp +++ b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp @@ -677,7 +677,6 @@ static void patch_callers_callsite(MacroAssembler *masm) { __ bind(L); } - static void gen_c2i_adapter(MacroAssembler *masm, int total_args_passed, int comp_args_on_stack, @@ -828,19 +827,6 @@ static void gen_c2i_adapter(MacroAssembler *masm, __ jmp(rcx); } -static void range_check(MacroAssembler* masm, Register pc_reg, Register temp_reg, - address code_start, address code_end, - Label& L_ok) { - Label L_fail; - __ lea(temp_reg, AddressLiteral(code_start, relocInfo::none)); - __ cmpptr(pc_reg, temp_reg); - __ jcc(Assembler::belowEqual, L_fail); - __ lea(temp_reg, AddressLiteral(code_end, relocInfo::none)); - __ cmpptr(pc_reg, temp_reg); - __ jcc(Assembler::below, L_ok); - __ bind(L_fail); -} - void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, int total_args_passed, int comp_args_on_stack, @@ -873,41 +859,6 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, // If this happens, control eventually transfers back to the compiled // caller, but with an uncorrected stack, causing delayed havoc. - if (VerifyAdapterCalls && - (Interpreter::code() != nullptr || StubRoutines::final_stubs_code() != nullptr)) { - // So, let's test for cascading c2i/i2c adapters right now. - // assert(Interpreter::contains($return_addr) || - // StubRoutines::contains($return_addr), - // "i2c adapter must return to an interpreter frame"); - __ block_comment("verify_i2c { "); - // Pick up the return address - __ movptr(rax, Address(rsp, 0)); - Label L_ok; - if (Interpreter::code() != nullptr) { - range_check(masm, rax, r11, - Interpreter::code()->code_start(), - Interpreter::code()->code_end(), - L_ok); - } - if (StubRoutines::initial_stubs_code() != nullptr) { - range_check(masm, rax, r11, - StubRoutines::initial_stubs_code()->code_begin(), - StubRoutines::initial_stubs_code()->code_end(), - L_ok); - } - if (StubRoutines::final_stubs_code() != nullptr) { - range_check(masm, rax, r11, - StubRoutines::final_stubs_code()->code_begin(), - StubRoutines::final_stubs_code()->code_end(), - L_ok); - } - const char* msg = "i2c adapter must return to an interpreter frame"; - __ block_comment(msg); - __ stop(msg); - __ bind(L_ok); - __ block_comment("} verify_i2ce "); - } - // Must preserve original SP for loading incoming arguments because // we need to align the outgoing SP for compiled code. __ movptr(r11, rsp); @@ -1052,12 +1003,12 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, } // --------------------------------------------------------------- -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint) { +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler) { address i2c_entry = __ pc(); gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs); @@ -1119,7 +1070,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + return; } int SharedRuntime::c_calling_convention(const BasicType *sig_bt, diff --git a/src/hotspot/cpu/zero/sharedRuntime_zero.cpp b/src/hotspot/cpu/zero/sharedRuntime_zero.cpp index 094ddc05a2e..ef8d3fc3c9e 100644 --- a/src/hotspot/cpu/zero/sharedRuntime_zero.cpp +++ b/src/hotspot/cpu/zero/sharedRuntime_zero.cpp @@ -51,18 +51,17 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt, return 0; } -AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters( - MacroAssembler *masm, - int total_args_passed, - int comp_args_on_stack, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint *fingerprint) { - return AdapterHandlerLibrary::new_entry( - fingerprint, - CAST_FROM_FN_PTR(address,zero_null_code_stub), - CAST_FROM_FN_PTR(address,zero_null_code_stub), - CAST_FROM_FN_PTR(address,zero_null_code_stub)); +void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler) { + handler->set_entry_points(CAST_FROM_FN_PTR(address,zero_null_code_stub), + CAST_FROM_FN_PTR(address,zero_null_code_stub), + CAST_FROM_FN_PTR(address,zero_null_code_stub), + nullptr); + return; } nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm, diff --git a/src/hotspot/share/cds/archiveBuilder.cpp b/src/hotspot/share/cds/archiveBuilder.cpp index 4839f795c54..1c3f855956d 100644 --- a/src/hotspot/share/cds/archiveBuilder.cpp +++ b/src/hotspot/share/cds/archiveBuilder.cpp @@ -325,7 +325,8 @@ size_t ArchiveBuilder::estimate_archive_size() { size_t symbol_table_est = SymbolTable::estimate_size_for_archive(); size_t dictionary_est = SystemDictionaryShared::estimate_size_for_archive(); size_t training_data_est = TrainingData::estimate_size_for_archive(); - _estimated_hashtable_bytes = symbol_table_est + dictionary_est + training_data_est; + size_t adapter_data_est = AdapterHandlerLibrary::estimate_size_for_archive(); + _estimated_hashtable_bytes = symbol_table_est + dictionary_est + training_data_est + adapter_data_est; if (CDSConfig::is_dumping_final_static_archive()) { _estimated_hashtable_bytes += 200 * 1024 * 1024; // FIXME -- need to iterate archived symbols?? @@ -586,6 +587,13 @@ ArchiveBuilder::FollowMode ArchiveBuilder::get_follow_mode(MetaspaceClosure::Ref ref->msotype() == MetaspaceObj::MethodTrainingDataType || ref->msotype() == MetaspaceObj::CompileTrainingDataType) { return TrainingData::need_data() ? make_a_copy : set_to_null; + } else if (ref->msotype() == MetaspaceObj::AdapterHandlerEntryType) { + if (CDSConfig::is_dumping_adapters()) { + AdapterHandlerEntry* entry = (AdapterHandlerEntry*)ref->obj(); + return AdapterHandlerLibrary::is_abstract_method_adapter(entry) ? set_to_null : make_a_copy; + } else { + return set_to_null; + } } else { if (ref->msotype() == MetaspaceObj::ClassType) { Klass* klass = (Klass*)ref->obj(); diff --git a/src/hotspot/share/cds/cdsConfig.cpp b/src/hotspot/share/cds/cdsConfig.cpp index 23763dbc55f..c6426c2f7cc 100644 --- a/src/hotspot/share/cds/cdsConfig.cpp +++ b/src/hotspot/share/cds/cdsConfig.cpp @@ -32,6 +32,7 @@ #include "classfile/classLoaderDataShared.hpp" #include "classfile/moduleEntry.hpp" #include "classfile/systemDictionaryShared.hpp" +#include "code/SCCache.hpp" #include "include/jvm_io.h" #include "logging/log.hpp" #include "prims/jvmtiExport.hpp" @@ -554,6 +555,10 @@ bool CDSConfig::check_vm_args_consistency(bool patch_mod_javabase, bool mode_fla // Cannot dump cached code until metadata and heap are dumped. disable_dumping_cached_code(); } + if (StoreCachedCode) { + log_info(cds)("ArchiveAdapters is enabled"); + FLAG_SET_ERGO_IF_DEFAULT(ArchiveAdapters, true); + } } } else { // Old workflow @@ -906,3 +911,7 @@ void CDSConfig::disable_dumping_cached_code() { void CDSConfig::enable_dumping_cached_code() { _is_dumping_cached_code = true; } + +bool CDSConfig::is_dumping_adapters() { + return (ArchiveAdapters && is_dumping_final_static_archive()); +} diff --git a/src/hotspot/share/cds/cdsConfig.hpp b/src/hotspot/share/cds/cdsConfig.hpp index be7e7f80688..809fafe360d 100644 --- a/src/hotspot/share/cds/cdsConfig.hpp +++ b/src/hotspot/share/cds/cdsConfig.hpp @@ -165,6 +165,8 @@ class CDSConfig : public AllStatic { static void disable_dumping_cached_code() NOT_CDS_RETURN; static void enable_dumping_cached_code() NOT_CDS_RETURN; + static bool is_dumping_adapters() NOT_CDS_RETURN_(false); + // Some CDS functions assume that they are called only within a single-threaded context. I.e., // they are called from: // - The VM thread (e.g., inside VM_PopulateDumpSharedSpace) diff --git a/src/hotspot/share/cds/cds_globals.hpp b/src/hotspot/share/cds/cds_globals.hpp index 6d161374229..674b86d32b1 100644 --- a/src/hotspot/share/cds/cds_globals.hpp +++ b/src/hotspot/share/cds/cds_globals.hpp @@ -172,6 +172,10 @@ \ product(bool, SkipArchiveHeapVerification, false, \ "Skip verification of CDS archive heap") \ + \ + product(bool, ArchiveAdapters, false, \ + "Archive AdapterFingerPrint and AdapterHandlerEntry." \ + "Requires AOT code cache") \ // end of CDS_FLAGS diff --git a/src/hotspot/share/cds/cppVtables.cpp b/src/hotspot/share/cds/cppVtables.cpp index 138268ca4fd..9faf3e4990c 100644 --- a/src/hotspot/share/cds/cppVtables.cpp +++ b/src/hotspot/share/cds/cppVtables.cpp @@ -282,6 +282,8 @@ intptr_t* CppVtables::get_archived_vtable(MetaspaceObj::Type msotype, address ob case MetaspaceObj::AnnotationsType: case MetaspaceObj::SharedClassPathEntryType: case MetaspaceObj::RecordComponentType: + case MetaspaceObj::AdapterHandlerEntryType: + case MetaspaceObj::AdapterFingerPrintType: // These have no vtables. break; default: diff --git a/src/hotspot/share/cds/metaspaceShared.cpp b/src/hotspot/share/cds/metaspaceShared.cpp index 279962a05fa..c43884bcbb7 100644 --- a/src/hotspot/share/cds/metaspaceShared.cpp +++ b/src/hotspot/share/cds/metaspaceShared.cpp @@ -492,6 +492,8 @@ void MetaspaceShared::serialize(SerializeClosure* soc) { soc->do_ptr((void**)&_archived_method_handle_intrinsics); LambdaFormInvokers::serialize(soc); + AdapterHandlerLibrary::serialize_shared_table_header(soc); + soc->do_tag(666); } @@ -615,6 +617,10 @@ char* VM_PopulateDumpSharedSpace::dump_read_only_tables() { // Write lambform lines into archive LambdaFormInvokers::dump_static_archive_invokers(); + if (CDSConfig::is_dumping_adapters()) { + AdapterHandlerLibrary::archive_adapter_table(); + } + // Write the other data to the output array. DumpRegion* ro_region = ArchiveBuilder::current()->ro_region(); char* start = ro_region->top(); diff --git a/src/hotspot/share/code/SCCache.cpp b/src/hotspot/share/code/SCCache.cpp index ff89e7570be..cd683fa32bd 100644 --- a/src/hotspot/share/code/SCCache.cpp +++ b/src/hotspot/share/code/SCCache.cpp @@ -176,6 +176,13 @@ void SCCache::init2() { close(); exit_vm_on_load_failure(); } + + // initialize the table of external routines so we can save + // generated code blobs that reference them + init_extrs_table(); + // initialize the table of initial stubs so we can save + // generated code blobs that reference them + init_early_stubs_table(); } void SCCache::print_timers_on(outputStream* st) { @@ -244,6 +251,8 @@ void SCCache::close() { } LogStreamHandle(Info, scc, codecache) info_scc; + // need a lock to traverse the code cache + MutexLocker locker(CodeCache_lock, Mutex::_no_safepoint_check_flag); if (info_scc.is_enabled()) { NMethodIterator iter(NMethodIterator::all); while (iter.next()) { @@ -587,24 +596,42 @@ SCCache::SCCache(const char* cache_path, int fd, uint load_size) { _table = new SCAddressTable(); } -void SCCache::init_table() { - SCCache* cache = SCCache::cache(); - if (cache != nullptr && cache->_table != nullptr) { - cache->_table->init(); +void SCCache::init_extrs_table() { + SCAddressTable* table = addr_table(); + if (table != nullptr) { + table->init_extrs(); + } +} +void SCCache::init_early_stubs_table() { + SCAddressTable* table = addr_table(); + if (table != nullptr) { + table->init_early_stubs(); + } +} +void SCCache::init_shared_blobs_table() { + SCAddressTable* table = addr_table(); + if (table != nullptr) { + table->init_shared_blobs(); + } +} +void SCCache::init_stubs_table() { + SCAddressTable* table = addr_table(); + if (table != nullptr) { + table->init_stubs(); } } void SCCache::init_opto_table() { - SCCache* cache = SCCache::cache(); - if (cache != nullptr && cache->_table != nullptr) { - cache->_table->init_opto(); + SCAddressTable* table = addr_table(); + if (table != nullptr) { + table->init_opto(); } } void SCCache::init_c1_table() { - SCCache* cache = SCCache::cache(); - if (cache != nullptr && cache->_table != nullptr) { - cache->_table->init_c1(); + SCAddressTable* table = addr_table(); + if (table != nullptr) { + table->init_c1(); } } @@ -1119,6 +1146,12 @@ bool SCCache::finish_write() { SCCEntry* entries_address = _store_entries; // Pointer to latest entry uint not_entrant_nb = 0; + uint stubs_count = 0; + uint adapters_count = 0; + uint shared_blobs_count = 0; + uint c1_blobs_count = 0; + uint opto_blobs_count = 0; + uint total_blobs_count = 0; uint max_size = 0; // Add old entries first if (_for_read && (_load_header != nullptr)) { @@ -1138,6 +1171,21 @@ bool SCCache::finish_write() { } else if (_load_entries[i].for_preload() && _load_entries[i].method() != nullptr) { // record entrant first version code for pre-loading preload_entries[preload_entries_cnt++] = entries_count; + } else if (entries_address[i].kind() == SCCEntry::Adapter) { + adapters_count++; + } else if (entries_address[i].kind() == SCCEntry::Stub) { + stubs_count++; + } else if (entries_address[i].kind() == SCCEntry::Blob) { + total_blobs_count++; + if (entries_address[i].comp_level() == CompLevel_none) { + shared_blobs_count++; + } else if ((entries_address[i].comp_level() == CompLevel_simple) || + (entries_address[i].comp_level() == CompLevel_limited_profile)) { + c1_blobs_count++; + } else { + assert(entries_address[i].comp_level() == CompLevel_full_optimization, "must be!"); + opto_blobs_count++; + } } { uint size = align_up(_load_entries[i].size(), DATA_ALIGNMENT); @@ -1236,7 +1284,12 @@ bool SCCache::finish_write() { copy_bytes((_store_buffer + entries_offset), (address)current, entries_size); current += entries_size; log_info(scc, exit)("Wrote %d SCCEntry entries (%d were not entrant, %d max size) to Startup Code Cache '%s'", entries_count, not_entrant_nb, max_size, _cache_path); - + log_info(scc, exit)(" Stubs: total=%d", stubs_count); + log_info(scc, exit)(" Adapters: total=%d", adapters_count); + log_info(scc, exit)(" Shared Blobs: total=%d",shared_blobs_count); + log_info(scc, exit)(" C1 Blobs: total=%d", c1_blobs_count); + log_info(scc, exit)(" Opto Blobs: total=%d", opto_blobs_count); + log_info(scc, exit)(" All Blobs: total=%d", total_blobs_count); uint size = (current - start); assert(size <= total_size, "%d > %d", size , total_size); @@ -1952,7 +2005,6 @@ bool SCCReader::read_relocations(CodeBuffer* buffer, CodeBuffer* orig_buffer, bool SCCReader::read_code(CodeBuffer* buffer, CodeBuffer* orig_buffer, uint code_offset) { assert(code_offset == align_up(code_offset, DATA_ALIGNMENT), "%d not aligned to %d", code_offset, DATA_ALIGNMENT); - assert(buffer->blob() != nullptr, "sanity"); SCCodeSection* scc_cs = (SCCodeSection*)addr(code_offset); for (int i = 0; i < (int)CodeBuffer::SECT_LIMIT; i++) { CodeSection* cs = buffer->code_section(i); @@ -1989,6 +2041,82 @@ bool SCCReader::read_code(CodeBuffer* buffer, CodeBuffer* orig_buffer, uint code return true; } +bool SCCache::load_adapter(CodeBuffer* buffer, uint32_t id, const char* name, uint32_t offsets[4]) { +#ifdef ASSERT + LogStreamHandle(Debug, scc, stubs) log; + if (log.is_enabled()) { + FlagSetting fs(PrintRelocations, true); + buffer->print_on(&log); + } +#endif + SCCache* cache = open_for_read(); + if (cache == nullptr) { + return false; + } + log_info(scc, stubs)("Looking up adapter %s (0x%x) in Startup Code Cache '%s'", + name, id, _cache->cache_path()); + SCCEntry* entry = cache->find_entry(SCCEntry::Adapter, id); + if (entry == nullptr) { + return false; + } + SCCReader reader(cache, entry, nullptr); + return reader.compile_adapter(buffer, name, offsets); +} +bool SCCReader::compile_adapter(CodeBuffer* buffer, const char* name, uint32_t offsets[4]) { + uint entry_position = _entry->offset(); + // Read name + uint name_offset = entry_position + _entry->name_offset(); + uint name_size = _entry->name_size(); // Includes '/0' + const char* stored_name = addr(name_offset); + log_info(scc, stubs)("%d (L%d): Reading adapter '%s' from Startup Code Cache '%s'", + compile_id(), comp_level(), name, _cache->cache_path()); + if (strncmp(stored_name, name, (name_size - 1)) != 0) { + log_warning(scc)("%d (L%d): Saved adapter's name '%s' is different from '%s'", + compile_id(), comp_level(), stored_name, name); + // n.b. this is not fatal -- we have just seen a hash id clash + // so no need to call cache->set_failed() + return false; + } + // Create fake original CodeBuffer + CodeBuffer orig_buffer(name); + // Read code + uint code_offset = entry_position + _entry->code_offset(); + if (!read_code(buffer, &orig_buffer, code_offset)) { + return false; + } + // Read relocations + uint reloc_offset = entry_position + _entry->reloc_offset(); + set_read_position(reloc_offset); + if (!read_relocations(buffer, &orig_buffer, nullptr, nullptr)) { + return false; + } + uint offset = read_position(); + int offsets_count = *(int*)addr(offset); + offset += sizeof(int); + assert(offsets_count == 4, "wrong caller expectations"); + set_read_position(offset); + for (int i = 0; i < offsets_count; i++) { + uint32_t arg = *(uint32_t*)addr(offset); + offset += sizeof(uint32_t); + log_debug(scc, stubs)("%d (L%d): Reading adapter '%s' offsets[%d] == 0x%x from Startup Code Cache '%s'", + compile_id(), comp_level(), stored_name, i, arg, _cache->cache_path()); + offsets[i] = arg; + } + log_debug(scc, stubs)("%d (L%d): Read adapter '%s' with '%d' args from Startup Code Cache '%s'", + compile_id(), comp_level(), stored_name, offsets_count, _cache->cache_path()); +#ifdef ASSERT + LogStreamHandle(Debug, scc, stubs) log; + if (log.is_enabled()) { + FlagSetting fs(PrintRelocations, true); + buffer->print_on(&log); + buffer->decode(); + } +#endif + // mark entry as loaded + ((SCCEntry *)_entry)->set_loaded(); + return true; +} + bool SCCache::load_exception_blob(CodeBuffer* buffer, int* pc_offset) { #ifdef ASSERT LogStreamHandle(Debug, scc, nmethod) log; @@ -2232,6 +2360,71 @@ bool SCCache::write_relocations(CodeBuffer* buffer, uint& all_reloc_size) { return success; } +bool SCCache::store_adapter(CodeBuffer* buffer, uint32_t id, const char* name, uint32_t offsets[4]) { + assert(CDSConfig::is_dumping_adapters(), "must be"); + SCCache* cache = open_for_write(); + if (cache == nullptr) { + return false; + } + log_info(scc, stubs)("Writing adapter '%s' (0x%x) to Startup Code Cache '%s'", name, id, cache->_cache_path); +#ifdef ASSERT + LogStreamHandle(Debug, scc, stubs) log; + if (log.is_enabled()) { + FlagSetting fs(PrintRelocations, true); + buffer->print_on(&log); + buffer->decode(); + } +#endif + // we need to take a lock to stop main thread racing with C1 and C2 compiler threads to + // write blobs in parallel with each other or with later nmethods + MutexLocker ml(Compile_lock); + if (!cache->align_write()) { + return false; + } + uint entry_position = cache->_write_position; + // Write name + uint name_offset = cache->_write_position - entry_position; + uint name_size = (uint)strlen(name) + 1; // Includes '/0' + uint n = cache->write_bytes(name, name_size); + if (n != name_size) { + return false; + } + // Write code section + if (!cache->align_write()) { + return false; + } + uint code_offset = cache->_write_position - entry_position; + uint code_size = 0; + if (!cache->write_code(buffer, code_size)) { + return false; + } + // Write relocInfo array + uint reloc_offset = cache->_write_position - entry_position; + uint reloc_size = 0; + if (!cache->write_relocations(buffer, reloc_size)) { + return false; + } + int extras_count = 4; + n = cache->write_bytes(&extras_count, sizeof(int)); + if (n != sizeof(int)) { + return false; + } + for (int i = 0; i < 4; i++) { + uint32_t arg = offsets[i]; + log_debug(scc, stubs)("Writing adapter '%s' (0x%x) offsets[%d] == 0x%x to Startup Code Cache '%s'", name, id, i, arg, cache->_cache_path); + n = cache->write_bytes(&arg, sizeof(uint32_t)); + if (n != sizeof(uint32_t)) { + return false; + } + } + uint entry_size = cache->_write_position - entry_position; + SCCEntry* entry = new (cache) SCCEntry(entry_position, entry_size, name_offset, name_size, + code_offset, code_size, reloc_offset, reloc_size, + SCCEntry::Adapter, id, 0); + log_info(scc, stubs)("Wrote adapter '%s' (0x%x) to Startup Code Cache '%s'", name, id, cache->_cache_path); + return true; +} + bool SCCache::write_code(CodeBuffer* buffer, uint& code_size) { assert(_write_position == align_up(_write_position, DATA_ALIGNMENT), "%d not aligned to %d", _write_position, DATA_ALIGNMENT); //assert(buffer->blob() != nullptr, "sanity"); @@ -2296,6 +2489,8 @@ bool SCCache::store_exception_blob(CodeBuffer* buffer, int pc_offset) { buffer->decode(); } #endif + // we need to take a lock to prevent race between compiler thread generating blob and the main thread generating adapter + MutexLocker ml(Compile_lock); if (!cache->align_write()) { return false; } @@ -3371,7 +3566,7 @@ static void print_helper1(outputStream* st, const char* name, int count) { st->print(" %s=%d", name, count); } } -static void print_helper(outputStream* st, const char* name, int stats[6+3][6], int idx) { +static void print_helper(outputStream* st, const char* name, int stats[6+3+1][6], int idx) { int total = stats[idx][0]; if (total > 0) { st->print(" %s:", name); @@ -3398,7 +3593,8 @@ void SCCache::print_statistics_on(outputStream* st) { uint* search_entries = (uint*)cache->addr(cache->_load_header->entries_offset()); // [id, index] SCCEntry* load_entries = (SCCEntry*)(search_entries + 2 * count); - int stats[6 + 3][6] = {0}; + // Entries are None, Adapter, Stub, Blob x 3 levels, Code x 6 levels + int stats[6 + 3 + 1][6] = {0}; for (uint i = 0; i < count; i++) { int index = search_entries[2*i + 1]; SCCEntry* entry = &(load_entries[index]); @@ -3428,6 +3624,7 @@ void SCCache::print_statistics_on(outputStream* st) { print_helper(st, "None", stats, SCCEntry::None); print_helper(st, "Stub", stats, SCCEntry::Stub); print_helper(st, "Blob", stats, SCCEntry::Blob); + print_helper(st, "Adapters", stats, SCCEntry::Adapter); for (int lvl = 0; lvl <= CompLevel_full_optimization + 1; lvl++) { ResourceMark rm; stringStream ss; @@ -3526,38 +3723,46 @@ void SCCReader::print_on(outputStream* st) { st->print_cr(" name: %s", name); } +// address table ids for generated routines, external addresses and C +// string addresses are partitioned into positive integer ranges +// defined by the following positive base and max values +// i.e. [_extrs_base, _extrs_base + _extrs_max -1], +// [_stubs_base, _stubs_base + _stubs_max -1], +// ... +// [_c_str_base, _c_str_base + _c_str_max -1], #define _extrs_max 80 #define _stubs_max 120 -#define _blobs_max 100 -#define _shared_blobs_max 24 +#define _all_blobs_max 100 +#define _blobs_max 24 #define _C2_blobs_max 25 -#define _C1_blobs_max (_blobs_max - _shared_blobs_max - _C2_blobs_max) +#define _C1_blobs_max (_all_blobs_max - _blobs_max - _C2_blobs_max) #define _all_max 300 +#define _c_str_max MAX_STR_COUNT +#define _extrs_base 0 +#define _stubs_base (_extrs_base + _extrs_max) +#define _blobs_base (_stubs_base + _stubs_max) +#define _C1_blobs_base (_blobs_base + _blobs_max) +#define _C2_blobs_base (_C1_blobs_base + _C1_blobs_max) +#if (_C2_blobs_base >= _all_max) +#error SCAddress table ranges need adjusting +#endif +#define _c_str_base _all_max + #define SET_ADDRESS(type, addr) \ { \ type##_addr[type##_length++] = (address) (addr); \ assert(type##_length <= type##_max, "increase size"); \ } -static bool initializing = false; -void SCAddressTable::init() { - if (_complete || initializing) return; // Done already - initializing = true; +static bool initializing_extrs = false; +void SCAddressTable::init_extrs() { + if (_extrs_complete || initializing_extrs) return; // Done already + initializing_extrs = true; _extrs_addr = NEW_C_HEAP_ARRAY(address, _extrs_max, mtCode); - _stubs_addr = NEW_C_HEAP_ARRAY(address, _stubs_max, mtCode); - _blobs_addr = NEW_C_HEAP_ARRAY(address, _blobs_max, mtCode); - - // Divide _blobs_addr array to chunks because they could be initialized in parrallel - _C2_blobs_addr = _blobs_addr + _shared_blobs_max;// C2 blobs addresses stored after shared blobs - _C1_blobs_addr = _C2_blobs_addr + _C2_blobs_max; // C1 blobs addresses stored after C2 blobs _extrs_length = 0; _stubs_length = 0; - _blobs_length = 0; // for shared blobs - _C1_blobs_length = 0; - _C2_blobs_length = 0; - _final_blobs_length = 0; // Depends on numnber of C1 blobs // Runtime methods #ifdef COMPILER2 @@ -3586,6 +3791,19 @@ void SCAddressTable::init() { SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_phantom); SET_ADDRESS(_extrs, ShenandoahRuntime::load_reference_barrier_phantom_narrow); #endif + SET_ADDRESS(_extrs, SharedRuntime::fixup_callers_callsite); + + SET_ADDRESS(_extrs, SharedRuntime::log_jni_monitor_still_held); + SET_ADDRESS(_extrs, SharedRuntime::rc_trace_method_entry); + SET_ADDRESS(_extrs, SharedRuntime::reguard_yellow_pages); + SET_ADDRESS(_extrs, SharedRuntime::dtrace_method_exit); + + SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method); + SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method_abstract); + SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method_ic_miss); + SET_ADDRESS(_extrs, SharedRuntime::resolve_opt_virtual_call_C); + SET_ADDRESS(_extrs, SharedRuntime::resolve_virtual_call_C); + SET_ADDRESS(_extrs, SharedRuntime::resolve_static_call_C); SET_ADDRESS(_extrs, SharedRuntime::complete_monitor_unlocking_C); SET_ADDRESS(_extrs, SharedRuntime::enable_stack_reserved_zone); @@ -3659,9 +3877,71 @@ void SCAddressTable::init() { while (*p != nullptr) { SET_ADDRESS(_extrs, *p++); } + + _extrs_complete = true; + log_info(scc,init)("External addresses recorded"); +} + +static bool initializing_early_stubs = false; +void SCAddressTable::init_early_stubs() { + if (_complete || initializing_early_stubs) return; // Done already + initializing_early_stubs = true; + _stubs_addr = NEW_C_HEAP_ARRAY(address, _stubs_max, mtCode); + _stubs_length = 0; + SET_ADDRESS(_stubs, StubRoutines::forward_exception_entry()); + _early_stubs_complete = true; + log_info(scc,init)("early stubs recorded"); +} + +static bool initializing_shared_blobs = false; +void SCAddressTable::init_shared_blobs() { + if (_complete || initializing_shared_blobs) return; // Done already + initializing_shared_blobs = true; + _blobs_addr = NEW_C_HEAP_ARRAY(address, _all_blobs_max, mtCode); + + // Divide _blobs_addr array to chunks because they could be initialized in parrallel + _C1_blobs_addr = _blobs_addr + _blobs_max;// C1 blobs addresses stored after shared blobs + _C2_blobs_addr = _C1_blobs_addr + _C1_blobs_max; // C2 blobs addresses stored after C1 blobs + + _blobs_length = 0; // for shared blobs + _C1_blobs_length = 0; + _C2_blobs_length = 0; + + // Blobs + SET_ADDRESS(_blobs, SharedRuntime::get_handle_wrong_method_stub()); + SET_ADDRESS(_blobs, SharedRuntime::get_ic_miss_stub()); + SET_ADDRESS(_blobs, SharedRuntime::get_resolve_opt_virtual_call_stub()); + SET_ADDRESS(_blobs, SharedRuntime::get_resolve_virtual_call_stub()); + SET_ADDRESS(_blobs, SharedRuntime::get_resolve_static_call_stub()); + SET_ADDRESS(_blobs, SharedRuntime::deopt_blob()->entry_point()); + SET_ADDRESS(_blobs, SharedRuntime::polling_page_safepoint_handler_blob()->entry_point()); + SET_ADDRESS(_blobs, SharedRuntime::polling_page_return_handler_blob()->entry_point()); +#ifdef COMPILER2 + SET_ADDRESS(_blobs, SharedRuntime::polling_page_vectors_safepoint_handler_blob()->entry_point()); +#endif + + assert(_blobs_length <= _blobs_max, "increase _blobs_max to %d", _blobs_length); + log_info(scc,init)("Early shared blobs recorded"); +} + +static bool initializing_stubs = false; +void SCAddressTable::init_stubs() { + if (_complete || initializing_stubs) return; // Done already + initializing_stubs = true; + // final blobs + SET_ADDRESS(_blobs, SharedRuntime::throw_AbstractMethodError_entry()); + SET_ADDRESS(_blobs, SharedRuntime::throw_IncompatibleClassChangeError_entry()); + SET_ADDRESS(_blobs, SharedRuntime::throw_NullPointerException_at_call_entry()); + SET_ADDRESS(_blobs, SharedRuntime::throw_StackOverflowError_entry()); + SET_ADDRESS(_blobs, SharedRuntime::throw_delayed_StackOverflowError_entry()); + + assert(_blobs_length <= _blobs_max, "increase _blobs_max to %d", _blobs_length); + + _shared_blobs_complete = true; + log_info(scc,init)("All shared blobs recorded"); + // Stubs SET_ADDRESS(_stubs, StubRoutines::method_entry_barrier()); - SET_ADDRESS(_stubs, StubRoutines::forward_exception_entry()); /* SET_ADDRESS(_stubs, StubRoutines::throw_AbstractMethodError_entry()); SET_ADDRESS(_stubs, StubRoutines::throw_IncompatibleClassChangeError_entry()); @@ -3830,29 +4110,8 @@ void SCAddressTable::init() { SET_ADDRESS(_stubs, StubRoutines::aarch64::large_arrays_hashcode(T_INT)); #endif - // Blobs - SET_ADDRESS(_blobs, SharedRuntime::get_handle_wrong_method_stub()); - SET_ADDRESS(_blobs, SharedRuntime::get_ic_miss_stub()); - SET_ADDRESS(_blobs, SharedRuntime::get_resolve_opt_virtual_call_stub()); - SET_ADDRESS(_blobs, SharedRuntime::get_resolve_virtual_call_stub()); - SET_ADDRESS(_blobs, SharedRuntime::get_resolve_static_call_stub()); - SET_ADDRESS(_blobs, SharedRuntime::deopt_blob()->entry_point()); - SET_ADDRESS(_blobs, SharedRuntime::polling_page_safepoint_handler_blob()->entry_point()); - SET_ADDRESS(_blobs, SharedRuntime::polling_page_return_handler_blob()->entry_point()); -#ifdef COMPILER2 - SET_ADDRESS(_blobs, SharedRuntime::polling_page_vectors_safepoint_handler_blob()->entry_point()); -#endif - - SET_ADDRESS(_blobs, SharedRuntime::throw_AbstractMethodError_entry()); - SET_ADDRESS(_blobs, SharedRuntime::throw_IncompatibleClassChangeError_entry()); - SET_ADDRESS(_blobs, SharedRuntime::throw_NullPointerException_at_call_entry()); - SET_ADDRESS(_blobs, SharedRuntime::throw_StackOverflowError_entry()); - SET_ADDRESS(_blobs, SharedRuntime::throw_delayed_StackOverflowError_entry()); - - assert(_blobs_length <= _shared_blobs_max, "increase _shared_blobs_max to %d", _blobs_length); - _final_blobs_length = _blobs_length; _complete = true; - log_info(scc,init)("External addresses and stubs recorded"); + log_info(scc,init)("Stubs recorded"); } void SCAddressTable::init_opto() { @@ -3885,7 +4144,6 @@ void SCAddressTable::init_opto() { #endif assert(_C2_blobs_length <= _C2_blobs_max, "increase _C2_blobs_max to %d", _C2_blobs_length); - _final_blobs_length = MAX2(_final_blobs_length, (_shared_blobs_max + _C2_blobs_length)); _opto_complete = true; log_info(scc,init)("OptoRuntime Blobs recorded"); } @@ -3937,18 +4195,11 @@ void SCAddressTable::init_c1() { #endif // COMPILER1 assert(_C1_blobs_length <= _C1_blobs_max, "increase _C1_blobs_max to %d", _C1_blobs_length); - _final_blobs_length = MAX2(_final_blobs_length, (_shared_blobs_max + _C2_blobs_max + _C1_blobs_length)); _c1_complete = true; log_info(scc,init)("Runtime1 Blobs recorded"); } #undef SET_ADDRESS -#undef _extrs_max -#undef _stubs_max -#undef _blobs_max -#undef _shared_blobs_max -#undef _C1_blobs_max -#undef _C2_blobs_max SCAddressTable::~SCAddressTable() { if (_extrs_addr != nullptr) { @@ -3962,7 +4213,11 @@ SCAddressTable::~SCAddressTable() { } } +#ifdef PRODUCT #define MAX_STR_COUNT 200 +#else +#define MAX_STR_COUNT 500 +#endif static const char* _C_strings[MAX_STR_COUNT] = {nullptr}; static int _C_strings_count = 0; static int _C_strings_s[MAX_STR_COUNT] = {0}; @@ -4044,7 +4299,7 @@ void SCCache::add_new_C_string(const char* str) { } void SCAddressTable::add_C_string(const char* str) { - if (str != nullptr && _complete && (_opto_complete || _c1_complete)) { + if (str != nullptr && _extrs_complete) { // Check previous strings address for (int i = 0; i < _C_strings_count; i++) { if (_C_strings[i] == str) { @@ -4110,33 +4365,40 @@ int search_address(address addr, address* table, uint length) { } address SCAddressTable::address_for_id(int idx) { - if (!_complete) { - fatal("SCA table is not complete"); + if (!_extrs_complete) { + fatal("SCA extrs table is not complete"); } if (idx == -1) { return (address)-1; } uint id = (uint)idx; - if (id >= _all_max && idx < (_all_max + _C_strings_count)) { - return address_for_C_string(idx - _all_max); + // special case for symbols based relative to os::init + if (id > (_c_str_base + _c_str_max)) { + return (address)os::init + idx; } - if (idx < 0 || id == (_extrs_length + _stubs_length + _final_blobs_length)) { + if (idx < 0) { fatal("Incorrect id %d for SCA table", id); } - if (idx > (_all_max + _C_strings_count)) { - return (address)os::init + idx; + // no need to compare unsigned id against 0 + if (/* id >= _extrs_base && */ id < _extrs_length) { + return _extrs_addr[id - _extrs_base]; + } + if (id >= _stubs_base && id < _stubs_base + _stubs_length) { + return _stubs_addr[id - _stubs_base]; + } + if (id >= _blobs_base && id < _blobs_base + _blobs_length) { + return _blobs_addr[id - _blobs_base]; } - if (id < _extrs_length) { - return _extrs_addr[id]; + if (id >= _C1_blobs_base && id < _C1_blobs_base + _C1_blobs_length) { + return _C1_blobs_addr[id - _C1_blobs_base]; } - id -= _extrs_length; - if (id < _stubs_length) { - return _stubs_addr[id]; + if (id >= _C2_blobs_base && id < _C2_blobs_base + _C2_blobs_length) { + return _C2_blobs_addr[id - _C2_blobs_base]; } - id -= _stubs_length; - if (id < _final_blobs_length) { - return _blobs_addr[id]; + if (id >= _c_str_base && id < (_c_str_base + (uint)_C_strings_count)) { + return address_for_C_string(id - _c_str_base); } + fatal("Incorrect id %d for SCA table", id); return nullptr; } @@ -4145,13 +4407,13 @@ int SCAddressTable::id_for_address(address addr, RelocIterator reloc, CodeBuffer if (addr == (address)-1) { // Static call stub has jump to itself return id; } - if (!_complete) { + if (!_extrs_complete) { fatal("SCA table is not complete"); } // Seach for C string id = id_for_C_string(addr); if (id >=0) { - return id + _all_max; + return id + _c_str_base; } if (StubRoutines::contains(addr)) { // Search in stubs @@ -4164,17 +4426,28 @@ int SCAddressTable::id_for_address(address addr, RelocIterator reloc, CodeBuffer const char* sub_name = (desc != nullptr) ? desc->name() : ""; fatal("Address " INTPTR_FORMAT " for Stub:%s is missing in SCA table", p2i(addr), sub_name); } else { - id += _extrs_length; + return _stubs_base + id; } } else { CodeBlob* cb = CodeCache::find_blob(addr); if (cb != nullptr) { + int id_base = _blobs_base; // Search in code blobs - id = search_address(addr, _blobs_addr, _final_blobs_length); + id = search_address(addr, _blobs_addr, _blobs_length); + if (id == -1) { + id_base = _C1_blobs_base; + // search C1 blobs + id = search_address(addr, _C1_blobs_addr, _C1_blobs_length); + } + if (id == -1) { + id_base = _C2_blobs_base; + // search C2 blobs + id = search_address(addr, _C2_blobs_addr, _C2_blobs_length); + } if (id < 0) { fatal("Address " INTPTR_FORMAT " for Blob:%s is missing in SCA table", p2i(addr), cb->name()); } else { - id += _extrs_length + _stubs_length; + return id_base + id; } } else { // Search in runtime functions @@ -4210,12 +4483,27 @@ int SCAddressTable::id_for_address(address addr, RelocIterator reloc, CodeBuffer #endif // !PRODUCT fatal("Address " INTPTR_FORMAT " for is missing in SCA table", p2i(addr)); } + } else { + return _extrs_base + id; } } } return id; } +#undef _extrs_max +#undef _stubs_max +#undef _all_blobs_max +#undef _blobs_max +#undef _C1_blobs_max +#undef _C2_blobs_max +#undef _extrs_base +#undef _stubs_base +#undef _blobs_base +#undef _C1_blobs_base +#undef _C2_blobs_base +#undef _c_str_base + void AOTRuntimeConstants::initialize_from_runtime() { BarrierSet* bs = BarrierSet::barrier_set(); if (bs->is_a(BarrierSet::CardTableBarrierSet)) { diff --git a/src/hotspot/share/code/SCCache.hpp b/src/hotspot/share/code/SCCache.hpp index d818a007d16..ea4b4840ab2 100644 --- a/src/hotspot/share/code/SCCache.hpp +++ b/src/hotspot/share/code/SCCache.hpp @@ -145,9 +145,10 @@ class SCCEntry { public: enum Kind { None = 0, - Stub = 1, - Blob = 2, - Code = 3 + Adapter = 1, + Stub = 2, + Blob = 3, + Code = 4 }; private: @@ -271,8 +272,10 @@ class SCAddressTable : public CHeapObj { uint _blobs_length; uint _C1_blobs_length; uint _C2_blobs_length; - uint _final_blobs_length; + bool _extrs_complete; + bool _early_stubs_complete; + bool _shared_blobs_complete; bool _complete; bool _opto_complete; bool _c1_complete; @@ -282,12 +285,18 @@ class SCAddressTable : public CHeapObj { _extrs_addr = nullptr; _stubs_addr = nullptr; _blobs_addr = nullptr; + _extrs_complete = false; + _early_stubs_complete = false; + _shared_blobs_complete = false; _complete = false; _opto_complete = false; _c1_complete = false; } ~SCAddressTable(); - void init(); + void init_extrs(); + void init_early_stubs(); + void init_shared_blobs(); + void init_stubs(); void init_opto(); void init_c1(); void add_C_string(const char* str); @@ -351,6 +360,8 @@ class SCCReader { // Concurent per compilation request bool compile(ciEnv* env, ciMethod* target, int entry_bci, AbstractCompiler* compiler); bool compile_blob(CodeBuffer* buffer, int* pc_offset); + bool compile_adapter(CodeBuffer* buffer, const char* name, uint32_t offsets[4]); + Klass* read_klass(const methodHandle& comp_method, bool shared); Method* read_method(const methodHandle& comp_method, bool shared); @@ -410,6 +421,10 @@ class SCCache : public CHeapObj { uint write_bytes(const void* buffer, uint nbytes); const char* addr(uint offset) const { return _load_buffer + offset; } + static SCAddressTable* addr_table() { + return is_on() && (cache()->_table != nullptr) ? cache()->_table : nullptr; + } + bool _lookup_failed; // Failed to lookup for info (skip only this code load) void set_lookup_failed() { _lookup_failed = true; } void clear_lookup_failed() { _lookup_failed = false; } @@ -470,7 +485,10 @@ class SCCache : public CHeapObj { void load_strings(); int store_strings(); - static void init_table(); + static void init_extrs_table(); + static void init_early_stubs_table(); + static void init_shared_blobs_table(); + static void init_stubs_table(); static void init_opto_table(); static void init_c1_table(); address address_for_id(int id) const { return _table->address_for_id(id); } @@ -520,6 +538,9 @@ class SCCache : public CHeapObj { static bool load_exception_blob(CodeBuffer* buffer, int* pc_offset); static bool store_exception_blob(CodeBuffer* buffer, int pc_offset); + static bool load_adapter(CodeBuffer* buffer, uint32_t id, const char* basic_sig, uint32_t offsets[4]); + static bool store_adapter(CodeBuffer* buffer, uint32_t id, const char* basic_sig, uint32_t offsets[4]); + static bool load_nmethod(ciEnv* env, ciMethod* target, int entry_bci, AbstractCompiler* compiler, CompLevel comp_level); static SCCEntry* store_nmethod(const methodHandle& method, diff --git a/src/hotspot/share/compiler/compileBroker.cpp b/src/hotspot/share/compiler/compileBroker.cpp index 48ff24b7a03..e539d4dbc48 100644 --- a/src/hotspot/share/compiler/compileBroker.cpp +++ b/src/hotspot/share/compiler/compileBroker.cpp @@ -2060,11 +2060,6 @@ bool CompileBroker::init_compiler_runtime() { // Switch back to VM state to do compiler initialization ThreadInVMfromNative tv(thread); - // Perform per-thread and global initializations - { - MutexLocker only_one (thread, CompileThread_lock); - SCCache::init_table(); - } comp->initialize(); } diff --git a/src/hotspot/share/memory/allocation.hpp b/src/hotspot/share/memory/allocation.hpp index 500b4d29f16..7ee0ff0c47b 100644 --- a/src/hotspot/share/memory/allocation.hpp +++ b/src/hotspot/share/memory/allocation.hpp @@ -318,7 +318,9 @@ class MetaspaceObj { f(KlassTrainingData) \ f(MethodTrainingData) \ f(CompileTrainingData) \ - f(SharedClassPathEntry) + f(SharedClassPathEntry) \ + f(AdapterHandlerEntry) \ + f(AdapterFingerPrint) #define METASPACE_OBJ_TYPE_DECLARE(name) name ## Type, #define METASPACE_OBJ_TYPE_NAME_CASE(name) case name ## Type: return #name; diff --git a/src/hotspot/share/oops/method.cpp b/src/hotspot/share/oops/method.cpp index c11342563a1..89f4c7f2c90 100644 --- a/src/hotspot/share/oops/method.cpp +++ b/src/hotspot/share/oops/method.cpp @@ -399,6 +399,7 @@ void Method::metaspace_pointers_do(MetaspaceClosure* it) { } else { it->push(&_constMethod); } + it->push(&_adapter); it->push(&_method_data); it->push(&_method_counters); NOT_PRODUCT(it->push(&_name);) @@ -412,6 +413,9 @@ void Method::metaspace_pointers_do(MetaspaceClosure* it) { void Method::remove_unshareable_info() { unlink_method(); + if (CDSConfig::is_dumping_adapters() && _adapter != nullptr) { + _adapter->remove_unshareable_info(); + } if (method_data() != nullptr) { method_data()->remove_unshareable_info(); } @@ -421,8 +425,16 @@ void Method::remove_unshareable_info() { JFR_ONLY(REMOVE_METHOD_ID(this);) } +void Method::restore_adapter(TRAPS) { + if (_adapter != nullptr) { + _adapter->restore_unshareable_info(CHECK); + _from_compiled_entry = _adapter->get_c2i_entry(); + } +} + void Method::restore_unshareable_info(TRAPS) { assert(is_method() && is_valid_method(this), "ensure C++ vtable is restored"); + restore_adapter(CHECK); if (method_data() != nullptr) { method_data()->restore_unshareable_info(CHECK); } @@ -1212,7 +1224,9 @@ void Method::unlink_code() { void Method::unlink_method() { assert(CDSConfig::is_dumping_archive(), "sanity"); _code = nullptr; - _adapter = nullptr; + if (!CDSConfig::is_dumping_adapters() || AdapterHandlerLibrary::is_abstract_method_adapter(_adapter)) { + _adapter = nullptr; + } _i2i_entry = nullptr; _from_compiled_entry = nullptr; _from_interpreted_entry = nullptr; @@ -1261,7 +1275,7 @@ void Method::link_method(const methodHandle& h_method, TRAPS) { // If the code cache is full, we may reenter this function for the // leftover methods that weren't linked. - if (adapter() != nullptr) { + if (adapter() != nullptr && !adapter()->is_shared()) { return; } assert( _code == nullptr, "nothing compiled yet" ); @@ -1269,7 +1283,7 @@ void Method::link_method(const methodHandle& h_method, TRAPS) { // Setup interpreter entrypoint assert(this == h_method(), "wrong h_method()" ); - assert(adapter() == nullptr, "init'd to null"); + assert(adapter() == nullptr || adapter()->is_linked(), "init'd to null or restored from cache"); address entry = Interpreter::entry_for_method(h_method); assert(entry != nullptr, "interpreter entry must be non-null"); // Sets both _i2i_entry and _from_interpreted_entry @@ -1290,7 +1304,9 @@ void Method::link_method(const methodHandle& h_method, TRAPS) { // called from the vtable. We need adapters on such methods that get loaded // later. Ditto for mega-morphic itable calls. If this proves to be a // problem we'll make these lazily later. - (void) make_adapters(h_method, CHECK); + if (_adapter == nullptr) { + (void) make_adapters(h_method, CHECK); + } // ONLY USE the h_method now as make_adapter may have blocked @@ -1571,6 +1587,11 @@ methodHandle Method::make_method_handle_intrinsic(vmIntrinsics::ID iid, #if INCLUDE_CDS void Method::restore_archived_method_handle_intrinsic(methodHandle m, TRAPS) { + m->restore_adapter(CHECK); + if (m->adapter() != nullptr) { + m->adapter()->restore_unshareable_info(CHECK); + m->set_from_compiled_entry(m->adapter()->get_c2i_entry()); + } m->link_method(m, CHECK); if (m->intrinsic_id() == vmIntrinsics::_linkToNative) { diff --git a/src/hotspot/share/oops/method.hpp b/src/hotspot/share/oops/method.hpp index 446e0f4bb13..84345ed12c2 100644 --- a/src/hotspot/share/oops/method.hpp +++ b/src/hotspot/share/oops/method.hpp @@ -126,6 +126,7 @@ class Method : public Metadata { #if INCLUDE_CDS void remove_unshareable_info(); + void restore_adapter(TRAPS); void restore_unshareable_info(TRAPS); static void restore_archived_method_handle_intrinsic(methodHandle m, TRAPS); #endif diff --git a/src/hotspot/share/oops/trainingData.cpp b/src/hotspot/share/oops/trainingData.cpp index 6ddc8603a4d..47012ed5cba 100644 --- a/src/hotspot/share/oops/trainingData.cpp +++ b/src/hotspot/share/oops/trainingData.cpp @@ -579,6 +579,8 @@ void MethodTrainingData::cleanup(Visitor& visitor) { if (_final_profile != nullptr && _final_profile->method() != _holder) { log_warning(cds)("Stale MDO for %s::%s", name()->as_klass_external_name(), signature()->as_utf8()); } + _final_profile = nullptr; + _final_counters = nullptr; _holder = nullptr; key()->make_empty(); } diff --git a/src/hotspot/share/runtime/globals.hpp b/src/hotspot/share/runtime/globals.hpp index 596808b88b5..e1d0f469d57 100644 --- a/src/hotspot/share/runtime/globals.hpp +++ b/src/hotspot/share/runtime/globals.hpp @@ -2026,6 +2026,9 @@ const int ObjectAlignmentInBytes = 8; product(bool, StressSecondarySupers, false, DIAGNOSTIC, \ "Use a terrible hash function in order to generate many collisions.") \ \ + develop(bool, TestAdapterLinkFailure, false, \ + "Test failure of adapter linking when loading from AOT cache.") \ + \ product(bool, UseThreadsLockThrottleLock, true, DIAGNOSTIC, \ "Use an extra lock during Thread start and exit to alleviate" \ "contention on Threads_lock.") \ diff --git a/src/hotspot/share/runtime/init.cpp b/src/hotspot/share/runtime/init.cpp index 2e6df97fa0b..ed0732d4cd4 100644 --- a/src/hotspot/share/runtime/init.cpp +++ b/src/hotspot/share/runtime/init.cpp @@ -174,6 +174,8 @@ jint init_globals() { InterfaceSupport_init(); VMRegImpl::set_regName(); // need this before generate_stubs (for printing oop maps). SharedRuntime::generate_stubs(); + SCCache::init_shared_blobs_table(); // need this after generate_stubs + SharedRuntime::init_adapter_library(); // do this after SCCache::init_shared_blobs_table return JNI_OK; } @@ -225,6 +227,7 @@ jint init_globals2() { } compiler_stubs_init(false /* in_compiler_thread */); // compiler's intrinsics stubs final_stubs_init(); // final StubRoutines stubs + SCCache::init_stubs_table(); MethodHandles::generate_adapters(); // All the flags that get adjusted by VM_Version_init and os::init_2 diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 285941bad47..7974324543c 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -23,11 +23,15 @@ */ #include "precompiled.hpp" +#include "cds/archiveBuilder.hpp" +#include "cds/archiveUtils.inline.hpp" +#include "cds/cdsConfig.hpp" #include "classfile/classLoader.hpp" #include "classfile/javaClasses.inline.hpp" #include "classfile/stringTable.hpp" #include "classfile/vmClasses.hpp" #include "classfile/vmSymbols.hpp" +#include "code/SCCache.hpp" #include "code/codeCache.hpp" #include "code/compiledIC.hpp" #include "code/nmethod.inline.hpp" @@ -156,8 +160,6 @@ void SharedRuntime::generate_stubs() { generate_throw_exception(SharedStubId::throw_NullPointerException_at_call_id, CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call)); - AdapterHandlerLibrary::initialize(); - #if COMPILER2_OR_JVMCI // Vectors are generated only by C2 and JVMCI. bool support_wide = is_wide_vector(MaxVectorSize); @@ -189,6 +191,10 @@ void SharedRuntime::generate_stubs() { } } +void SharedRuntime::init_adapter_library() { + AdapterHandlerLibrary::initialize(); +} + static void print_counter_on(outputStream* st, const char* name, PerfTickCounters* counter, uint cnt) { st->print(" %-28s " JLONG_FORMAT_W(6) "us", name, counter->elapsed_counter_value_us()); if (TraceThreadTime) { @@ -2256,13 +2262,14 @@ void SharedRuntime::print_call_statistics_on(outputStream* st) { #ifndef PRODUCT static int _lookups; // number of calls to lookup static int _equals; // number of buckets checked with matching hash -static int _hits; // number of successful lookups +static int _archived_hits; // number of successful lookups in archived table +static int _runtime_hits; // number of successful lookups in runtime table static int _compact; // number of equals calls with compact signature #endif // A simple wrapper class around the calling convention information // that allows sharing of adapters for the same calling convention. -class AdapterFingerPrint : public CHeapObj { +class AdapterFingerPrint : public MetaspaceObj { private: enum { _basic_type_bits = 4, @@ -2273,12 +2280,29 @@ class AdapterFingerPrint : public CHeapObj { // TO DO: Consider integrating this with a more global scheme for compressing signatures. // For now, 4 bits per components (plus T_VOID gaps after double/long) is not excessive. - union { - int _compact[_compact_int_count]; - int* _fingerprint; - } _value; - int _length; // A negative length indicates the fingerprint is in the compact form, - // Otherwise _value._fingerprint is the array. + int _length; + int _value[_compact_int_count]; + + // Private construtor. Use allocate() to get an instance. + AdapterFingerPrint(int total_args_passed, BasicType* sig_bt) { + // Pack the BasicTypes with 8 per int + _length = (total_args_passed + (_basic_types_per_int-1)) / _basic_types_per_int; + int sig_index = 0; + for (int index = 0; index < _length; index++) { + int value = 0; + for (int byte = 0; sig_index < total_args_passed && byte < _basic_types_per_int; byte++) { + int bt = adapter_encoding(sig_bt[sig_index++]); + assert((bt & _basic_type_mask) == bt, "must fit in 4 bits"); + value = (value << _basic_type_bits) | bt; + } + _value[index] = value; + } + } + + // Call deallocate instead + ~AdapterFingerPrint() { + FreeHeap(this); + } // Remap BasicTypes that are handled equivalently by the adapters. // These are correct for the current system but someday it might be @@ -2315,64 +2339,63 @@ class AdapterFingerPrint : public CHeapObj { } } - public: - AdapterFingerPrint(int total_args_passed, BasicType* sig_bt) { - // The fingerprint is based on the BasicType signature encoded - // into an array of ints with eight entries per int. - int* ptr; - int len = (total_args_passed + (_basic_types_per_int-1)) / _basic_types_per_int; - if (len <= _compact_int_count) { - assert(_compact_int_count == 3, "else change next line"); - _value._compact[0] = _value._compact[1] = _value._compact[2] = 0; - // Storing the signature encoded as signed chars hits about 98% - // of the time. - _length = -len; - ptr = _value._compact; - } else { - _length = len; - _value._fingerprint = NEW_C_HEAP_ARRAY(int, _length, mtCode); - ptr = _value._fingerprint; - } + void* operator new(size_t size, size_t fp_size) throw() { + assert(fp_size >= size, "sanity check"); + void* p = AllocateHeap(fp_size, mtCode); + memset(p, 0, fp_size); + return p; + } - // Now pack the BasicTypes with 8 per int - int sig_index = 0; - for (int index = 0; index < len; index++) { - int value = 0; - for (int byte = 0; sig_index < total_args_passed && byte < _basic_types_per_int; byte++) { - int bt = adapter_encoding(sig_bt[sig_index++]); - assert((bt & _basic_type_mask) == bt, "must fit in 4 bits"); - value = (value << _basic_type_bits) | bt; + template + void iterate_args(Function function) { + for (int i = 0; i < length(); i++) { + unsigned val = (unsigned)value(i); + // args are packed so that first/lower arguments are in the highest + // bits of each int value, so iterate from highest to the lowest + for (int j = 32 - _basic_type_bits; j >= 0; j -= _basic_type_bits) { + unsigned v = (val >> j) & _basic_type_mask; + if (v == 0) { + continue; + } + function(v); } - ptr[index] = value; } } - ~AdapterFingerPrint() { - if (_length > 0) { - FREE_C_HEAP_ARRAY(int, _value._fingerprint); - } + public: + static int allocation_size(int total_args_passed, BasicType* sig_bt) { + int len = (total_args_passed + (_basic_types_per_int-1)) / _basic_types_per_int; + return sizeof(AdapterFingerPrint) + (len > _compact_int_count ? (len - _compact_int_count) * sizeof(int) : 0); + } + + static AdapterFingerPrint* allocate(int total_args_passed, BasicType* sig_bt) { + int size_in_bytes = allocation_size(total_args_passed, sig_bt); + return new (size_in_bytes) AdapterFingerPrint(total_args_passed, sig_bt); + } + + static void deallocate(AdapterFingerPrint* fp) { + fp->~AdapterFingerPrint(); } int value(int index) { - if (_length < 0) { - return _value._compact[index]; - } - return _value._fingerprint[index]; + return _value[index]; } + int length() { if (_length < 0) return -_length; return _length; } bool is_compact() { - return _length <= 0; + return _length <= _compact_int_count; } unsigned int compute_hash() { int hash = 0; for (int i = 0; i < length(); i++) { int v = value(i); - hash = (hash << 8) ^ v ^ (hash >> 5); + //Add arithmetic operation to the hash, like +3 to improve hashing + hash = ((hash << 8) ^ v ^ (hash >> 5)) + 3; } return (unsigned int)hash; } @@ -2386,59 +2409,90 @@ class AdapterFingerPrint : public CHeapObj { return st.as_string(); } -#ifndef PRODUCT - // Reconstitutes the basic type arguments from the fingerprint, - // producing strings like LIJDF const char* as_basic_args_string() { stringStream st; bool long_prev = false; - for (int i = 0; i < length(); i++) { - unsigned val = (unsigned)value(i); - // args are packed so that first/lower arguments are in the highest - // bits of each int value, so iterate from highest to the lowest - for (int j = 32 - _basic_type_bits; j >= 0; j -= _basic_type_bits) { - unsigned v = (val >> j) & _basic_type_mask; - if (v == 0) { - assert(i == length() - 1, "Only expect zeroes in the last word"); - continue; - } - if (long_prev) { - long_prev = false; - if (v == T_VOID) { - st.print("J"); - } else { - st.print("L"); - } - } - switch (v) { - case T_INT: st.print("I"); break; - case T_LONG: long_prev = true; break; - case T_FLOAT: st.print("F"); break; - case T_DOUBLE: st.print("D"); break; - case T_VOID: break; - default: ShouldNotReachHere(); + iterate_args([&] (int arg) { + if (long_prev) { + long_prev = false; + if (arg == T_VOID) { + st.print("J"); + } else { + st.print("L"); } } - } + switch (arg) { + case T_INT: st.print("I"); break; + case T_LONG: long_prev = true; break; + case T_FLOAT: st.print("F"); break; + case T_DOUBLE: st.print("D"); break; + case T_VOID: break; + default: ShouldNotReachHere(); + } + }); if (long_prev) { st.print("L"); } return st.as_string(); } -#endif // !product + + BasicType* as_basic_type(int& nargs) { + nargs = 0; + GrowableArray btarray; + bool long_prev = false; + + iterate_args([&] (int arg) { + if (long_prev) { + long_prev = false; + if (arg == T_VOID) { + btarray.append(T_LONG); + } else { + btarray.append(T_OBJECT); // it could be T_ARRAY; it shouldn't matter + } + } + switch (arg) { + case T_INT: // fallthrough + case T_FLOAT: // fallthrough + case T_DOUBLE: + case T_VOID: + btarray.append((BasicType)arg); + break; + case T_LONG: + long_prev = true; + break; + default: ShouldNotReachHere(); + } + }); + + if (long_prev) { + btarray.append(T_OBJECT); + } + + nargs = btarray.length(); + BasicType* sig_bt = NEW_RESOURCE_ARRAY(BasicType, nargs); + int index = 0; + GrowableArrayIterator iter = btarray.begin(); + while (iter != btarray.end()) { + sig_bt[index++] = *iter; + ++iter; + } + assert(index == btarray.length(), "sanity check"); +#ifdef ASSERT + { + AdapterFingerPrint* compare_fp = AdapterFingerPrint::allocate(nargs, sig_bt); + assert(this->equals(compare_fp), "sanity check"); + AdapterFingerPrint::deallocate(compare_fp); + } +#endif + return sig_bt; + } bool equals(AdapterFingerPrint* other) { if (other->_length != _length) { return false; - } - if (_length < 0) { - assert(_compact_int_count == 3, "else change next line"); - return _value._compact[0] == other->_value._compact[0] && - _value._compact[1] == other->_value._compact[1] && - _value._compact[2] == other->_value._compact[2]; } else { for (int i = 0; i < _length; i++) { - if (_value._fingerprint[i] != other->_value._fingerprint[i]) { + if (_value[i] != other->_value[i]) { return false; } } @@ -2446,6 +2500,11 @@ class AdapterFingerPrint : public CHeapObj { return true; } + // methods required by virtue of being a MetaspaceObj + void metaspace_pointers_do(MetaspaceClosure* it) { return; /* nothing to do here */ } + int size() const { return (int)heap_word_size(sizeof(AdapterFingerPrint) + (_length > _compact_int_count ? (_length - _compact_int_count) * sizeof(int) : 0)); } + MetaspaceObj::Type type() const { return AdapterFingerPrintType; } + static bool equals(AdapterFingerPrint* const& fp1, AdapterFingerPrint* const& fp2) { NOT_PRODUCT(_equals++); return fp1->equals(fp2); @@ -2456,25 +2515,54 @@ class AdapterFingerPrint : public CHeapObj { } }; +#if INCLUDE_CDS +static inline bool adapter_fp_equals_compact_hashtable_entry(AdapterHandlerEntry* entry, AdapterFingerPrint* fp, int len_unused) { + return AdapterFingerPrint::equals(entry->fingerprint(), fp); +} + +class ArchivedAdapterTable : public OffsetCompactHashtable< + AdapterFingerPrint*, + AdapterHandlerEntry*, + adapter_fp_equals_compact_hashtable_entry> {}; +#endif // INCLUDE_CDS + // A hashtable mapping from AdapterFingerPrints to AdapterHandlerEntries using AdapterHandlerTable = ResourceHashtable; static AdapterHandlerTable* _adapter_handler_table; +static GrowableArray* _adapter_handler_list = nullptr; // Find a entry with the same fingerprint if it exists -static AdapterHandlerEntry* lookup(int total_args_passed, BasicType* sig_bt) { +AdapterHandlerEntry* AdapterHandlerLibrary::lookup(AdapterFingerPrint* fp) { NOT_PRODUCT(_lookups++); - assert_lock_strong(AdapterHandlerLibrary_lock); - AdapterFingerPrint fp(total_args_passed, sig_bt); - AdapterHandlerEntry** entry = _adapter_handler_table->get(&fp); + AdapterHandlerEntry* entry = nullptr; +#if INCLUDE_CDS + // Search archived table first. It is read-only table so can be searched without lock + entry = _archived_adapter_handler_table.lookup(fp, fp->compute_hash(), 0 /* unused */); if (entry != nullptr) { #ifndef PRODUCT - if (fp.is_compact()) _compact++; - _hits++; + if (fp->is_compact()) { + _compact++; + } + _archived_hits++; +#endif + return entry; + } +#endif // INCLUDE_CDS + assert_lock_strong(AdapterHandlerLibrary_lock); + AdapterHandlerEntry** entry_p = _adapter_handler_table->get(fp); + if (entry_p != nullptr) { + entry = *entry_p; + assert(entry->fingerprint()->equals(fp), "fingerprint mismatch key fp %s %s (hash=%d) != found fp %s %s (hash=%d)", + entry->fingerprint()->as_basic_args_string(), entry->fingerprint()->as_string(), entry->fingerprint()->compute_hash(), + fp->as_basic_args_string(), fp->as_string(), fp->compute_hash()); +#ifndef PRODUCT + if (fp->is_compact()) _compact++; + _runtime_hits++; #endif - return *entry; + return entry; } return nullptr; } @@ -2488,8 +2576,9 @@ void AdapterHandlerLibrary::print_statistics_on(outputStream* st) { ts.print(st, "AdapterHandlerTable"); st->print_cr("AdapterHandlerTable (table_size=%d, entries=%d)", _adapter_handler_table->table_size(), _adapter_handler_table->number_of_entries()); - st->print_cr("AdapterHandlerTable: lookups %d equals %d hits %d compact %d", - _lookups, _equals, _hits, _compact); + int total_hits = _archived_hits + _runtime_hits; + st->print_cr("AdapterHandlerTable: lookups %d equals %d hits %d (archived=%d+runtime=%d) compact %d", + _lookups, _equals, total_hits, _archived_hits, _runtime_hits, _compact); } #endif // !PRODUCT @@ -2501,6 +2590,9 @@ AdapterHandlerEntry* AdapterHandlerLibrary::_int_arg_handler = nullptr; AdapterHandlerEntry* AdapterHandlerLibrary::_obj_arg_handler = nullptr; AdapterHandlerEntry* AdapterHandlerLibrary::_obj_int_arg_handler = nullptr; AdapterHandlerEntry* AdapterHandlerLibrary::_obj_obj_arg_handler = nullptr; +#if INCLUDE_CDS +ArchivedAdapterTable AdapterHandlerLibrary::_archived_adapter_handler_table; +#endif // INCLUDE_CDS const int AdapterHandlerLibrary_size = 16*K; BufferBlob* AdapterHandlerLibrary::_buffer = nullptr; @@ -2544,30 +2636,30 @@ void AdapterHandlerLibrary::initialize() { // Pass wrong_method_abstract for the c2i transitions to return // AbstractMethodError for invalid invocations. address wrong_method_abstract = SharedRuntime::get_handle_wrong_method_abstract_stub(); - _abstract_method_handler = AdapterHandlerLibrary::new_entry(new AdapterFingerPrint(0, nullptr), + _abstract_method_handler = AdapterHandlerLibrary::new_entry(AdapterFingerPrint::allocate(0, nullptr), SharedRuntime::throw_AbstractMethodError_entry(), wrong_method_abstract, wrong_method_abstract); _buffer = BufferBlob::create("adapters", AdapterHandlerLibrary_size); - _no_arg_handler = create_adapter(no_arg_blob, 0, nullptr, true); + _no_arg_handler = create_simple_adapter(no_arg_blob, 0, nullptr); BasicType obj_args[] = { T_OBJECT }; - _obj_arg_handler = create_adapter(obj_arg_blob, 1, obj_args, true); + _obj_arg_handler = create_simple_adapter(obj_arg_blob, 1, obj_args); BasicType int_args[] = { T_INT }; - _int_arg_handler = create_adapter(int_arg_blob, 1, int_args, true); + _int_arg_handler = create_simple_adapter(int_arg_blob, 1, int_args); BasicType obj_int_args[] = { T_OBJECT, T_INT }; - _obj_int_arg_handler = create_adapter(obj_int_arg_blob, 2, obj_int_args, true); + _obj_int_arg_handler = create_simple_adapter(obj_int_arg_blob, 2, obj_int_args); BasicType obj_obj_args[] = { T_OBJECT, T_OBJECT }; - _obj_obj_arg_handler = create_adapter(obj_obj_arg_blob, 2, obj_obj_args, true); + _obj_obj_arg_handler = create_simple_adapter(obj_obj_arg_blob, 2, obj_obj_args); assert(no_arg_blob != nullptr && - obj_arg_blob != nullptr && - int_arg_blob != nullptr && - obj_int_arg_blob != nullptr && - obj_obj_arg_blob != nullptr, "Initial adapters must be properly created"); + obj_arg_blob != nullptr && + int_arg_blob != nullptr && + obj_int_arg_blob != nullptr && + obj_obj_arg_blob != nullptr, "Initial adapters must be properly created"); } // Outside of the lock @@ -2578,14 +2670,35 @@ void AdapterHandlerLibrary::initialize() { post_adapter_creation(obj_obj_arg_blob, _obj_obj_arg_handler); } +AdapterHandlerEntry* AdapterHandlerLibrary::create_simple_adapter(AdapterBlob*& adapter_blob, + int total_args_passed, + BasicType* sig_bt) { + AdapterFingerPrint* fp = AdapterFingerPrint::allocate(total_args_passed, sig_bt); + // We may find the adapter in the table if it is loaded from the AOT cache + AdapterHandlerEntry* entry = lookup(fp); + if (entry != nullptr) { + assert(entry->is_shared() && !entry->is_linked(), "Non null AdapterHandlerEntry should be in the AOT cache in unlinked state"); + if (!link_adapter_handler(entry, adapter_blob)) { + if (!generate_adapter_code(adapter_blob, entry, total_args_passed, sig_bt, /* is_transient */ false)) { + return nullptr; + } + } + // AdapterFingerPrint is already in the cache, no need to keep this one + AdapterFingerPrint::deallocate(fp); + } else { + entry = create_adapter(adapter_blob, fp, total_args_passed, sig_bt, /* is_transient */ false); + } + return entry; +} + AdapterHandlerEntry* AdapterHandlerLibrary::new_entry(AdapterFingerPrint* fingerprint, address i2c_entry, address c2i_entry, address c2i_unverified_entry, address c2i_no_clinit_check_entry) { // Insert an entry into the table - return new AdapterHandlerEntry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, - c2i_no_clinit_check_entry); + return AdapterHandlerEntry::allocate(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, + c2i_no_clinit_check_entry); } AdapterHandlerEntry* AdapterHandlerLibrary::get_simple_adapter(const methodHandle& method) { @@ -2695,23 +2808,27 @@ AdapterHandlerEntry* AdapterHandlerLibrary::get_adapter(const methodHandle& meth MutexLocker mu(AdapterHandlerLibrary_lock); // Lookup method signature's fingerprint - entry = lookup(total_args_passed, sig_bt); + AdapterFingerPrint *fp = AdapterFingerPrint::allocate(total_args_passed, sig_bt); + entry = lookup(fp); if (entry != nullptr) { #ifdef ASSERT if (VerifyAdapterSharing) { AdapterBlob* comparison_blob = nullptr; - AdapterHandlerEntry* comparison_entry = create_adapter(comparison_blob, total_args_passed, sig_bt, false); + AdapterFingerPrint* comparison_fp = AdapterFingerPrint::allocate(total_args_passed, sig_bt); + AdapterHandlerEntry* comparison_entry = create_adapter(comparison_blob, comparison_fp, total_args_passed, sig_bt, true); assert(comparison_blob == nullptr, "no blob should be created when creating an adapter for comparison"); assert(comparison_entry->compare_code(entry), "code must match"); + AdapterFingerPrint::deallocate(comparison_fp); // Release the one just created and return the original - delete comparison_entry; + AdapterHandlerEntry::deallocate(comparison_entry); } #endif + AdapterFingerPrint::deallocate(fp); return entry; } - entry = create_adapter(new_adapter, total_args_passed, sig_bt, /* allocate_code_blob */ true); + entry = create_adapter(new_adapter, fp, total_args_passed, sig_bt, /* is_transient */ false); } // Outside of the lock @@ -2721,87 +2838,203 @@ AdapterHandlerEntry* AdapterHandlerLibrary::get_adapter(const methodHandle& meth return entry; } -AdapterHandlerEntry* AdapterHandlerLibrary::create_adapter(AdapterBlob*& new_adapter, - int total_args_passed, - BasicType* sig_bt, - bool allocate_code_blob) { - if (log_is_enabled(Info, perf, class, link)) { - ClassLoader::perf_method_adapters_count()->inc(); +bool AdapterHandlerLibrary::lookup_aot_cache(AdapterHandlerEntry* handler, CodeBuffer* buffer) { + ResourceMark rm; + const char* name = AdapterHandlerLibrary::name(handler->fingerprint()); + const uint32_t id = AdapterHandlerLibrary::id(handler->fingerprint()); + uint32_t offsets[4]; + if (SCCache::load_adapter(buffer, id, name, offsets)) { + address i2c_entry = buffer->insts_begin(); + assert(offsets[0] == 0, "sanity check"); + handler->set_entry_points(i2c_entry, i2c_entry + offsets[1], i2c_entry + offsets[2], i2c_entry + offsets[3]); + return true; } + return false; +} - // StubRoutines::_final_stubs_code is initialized after this function can be called. As a result, - // VerifyAdapterCalls and VerifyAdapterSharing can fail if we re-use code that generated prior - // to all StubRoutines::_final_stubs_code being set. Checks refer to runtime range checks generated - // in an I2C stub that ensure that an I2C stub is called from an interpreter frame or stubs. - bool contains_all_checks = StubRoutines::final_stubs_code() != nullptr; +#ifndef PRODUCT +void AdapterHandlerLibrary::print_adapter_handler_info(AdapterHandlerEntry* handler, AdapterBlob* adapter_blob) { + ttyLocker ttyl; + ResourceMark rm; + int insts_size = adapter_blob->code_size(); + handler->print_adapter_on(tty); + tty->print_cr("i2c argument handler for: %s %s (%d bytes generated)", + handler->fingerprint()->as_basic_args_string(), + handler->fingerprint()->as_string(), insts_size); + tty->print_cr("c2i argument handler starts at " INTPTR_FORMAT, p2i(handler->get_c2i_entry())); + if (Verbose || PrintStubCode) { + address first_pc = handler->base_address(); + if (first_pc != nullptr) { + Disassembler::decode(first_pc, first_pc + insts_size, tty, &adapter_blob->asm_remarks()); + tty->cr(); + } + } +} +#endif // PRODUCT - VMRegPair stack_regs[16]; - VMRegPair* regs = (total_args_passed <= 16) ? stack_regs : NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed); +bool AdapterHandlerLibrary::generate_adapter_code(AdapterBlob*& adapter_blob, + AdapterHandlerEntry* handler, + int total_args_passed, + BasicType* sig_bt, + bool is_transient) { + if (log_is_enabled(Info, perf, class, link)) { + ClassLoader::perf_method_adapters_count()->inc(); + } - // Get a description of the compiled java calling convention and the largest used (VMReg) stack slot usage - int comp_args_on_stack = SharedRuntime::java_calling_convention(sig_bt, regs, total_args_passed); BufferBlob* buf = buffer_blob(); // the temporary code buffer in CodeCache CodeBuffer buffer(buf); short buffer_locs[20]; buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs, - sizeof(buffer_locs)/sizeof(relocInfo)); - - // Make a C heap allocated version of the fingerprint to store in the adapter - AdapterFingerPrint* fingerprint = new AdapterFingerPrint(total_args_passed, sig_bt); - MacroAssembler _masm(&buffer); - AdapterHandlerEntry* entry = SharedRuntime::generate_i2c2i_adapters(&_masm, - total_args_passed, - comp_args_on_stack, - sig_bt, - regs, - fingerprint); + sizeof(buffer_locs)/sizeof(relocInfo)); + MacroAssembler masm(&buffer); + VMRegPair stack_regs[16]; + VMRegPair* regs = (total_args_passed <= 16) ? stack_regs : NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed); + // Get a description of the compiled java calling convention and the largest used (VMReg) stack slot usage + int comp_args_on_stack = SharedRuntime::java_calling_convention(sig_bt, regs, total_args_passed); + SharedRuntime::generate_i2c2i_adapters(&masm, + total_args_passed, + comp_args_on_stack, + sig_bt, + regs, + handler); + if (CDSConfig::is_dumping_adapters()) { + // try to save generated code + const char* name = AdapterHandlerLibrary::name(handler->fingerprint()); + const uint32_t id = AdapterHandlerLibrary::id(handler->fingerprint()); + uint32_t offsets[4]; + offsets[0] = 0; + offsets[1] = handler->get_c2i_entry() - handler->get_i2c_entry(); + offsets[2] = handler->get_c2i_unverified_entry() - handler->get_i2c_entry(); + offsets[3] = handler->get_c2i_no_clinit_check_entry() - handler->get_i2c_entry(); + SCCache::store_adapter(&buffer, id, name, offsets); + } #ifdef ASSERT if (VerifyAdapterSharing) { - entry->save_code(buf->code_begin(), buffer.insts_size()); - if (!allocate_code_blob) { - return entry; + handler->save_code(buf->code_begin(), buffer.insts_size()); + if (is_transient) { + return true; } } #endif - new_adapter = AdapterBlob::create(&buffer); - NOT_PRODUCT(int insts_size = buffer.insts_size()); - if (new_adapter == nullptr) { + adapter_blob = AdapterBlob::create(&buffer); + if (adapter_blob == nullptr) { // CodeCache is full, disable compilation // Ought to log this but compile log is only per compile thread // and we're some non descript Java thread. - return nullptr; + return false; } - entry->relocate(new_adapter->content_begin()); + handler->relocate(adapter_blob->content_begin()); #ifndef PRODUCT // debugging support if (PrintAdapterHandlers || PrintStubCode) { - ttyLocker ttyl; - entry->print_adapter_on(tty); - tty->print_cr("i2c argument handler #%d for: %s %s (%d bytes generated)", - _adapter_handler_table->number_of_entries(), fingerprint->as_basic_args_string(), - fingerprint->as_string(), insts_size); - tty->print_cr("c2i argument handler starts at " INTPTR_FORMAT, p2i(entry->get_c2i_entry())); - if (Verbose || PrintStubCode) { - address first_pc = entry->base_address(); - if (first_pc != nullptr) { - Disassembler::decode(first_pc, first_pc + insts_size, tty - NOT_PRODUCT(COMMA &new_adapter->asm_remarks())); - tty->cr(); - } - } + print_adapter_handler_info(handler, adapter_blob); } #endif + return true; +} - // Add the entry only if the entry contains all required checks (see sharedRuntime_xxx.cpp) - // The checks are inserted only if -XX:+VerifyAdapterCalls is specified. - if (contains_all_checks || !VerifyAdapterCalls) { +AdapterHandlerEntry* AdapterHandlerLibrary::create_adapter(AdapterBlob*& adapter_blob, + AdapterFingerPrint* fingerprint, + int total_args_passed, + BasicType* sig_bt, + bool is_transient) { + AdapterHandlerEntry* handler = AdapterHandlerLibrary::new_entry(fingerprint); + if (!generate_adapter_code(adapter_blob, handler, total_args_passed, sig_bt, is_transient)) { + return nullptr; + } + if (!is_transient) { assert_lock_strong(AdapterHandlerLibrary_lock); - _adapter_handler_table->put(fingerprint, entry); + _adapter_handler_table->put(fingerprint, handler); } - return entry; + return handler; +} + +#if INCLUDE_CDS +bool AdapterHandlerLibrary::link_adapter_handler(AdapterHandlerEntry* handler, AdapterBlob*& adapter_blob) { +#ifndef PRODUCT + if (TestAdapterLinkFailure) { + return false; + } +#endif + BufferBlob* buf = buffer_blob(); // the temporary code buffer in CodeCache + CodeBuffer buffer(buf); + short buffer_locs[20]; + buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs, + sizeof(buffer_locs)/sizeof(relocInfo)); + + if (!lookup_aot_cache(handler, &buffer)) { + return false; + } + adapter_blob = AdapterBlob::create(&buffer); + if (adapter_blob == nullptr) { + // CodeCache is full, disable compilation + // Ought to log this but compile log is only per compile thread + // and we're some non descript Java thread. + return false; + } + handler->relocate(adapter_blob->content_begin()); +#ifndef PRODUCT + // debugging support + if (PrintAdapterHandlers || PrintStubCode) { + print_adapter_handler_info(handler, adapter_blob); + } +#endif + return true; +} + +class CopyAdapterTableToArchive : StackObj { +private: + CompactHashtableWriter* _writer; + ArchiveBuilder* _builder; +public: + CopyAdapterTableToArchive(CompactHashtableWriter* writer) : _writer(writer), + _builder(ArchiveBuilder::current()) + {} + + bool do_entry(AdapterFingerPrint* fp, AdapterHandlerEntry* entry) { + LogStreamHandle(Trace, cds) lsh; + if (ArchiveBuilder::current()->has_been_archived((address)entry)) { + assert(ArchiveBuilder::current()->has_been_archived((address)fp), "must be"); + AdapterFingerPrint* buffered_fp = ArchiveBuilder::current()->get_buffered_addr(fp); + assert(buffered_fp != nullptr,"sanity check"); + AdapterHandlerEntry* buffered_entry = ArchiveBuilder::current()->get_buffered_addr(entry); + assert(buffered_entry != nullptr,"sanity check"); + + uint hash = fp->compute_hash(); + u4 delta = _builder->buffer_to_offset_u4((address)buffered_entry); + _writer->add(hash, delta); + if (lsh.is_enabled()) { + address fp_runtime_addr = (address)buffered_fp + ArchiveBuilder::current()->buffer_to_requested_delta(); + address entry_runtime_addr = (address)buffered_entry + ArchiveBuilder::current()->buffer_to_requested_delta(); + log_trace(cds)("Added fp=%p (%s), entry=%p to the archived adater table", buffered_fp, buffered_fp->as_basic_args_string(), buffered_entry); + } + } else { + if (lsh.is_enabled()) { + log_trace(cds)("Skipping adapter handler %p (fp=%s) as it is not archived", entry, fp->as_basic_args_string()); + } + } + return true; + } +}; + +size_t AdapterHandlerLibrary::estimate_size_for_archive() { + return CompactHashtableWriter::estimate_size(_adapter_handler_table->number_of_entries()); +} + +void AdapterHandlerLibrary::archive_adapter_table() { + CompactHashtableStats stats; + CompactHashtableWriter writer(_adapter_handler_table->number_of_entries(), &stats); + CopyAdapterTableToArchive copy(&writer); + _adapter_handler_table->iterate(©); + writer.dump(&_archived_adapter_handler_table, "archived adapter table"); +} + +void AdapterHandlerLibrary::serialize_shared_table_header(SerializeClosure* soc) { + _archived_adapter_handler_table.serialize_header(soc); } +#endif // INCLUDE_CDS address AdapterHandlerEntry::base_address() { address base = _i2c_entry; @@ -2827,12 +3060,66 @@ void AdapterHandlerEntry::relocate(address new_base) { assert(base_address() == new_base, ""); } +void AdapterHandlerEntry::metaspace_pointers_do(MetaspaceClosure* it) { + LogStreamHandle(Trace, cds) lsh; + if (lsh.is_enabled()) { + lsh.print("Iter(AdapterHandlerEntry): %p(%s)", this, _fingerprint->as_basic_args_string()); + lsh.cr(); + } + it->push(&_fingerprint); +} + +#if INCLUDE_CDS +void AdapterHandlerEntry::remove_unshareable_info() { + set_entry_points(nullptr, nullptr, nullptr, nullptr, false); +} + +void AdapterHandlerEntry::restore_unshareable_info(TRAPS) { + PerfTraceElapsedTime timer(ClassLoader::perf_method_adapters_time()); + // A fixed set of simple adapters are eagerly linked during JVM initialization + // in AdapterHandlerTable::initialize(). + // Others may already have been linked because they are shared by other methods. + if (is_linked()) { + return; + } + AdapterBlob* adapter_blob = nullptr; + { + MutexLocker mu(AdapterHandlerLibrary_lock); + assert(_fingerprint != nullptr, "_fingerprint must not be null"); +#ifdef ASSERT + AdapterHandlerEntry* entry = AdapterHandlerLibrary::lookup(_fingerprint); + assert(entry == this, "sanity check"); +#endif + if (!AdapterHandlerLibrary::link_adapter_handler(this, adapter_blob)) { + ResourceMark rm; + log_warning(cds)("Failed to link AdapterHandlerEntry to its code in the AOT code cache"); + int nargs; + BasicType* bt = _fingerprint->as_basic_type(nargs); + if (!AdapterHandlerLibrary::generate_adapter_code(adapter_blob, this, nargs, bt, /* is_transient */ false)) { + if (!is_init_completed()) { + // Don't throw exceptions during VM initialization because java.lang.* classes + // might not have been initialized, causing problems when constructing the + // Java exception object. + vm_exit_during_initialization("Out of space in CodeCache for adapters"); + } else { + THROW_MSG(vmSymbols::java_lang_OutOfMemoryError(), "Out of space in CodeCache for adapters"); + } + } + } + } + // Outside of the lock + if (adapter_blob != nullptr) { + post_adapter_creation(adapter_blob, this); + } + assert(_linked, "AdapterHandlerEntry must now be linked"); +} +#endif // INCLUDE_CDS AdapterHandlerEntry::~AdapterHandlerEntry() { - delete _fingerprint; #ifdef ASSERT FREE_C_HEAP_ARRAY(unsigned char, _saved_code); #endif + FreeHeap(this); } @@ -3137,28 +3424,57 @@ JRT_END bool AdapterHandlerLibrary::contains(const CodeBlob* b) { bool found = false; - auto findblob = [&] (AdapterFingerPrint* key, AdapterHandlerEntry* a) { - return (found = (b == CodeCache::find_blob(a->get_i2c_entry()))); + auto findblob_archived_table = [&] (AdapterHandlerEntry* handler) { + return (found = (b == CodeCache::find_blob(handler->get_i2c_entry()))); }; - assert_locked_or_safepoint(AdapterHandlerLibrary_lock); - _adapter_handler_table->iterate(findblob); + _archived_adapter_handler_table.iterate(findblob_archived_table); + if (!found) { + auto findblob_runtime_table = [&] (AdapterFingerPrint* key, AdapterHandlerEntry* a) { + return (found = (b == CodeCache::find_blob(a->get_i2c_entry()))); + }; + assert_locked_or_safepoint(AdapterHandlerLibrary_lock); + _adapter_handler_table->iterate(findblob_runtime_table); + } return found; } +const char* AdapterHandlerLibrary::name(AdapterFingerPrint* fingerprint) { + return fingerprint->as_basic_args_string(); +} + +uint32_t AdapterHandlerLibrary::id(AdapterFingerPrint* fingerprint) { + unsigned int hash = fingerprint->compute_hash(); + return hash; +} + void AdapterHandlerLibrary::print_handler_on(outputStream* st, const CodeBlob* b) { bool found = false; - auto findblob = [&] (AdapterFingerPrint* key, AdapterHandlerEntry* a) { - if (b == CodeCache::find_blob(a->get_i2c_entry())) { + auto findblob_archived_table = [&] (AdapterHandlerEntry* handler) { + if (b == CodeCache::find_blob(handler->get_i2c_entry())) { found = true; st->print("Adapter for signature: "); - a->print_adapter_on(st); + handler->print_adapter_on(st); return true; } else { return false; // keep looking + } }; - assert_locked_or_safepoint(AdapterHandlerLibrary_lock); - _adapter_handler_table->iterate(findblob); + _archived_adapter_handler_table.iterate(findblob_archived_table); + if (!found) { + auto findblob_runtime_table = [&] (AdapterFingerPrint* key, AdapterHandlerEntry* a) { + if (b == CodeCache::find_blob(a->get_i2c_entry())) { + found = true; + st->print("Adapter for signature: "); + a->print_adapter_on(st); + return true; + } else { + return false; // keep looking + } + }; + assert_locked_or_safepoint(AdapterHandlerLibrary_lock); + _adapter_handler_table->iterate(findblob_runtime_table); + } assert(found, "Should have found handler"); } @@ -3179,6 +3495,13 @@ void AdapterHandlerEntry::print_adapter_on(outputStream* st) const { st->cr(); } +bool AdapterHandlerLibrary::is_abstract_method_adapter(AdapterHandlerEntry* entry) { + if (entry == _abstract_method_handler) { + return true; + } + return false; +} + JRT_LEAF(void, SharedRuntime::enable_stack_reserved_zone(JavaThread* current)) assert(current == JavaThread::current(), "pre-condition"); StackOverflow* overflow_state = current->stack_overflow_state(); diff --git a/src/hotspot/share/runtime/sharedRuntime.hpp b/src/hotspot/share/runtime/sharedRuntime.hpp index ffa14d2e679..a1760228f10 100644 --- a/src/hotspot/share/runtime/sharedRuntime.hpp +++ b/src/hotspot/share/runtime/sharedRuntime.hpp @@ -25,10 +25,12 @@ #ifndef SHARE_RUNTIME_SHAREDRUNTIME_HPP #define SHARE_RUNTIME_SHAREDRUNTIME_HPP +#include "classfile/compactHashtable.hpp" #include "code/codeBlob.hpp" #include "code/vmreg.hpp" #include "interpreter/linkResolver.hpp" #include "memory/allStatic.hpp" +#include "memory/metaspaceClosure.hpp" #include "memory/resourceArea.hpp" #include "runtime/stubDeclarations.hpp" #include "utilities/macros.hpp" @@ -116,6 +118,7 @@ class SharedRuntime: AllStatic { // For c2: call to runtime to return a buffer lease. static RuntimeStub* generate_jfr_return_lease(); #endif + static void init_adapter_library(); static const char *stub_name(SharedStubId id) { assert(id > SharedStubId::NO_STUBID && id < SharedStubId::NUM_STUBIDS, "stub id out of range"); @@ -466,12 +469,12 @@ class SharedRuntime: AllStatic { // pointer as needed. This means the i2c adapter code doesn't need any special // handshaking path with compiled code to keep the stack walking correct. - static AdapterHandlerEntry* generate_i2c2i_adapters(MacroAssembler *_masm, - int total_args_passed, - int max_arg, - const BasicType *sig_bt, - const VMRegPair *regs, - AdapterFingerPrint* fingerprint); + static void generate_i2c2i_adapters(MacroAssembler *_masm, + int total_args_passed, + int max_arg, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterHandlerEntry* handler); static void gen_i2c_adapter(MacroAssembler *_masm, int total_args_passed, @@ -678,7 +681,7 @@ class SharedRuntime: AllStatic { // used by the adapters. The code generation happens here because it's very // similar to what the adapters have to do. -class AdapterHandlerEntry : public CHeapObj { +class AdapterHandlerEntry : public MetaspaceObj { friend class AdapterHandlerLibrary; private: @@ -687,6 +690,7 @@ class AdapterHandlerEntry : public CHeapObj { address _c2i_entry; address _c2i_unverified_entry; address _c2i_no_clinit_check_entry; + bool _linked; #ifdef ASSERT // Captures code and signature used to generate this adapter when @@ -702,7 +706,8 @@ class AdapterHandlerEntry : public CHeapObj { _i2c_entry(i2c_entry), _c2i_entry(c2i_entry), _c2i_unverified_entry(c2i_unverified_entry), - _c2i_no_clinit_check_entry(c2i_no_clinit_check_entry) + _c2i_no_clinit_check_entry(c2i_no_clinit_check_entry), + _linked(false) #ifdef ASSERT , _saved_code_length(0) #endif @@ -711,11 +716,33 @@ class AdapterHandlerEntry : public CHeapObj { ~AdapterHandlerEntry(); public: + static AdapterHandlerEntry* allocate(AdapterFingerPrint* fingerprint, + address i2c_entry, + address c2i_entry, + address c2i_unverified_entry, + address c2i_no_clinit_check_entry) + { + return new (mtCode) AdapterHandlerEntry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); + } + + static void deallocate(AdapterHandlerEntry *handler) { + handler->~AdapterHandlerEntry(); + } + + void set_entry_points(address i2c_entry, address c2i_entry, address c2i_unverified_entry, address c2i_no_clinit_check_entry, bool linked = true) { + _i2c_entry = i2c_entry; + _c2i_entry = c2i_entry; + _c2i_unverified_entry = c2i_unverified_entry; + _c2i_no_clinit_check_entry = c2i_no_clinit_check_entry; + _linked = linked; + } + address get_i2c_entry() const { return _i2c_entry; } address get_c2i_entry() const { return _c2i_entry; } address get_c2i_unverified_entry() const { return _c2i_unverified_entry; } address get_c2i_no_clinit_check_entry() const { return _c2i_no_clinit_check_entry; } + bool is_linked() const { return _linked; } address base_address(); void relocate(address new_base); @@ -729,8 +756,19 @@ class AdapterHandlerEntry : public CHeapObj { //virtual void print_on(outputStream* st) const; DO NOT USE void print_adapter_on(outputStream* st) const; + + void metaspace_pointers_do(MetaspaceClosure* it); + int size() const {return (int)heap_word_size(sizeof(AdapterHandlerEntry)); } + MetaspaceObj::Type type() const { return AdapterHandlerEntryType; } + + void remove_unshareable_info() NOT_CDS_RETURN; + void restore_unshareable_info(TRAPS) NOT_CDS_RETURN; }; +#if INCLUDE_CDS +class ArchivedAdapterTable; +#endif // INCLUDE_CDS + class AdapterHandlerLibrary: public AllStatic { friend class SharedRuntime; private: @@ -741,31 +779,55 @@ class AdapterHandlerLibrary: public AllStatic { static AdapterHandlerEntry* _obj_arg_handler; static AdapterHandlerEntry* _obj_int_arg_handler; static AdapterHandlerEntry* _obj_obj_arg_handler; - +#if INCLUDE_CDS + static ArchivedAdapterTable _archived_adapter_handler_table; +#endif // INCLUDE_CDS static BufferBlob* buffer_blob(); static void initialize(); + static AdapterHandlerEntry* create_simple_adapter(AdapterBlob*& new_adapter, + int total_args_passed, + BasicType* sig_bt); + static AdapterHandlerEntry* get_simple_adapter(const methodHandle& method); + static bool lookup_aot_cache(AdapterHandlerEntry* handler, CodeBuffer* buffer); static AdapterHandlerEntry* create_adapter(AdapterBlob*& new_adapter, + AdapterFingerPrint* fingerprint, int total_args_passed, BasicType* sig_bt, - bool allocate_code_blob); - static AdapterHandlerEntry* get_simple_adapter(const methodHandle& method); + bool is_transient); +#ifndef PRODUCT + static void print_adapter_handler_info(AdapterHandlerEntry* handler, AdapterBlob* adapter_blob); +#endif // PRODUCT public: static AdapterHandlerEntry* new_entry(AdapterFingerPrint* fingerprint, - address i2c_entry, - address c2i_entry, - address c2i_unverified_entry, + address i2c_entry = nullptr, + address c2i_entry = nullptr, + address c2i_unverified_entry = nullptr, address c2i_no_clinit_check_entry = nullptr); static void create_native_wrapper(const methodHandle& method); static AdapterHandlerEntry* get_adapter(const methodHandle& method); + static AdapterHandlerEntry* lookup(AdapterFingerPrint* fp); + static bool generate_adapter_code(AdapterBlob*& adapter_blob, + AdapterHandlerEntry* handler, + int total_args_passed, + BasicType* sig_bt, + bool is_transient); static void print_handler(const CodeBlob* b) { print_handler_on(tty, b); } static void print_handler_on(outputStream* st, const CodeBlob* b); static bool contains(const CodeBlob* b); + static const char* name(AdapterFingerPrint* fingerprint); + static uint32_t id(AdapterFingerPrint* fingerprint); #ifndef PRODUCT static void print_statistics_on(outputStream* st); #endif // PRODUCT + static bool is_abstract_method_adapter(AdapterHandlerEntry* adapter); + + static bool link_adapter_handler(AdapterHandlerEntry* handler, AdapterBlob*& adapter_blob) NOT_CDS_RETURN_(false); + static size_t estimate_size_for_archive() NOT_CDS_RETURN_(0); + static void archive_adapter_table() NOT_CDS_RETURN; + static void serialize_shared_table_header(SerializeClosure* soc) NOT_CDS_RETURN; }; #endif // SHARE_RUNTIME_SHAREDRUNTIME_HPP