From 4aea44c0438e3b635b6bea94869fd92ce96713a9 Mon Sep 17 00:00:00 2001 Date: Wed, 29 Jun 2022 10:00:02 +0800 Subject: Apply TBI to ZGC of JDK17 --- src/hotspot/cpu/aarch64/aarch64.ad | 13 ++- .../cpu/aarch64/c1_LIRAssembler_aarch64.cpp | 5 + .../cpu/aarch64/c1_LIRAssembler_aarch64.hpp | 9 +- .../cpu/aarch64/compiledIC_aarch64.cpp | 9 +- .../gc/shared/barrierSetNMethod_aarch64.cpp | 80 ++++++++++---- .../gc/z/zBarrierSetAssembler_aarch64.cpp | 20 ++++ .../gc/z/zBarrierSetAssembler_aarch64.hpp | 2 + .../cpu/aarch64/gc/z/zGlobals_aarch64.cpp | 102 ++++++++++++++++++ .../cpu/aarch64/gc/z/zGlobals_aarch64.hpp | 5 + src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad | 8 +- .../cpu/aarch64/globalDefinitions_aarch64.hpp | 3 + src/hotspot/cpu/aarch64/globals_aarch64.hpp | 5 +- .../aarch64/jvmciCodeInstaller_aarch64.cpp | 16 ++- .../cpu/aarch64/macroAssembler_aarch64.cpp | 40 +++++-- .../cpu/aarch64/nativeInst_aarch64.cpp | 31 +++++- .../cpu/aarch64/nativeInst_aarch64.hpp | 16 ++- .../cpu/aarch64/vm_version_aarch64.cpp | 28 +++++ src/hotspot/os/linux/gc/z/zNUMA_linux.cpp | 4 + .../gc/z/zPhysicalMemoryBacking_linux.cpp | 61 ++++++++++- src/hotspot/share/asm/codeBuffer.cpp | 3 +- src/hotspot/share/code/relocInfo.hpp | 29 ++++- src/hotspot/share/gc/z/vmStructs_z.cpp | 1 + src/hotspot/share/gc/z/vmStructs_z.hpp | 3 + src/hotspot/share/gc/z/zAddress.cpp | 6 ++ src/hotspot/share/gc/z/zAddress.hpp | 3 + src/hotspot/share/gc/z/zAddress.inline.hpp | 30 ++++-- src/hotspot/share/gc/z/zBarrierSetNMethod.cpp | 7 ++ src/hotspot/share/gc/z/zGlobals.cpp | 1 + src/hotspot/share/gc/z/zGlobals.hpp | 1 + src/hotspot/share/gc/z/zMark.cpp | 7 ++ src/hotspot/share/gc/z/zMarkStackEntry.hpp | 3 +- src/hotspot/share/gc/z/zNMethod.cpp | 65 ++++++++++- src/hotspot/share/gc/z/zNMethod.hpp | 6 +- src/hotspot/share/gc/z/zNMethodData.cpp | 12 +++ src/hotspot/share/gc/z/zNMethodData.hpp | 9 ++ src/hotspot/share/gc/z/zPhysicalMemory.cpp | 54 ++++++++++ src/hotspot/share/gc/z/zVirtualMemory.cpp | 15 +++ .../writers/jfrStreamWriterHost.inline.hpp | 3 + src/hotspot/share/prims/jni.cpp | 6 ++ .../sun/jvm/hotspot/gc/z/ZAddress.java | 15 ++- .../sun/jvm/hotspot/gc/z/ZCollectedHeap.java | 3 + .../sun/jvm/hotspot/gc/z/ZGlobals.java | 4 + .../hotspot/gc/z/ZGlobalsForVMStructs.java | 6 ++ .../classes/sun/jvm/hotspot/gc/z/ZUtils.java | 5 + 44 files changed, 685 insertions(+), 69 deletions(-) diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad index 91ea50c00..b3d89863e 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -1750,7 +1750,11 @@ int MachCallStaticJavaNode::ret_addr_offset() int MachCallDynamicJavaNode::ret_addr_offset() { - return 16; // movz, movk, movk, bl + if (!UseTBI) { + return 16; // movz, movk, movk, bl + } else { + return 20; // movz, movk, movk, movk, bl + } } int MachCallRuntimeNode::ret_addr_offset() { @@ -1768,7 +1772,12 @@ int MachCallRuntimeNode::ret_addr_offset() { if (cb) { return 1 * NativeInstruction::instruction_size; } else { - return 6 * NativeInstruction::instruction_size; + // lea will emit 3 or an extra movk to make 4 instructions. + if (!UseTBI) { + return 6 * NativeInstruction::instruction_size; + } else { + return 7 * NativeInstruction::instruction_size; + } } } diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp index 5ce3ecf9e..c69f8aaf1 100644 --- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp @@ -94,6 +94,11 @@ static void select_different_registers(Register preserve, assert_different_registers(preserve, tmp1, tmp2, tmp3); } +int LIR_Assembler::_call_stub_size = 13 * NativeInstruction::instruction_size; + +void LIR_Assembler::init_for_tbi() { + _call_stub_size = 15 * NativeInstruction::instruction_size; +} bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; } diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp index c31ad9b21..d39f57087 100644 --- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp @@ -70,16 +70,19 @@ friend class ArrayCopyStub; void deoptimize_trap(CodeEmitInfo *info); + // call stub: CompiledStaticCall::to_interp_stub_size() + + // CompiledStaticCall::to_trampoline_stub_size() + static int _call_stub_size; + enum { - // call stub: CompiledStaticCall::to_interp_stub_size() + - // CompiledStaticCall::to_trampoline_stub_size() - _call_stub_size = 13 * NativeInstruction::instruction_size, _exception_handler_size = DEBUG_ONLY(1*K) NOT_DEBUG(175), _deopt_handler_size = 7 * NativeInstruction::instruction_size }; public: + static void init_for_tbi(); + void store_parameter(Register r, int offset_from_esp_in_words); void store_parameter(jint c, int offset_from_esp_in_words); void store_parameter(jobject c, int offset_from_esp_in_words); diff --git a/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp b/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp index e922fc1cd..c6a9d8387 100644 --- a/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp @@ -71,8 +71,13 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) #undef __ int CompiledStaticCall::to_interp_stub_size() { - // isb; movk; movz; movz; movk; movz; movz; br - return 8 * NativeInstruction::instruction_size; + if (!UseTBI) { + // isb; movk; movz; movz; movk; movz; movz; br + return 8 * NativeInstruction::instruction_size; + } else { + // emit_to_interp_stub will emit 2 extra movk instructions. + return 10 * NativeInstruction::instruction_size; + } } int CompiledStaticCall::to_trampoline_stub_size() { diff --git a/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp index 8598fb7e7..4942dca93 100644 --- a/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp @@ -34,25 +34,6 @@ #include "utilities/align.hpp" #include "utilities/debug.hpp" -class NativeNMethodBarrier: public NativeInstruction { - address instruction_address() const { return addr_at(0); } - - int *guard_addr() { - return reinterpret_cast(instruction_address() + 10 * 4); - } - -public: - int get_value() { - return Atomic::load_acquire(guard_addr()); - } - - void set_value(int value) { - Atomic::release_store(guard_addr(), value); - } - - void verify() const; -}; - // Store the instruction bitmask, bits and name for checking the barrier. struct CheckInsn { uint32_t mask; @@ -60,7 +41,20 @@ struct CheckInsn { const char *name; }; -static const struct CheckInsn barrierInsn[] = { +static const struct CheckInsn barrierInsn48[] = { + { 0xff000000, 0x18000000, "ldr (literal)" }, + { 0xfffff0ff, 0xd50330bf, "dmb" }, + { 0xffc00000, 0xb9400000, "ldr"}, + { 0x7f20001f, 0x6b00001f, "cmp"}, + { 0xff00001f, 0x54000000, "b.eq"}, + { 0xff800000, 0xd2800000, "mov"}, + { 0xff800000, 0xf2800000, "movk"}, + { 0xff800000, 0xf2800000, "movk"}, + { 0xfffffc1f, 0xd63f0000, "blr"}, + { 0xfc000000, 0x14000000, "b"} +}; + +static const struct CheckInsn barrierInsn64[] = { { 0xff000000, 0x18000000, "ldr (literal)" }, { 0xfffff0ff, 0xd50330bf, "dmb" }, { 0xffc00000, 0xb9400000, "ldr"}, @@ -69,16 +63,52 @@ static const struct CheckInsn barrierInsn[] = { { 0xff800000, 0xd2800000, "mov"}, { 0xff800000, 0xf2800000, "movk"}, { 0xff800000, 0xf2800000, "movk"}, + { 0xff800000, 0xf2800000, "movk"}, { 0xfffffc1f, 0xd63f0000, "blr"}, { 0xfc000000, 0x14000000, "b"} }; +static const unsigned int barrier_inst_len48 = sizeof(barrierInsn48) / sizeof(struct CheckInsn); +static const unsigned int barrier_inst_len64 = sizeof(barrierInsn64) / sizeof(struct CheckInsn); + +static int get_entry_barrier_size() { + return UseTBI ? (4 * (int)barrier_inst_len64) : (4 * (int)barrier_inst_len48); +} + +class NativeNMethodBarrier: public NativeInstruction { + address instruction_address() const { return addr_at(0); } + + int *guard_addr() { + return reinterpret_cast(instruction_address() + get_entry_barrier_size()); + } + +public: + int get_value() { + return Atomic::load_acquire(guard_addr()); + } + + void set_value(int value) { + Atomic::release_store(guard_addr(), value); + } + + void verify() const; +}; + // The encodings must match the instructions emitted by // BarrierSetAssembler::nmethod_entry_barrier. The matching ignores the specific // register numbers and immediate values in the encoding. void NativeNMethodBarrier::verify() const { + const CheckInsn *barrierInsn; + unsigned int barrier_inst_len; + if (UseTBI) { + barrierInsn = barrierInsn64; + barrier_inst_len = barrier_inst_len64; + } else { + barrierInsn = barrierInsn48; + barrier_inst_len = barrier_inst_len48; + } intptr_t addr = (intptr_t) instruction_address(); - for(unsigned int i = 0; i < sizeof(barrierInsn)/sizeof(struct CheckInsn); i++ ) { + for(unsigned int i = 0; i < barrier_inst_len; i++) { uint32_t inst = *((uint32_t*) addr); if ((inst & barrierInsn[i].mask) != barrierInsn[i].bits) { tty->print_cr("Addr: " INTPTR_FORMAT " Code: 0x%x", addr, inst); @@ -132,10 +162,14 @@ void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) { // not find the expected native instruction at this offset, which needs updating. // Note that this offset is invariant of PreserveFramePointer. -static const int entry_barrier_offset = -4 * 11; +// offset = entry barrier insns (10 or 11, depending on UseTBI) + int32 (guard value) +// @see BarrierSetAssembler::nmethod_entry_barrier +static int entry_barrier_offset() { + return -(get_entry_barrier_size() + 4); +} static NativeNMethodBarrier* native_nmethod_barrier(nmethod* nm) { - address barrier_address = nm->code_begin() + nm->frame_complete_offset() + entry_barrier_offset; + address barrier_address = nm->code_begin() + nm->frame_complete_offset() + entry_barrier_offset(); NativeNMethodBarrier* barrier = reinterpret_cast(barrier_address); debug_only(barrier->verify()); return barrier; diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp index 6b42982ed..cafd4e58f 100644 --- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp @@ -203,6 +203,26 @@ void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_jobject_in_native"); } +static void change_immediate(uint32_t& instr, uint32_t imm, uint32_t start, uint32_t end) { + uint32_t imm_mask = ((1u << start) - 1u) ^ ((1u << (end + 1)) - 1u); + instr &= ~imm_mask; + instr |= imm << start; +} + +void ZBarrierSetAssembler::patch_barrier_relocation(address addr) { + uint32_t* const patch_addr = (uint32_t*)addr; + + // The next 3 insns should be movz, andr, cbnz. + assert(nativeInstruction_at(addr)->is_movz() && + Instruction_aarch64::extract(*(patch_addr + 1), 30, 24) == 0b0001010 && + Instruction_aarch64::extract(*(patch_addr + 2), 31, 24) == 0b10110101, + "wrong insns in barrier patch"); + + change_immediate(*patch_addr, (uint16_t) (ZAddressBadMask >> 48), 5, 20); + OrderAccess::fence(); + ICache::invalidate_word((address)patch_addr); +} + #ifdef COMPILER1 #undef __ diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp index cca873825..1594ffe64 100644 --- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp @@ -77,6 +77,8 @@ public: Register tmp, Label& slowpath); + void patch_barrier_relocation(address addr); + #ifdef COMPILER1 void generate_c1_load_barrier_test(LIR_Assembler* ce, LIR_Opr ref) const; diff --git a/src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.cpp index 6aa6d41fe..031014511 100644 --- a/src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.cpp @@ -34,6 +34,7 @@ #include #endif // LINUX +// Address layout when using multi-mapping // // The heap can have three different layouts, depending on the max heap size. // @@ -142,6 +143,97 @@ // * 63-48 Fixed (16-bits, always zero) // +// Address layout when using TBI (Top Byte Ignore) +// +// The heap can have three different layouts, depending on the max heap size. +// +// Address Space & Pointer Layout 1 +// -------------------------------- +// +// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB) +// . . +// . . +// . . +// +--------------------------------+ 0x0000080000000000 (8TB) +// | Heap | +// +--------------------------------+ 0x0000040000000000 (4TB) +// . . +// +--------------------------------+ 0x0000000000000000 +// +// 6 5 5 4 4 +// 2 9 8 2 1 0 +// ++----+-------------------+-----------------------------------------------+ +// 0|1111|000 00000000 000000|11 11111111 11111111 11111111 11111111 11111111| +// ++----+-------------------+-----------------------------------------------+ +// | | | +// | | * 41-0 Object Offset (42-bits, 4TB address space) +// | | +// | * 58-42 Fixed (18-bits, always zero) +// | +// * 62-59 Metadata Bits (4-bits) 0001 = Marked0 +// 0010 = Marked1 +// 0100 = Remapped +// 1000 = Finalizable +// +// +// Address Space & Pointer Layout 2 +// -------------------------------- +// +// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB) +// . . +// . . +// . . +// +--------------------------------+ 0x0000100000000000 (16TB) +// | Heap | +// +--------------------------------+ 0x0000080000000000 (8TB) +// . . +// +--------------------------------+ 0x0000000000000000 +// +// 6 5 5 4 4 +// 2 9 8 3 2 0 +// ++----+------------------+------------------------------------------------+ +// 0|1111|000 00000000 00000|111 11111111 11111111 11111111 11111111 11111111| +// ++----+------------------+------------------------------------------------+ +// | | | +// | | * 42-0 Object Offset (43-bits, 8TB address space) +// | | +// | * 58-43 Fixed (17-bits, always zero) +// | +// * 62-59 Metadata Bits (4-bits) 0001 = Marked0 +// 0010 = Marked1 +// 0100 = Remapped +// 1000 = Finalizable +// +// +// Address Space & Pointer Layout 3 +// -------------------------------- +// +// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB) +// . . +// . . +// . . +// +--------------------------------+ 0x0000200000000000 (32TB) +// | Heap | +// +--------------------------------+ 0x0000100000000000 (16TB) +// . . +// +--------------------------------+ 0x0000000000000000 +// +// 6 5 5 4 4 +// 2 9 8 4 3 0 +// ++----+-----------------+-------------------------------------------------+ +// 0|1111|000 00000000 0000|1111 11111111 11111111 11111111 11111111 11111111| +// ++----+-----------------+-------------------------------------------------+ +// | | | +// | | * 43-0 Object Offset (44-bits, 16TB address space) +// | | +// | * 58-44 Fixed (17-bits, always zero) +// | +// * 62-59 Metadata Bits (4-bits) 0001 = Marked0 +// 0010 = Marked1 +// 0100 = Remapped +// 1000 = Finalizable +// + // Default value if probing is not implemented for a certain platform: 128TB static const size_t DEFAULT_MAX_ADDRESS_BIT = 47; // Minimum value returned, if probing fails: 64GB @@ -196,6 +288,13 @@ static size_t probe_valid_max_address_bit() { #endif // LINUX } +uintptr_t ZPlatformAddressBase() { + if (UseTBI) { + return (uintptr_t)1 << ZPlatformAddressOffsetBits(); + } + return 0; +} + size_t ZPlatformAddressOffsetBits() { const static size_t valid_max_address_offset_bits = probe_valid_max_address_bit() + 1; const size_t max_address_offset_bits = valid_max_address_offset_bits - 3; @@ -206,5 +305,8 @@ size_t ZPlatformAddressOffsetBits() { } size_t ZPlatformAddressMetadataShift() { + if (UseTBI) { + return ZPlatformAddressMetadataShiftForTbi; + } return ZPlatformAddressOffsetBits(); } diff --git a/src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.hpp index 3187808b6..264b51ec1 100644 --- a/src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.hpp @@ -28,6 +28,11 @@ const size_t ZPlatformGranuleSizeShift = 21; // 2MB const size_t ZPlatformHeapViews = 3; const size_t ZPlatformCacheLineSize = 64; +// The highest bit (bit 63) of the address is occupied by StackWatermarkState (see StackWatermarkState::create). +// So here we use bits 59~62 as the metadata bits of ZGC. +const size_t ZPlatformAddressMetadataShiftForTbi = 59; + +uintptr_t ZPlatformAddressBase(); size_t ZPlatformAddressOffsetBits(); size_t ZPlatformAddressMetadataShift(); diff --git a/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad b/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad index bd1c2cc9f..426a1cc2a 100644 --- a/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad +++ b/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad @@ -26,6 +26,7 @@ source_hpp %{ #include "gc/shared/gc_globals.hpp" #include "gc/z/c2/zBarrierSetC2.hpp" #include "gc/z/zThreadLocalData.hpp" +#include "gc/z/zBarrierSetAssembler.hpp" %} @@ -36,7 +37,12 @@ static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address return; } ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, barrier_data); - __ ldr(tmp, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); + if (UseTBI) { + __ relocate(barrier_Relocation::spec()); + __ movz(tmp, barrier_Relocation::unpatched, 48); + } else { + __ ldr(tmp, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); + } __ andr(tmp, tmp, ref); __ cbnz(tmp, *stub->entry()); __ bind(*stub->continuation()); diff --git a/src/hotspot/cpu/aarch64/globalDefinitions_aarch64.hpp b/src/hotspot/cpu/aarch64/globalDefinitions_aarch64.hpp index 3c779bb11..4a2af011f 100644 --- a/src/hotspot/cpu/aarch64/globalDefinitions_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/globalDefinitions_aarch64.hpp @@ -67,4 +67,7 @@ const bool CCallingConventionRequiresIntsAsLongs = false; #define NOT_R18_RESERVED(code) code #endif +#define CLEAR_TOP_BYTE(addr) ((addr) & ((1UL << 56) - 1)) +#define CLEAR_COLOR_BITS(addr) (UseTBI ? CLEAR_TOP_BYTE(addr) : (addr)) + #endif // CPU_AARCH64_GLOBALDEFINITIONS_AARCH64_HPP diff --git a/src/hotspot/cpu/aarch64/globals_aarch64.hpp b/src/hotspot/cpu/aarch64/globals_aarch64.hpp index fefc2e5c3..cae9fb60c 100644 --- a/src/hotspot/cpu/aarch64/globals_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/globals_aarch64.hpp @@ -119,7 +119,10 @@ define_pd_global(intx, InlineSmallCode, 1000); product(uint, OnSpinWaitInstCount, 1, DIAGNOSTIC, \ "The number of OnSpinWaitInst instructions to generate." \ "It cannot be used with OnSpinWaitInst=none.") \ - range(1, 99) + range(1, 99) \ + product(bool, UseTBI, false, EXPERIMENTAL, \ + "Use the \"Top Byte Ignore\" feature for ZGC." \ + "And use 64 bit literal addresses instead of 48 bit.") // end of ARCH_FLAGS diff --git a/src/hotspot/cpu/aarch64/jvmciCodeInstaller_aarch64.cpp b/src/hotspot/cpu/aarch64/jvmciCodeInstaller_aarch64.cpp index 17b978012..b52b3f2b3 100644 --- a/src/hotspot/cpu/aarch64/jvmciCodeInstaller_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/jvmciCodeInstaller_aarch64.cpp @@ -57,9 +57,19 @@ void CodeInstaller::pd_patch_OopConstant(int pc_offset, JVMCIObject constant, JV assert(Instruction_aarch64::extract(insn->encoding(), 31, 21) == 0b11010010101 && nativeInstruction_at(pc+4)->is_movk(), "wrong insn in patch"); } else { - // Move wide constant: movz n, movk, movk. - assert(nativeInstruction_at(pc+4)->is_movk() - && nativeInstruction_at(pc+8)->is_movk(), "wrong insn in patch"); + if (!UseTBI) { + // Move wide constant: movz n, movk, movk. + assert(nativeInstruction_at(pc+4)->is_movk() + && nativeInstruction_at(pc+8)->is_movk(), "wrong insn in patch"); + } else { + // Move wide constant: movz n, movk, movk, movk. + // JVMCI would require code generated for it to use 64 bit literals and + // it therefore disables UseTBI until support is enabled. + // @see src/hotspot/cpu/aarch64/vm_version_aarch64.cpp - VM_Version::initialize() + assert(nativeInstruction_at(pc+4)->is_movk() + && nativeInstruction_at(pc+8)->is_movk() + && nativeInstruction_at(pc+12)->is_movk(), "wrong insn in patch"); + } } } #endif // ASSERT diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp index 676a548d0..d89d655af 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp @@ -176,9 +176,10 @@ int MacroAssembler::patch_oop(address insn_addr, address o) { unsigned insn = *(unsigned*)insn_addr; assert(nativeInstruction_at(insn_addr+4)->is_movk(), "wrong insns in patch"); - // OOPs are either narrow (32 bits) or wide (48 bits). We encode + // OOPs are either narrow (32 bits) or wide (48 or 64 bits). We encode // narrow OOPs by setting the upper 16 bits in the first // instruction. + // 64 bit addresses are only enabled with UseTBI set. if (Instruction_aarch64::extract(insn, 31, 21) == 0b11010010101) { // Move narrow OOP uint32_t n = CompressedOops::narrow_oop_value(cast_to_oop(o)); @@ -193,6 +194,12 @@ int MacroAssembler::patch_oop(address insn_addr, address o) { Instruction_aarch64::patch(insn_addr+4, 20, 5, (dest >>= 16) & 0xffff); Instruction_aarch64::patch(insn_addr+8, 20, 5, (dest >>= 16) & 0xffff); instructions = 3; + + if (UseTBI) { + assert(nativeInstruction_at(insn_addr+12)->is_movk(), "wrong insns in patch"); + Instruction_aarch64::patch(insn_addr+12, 20, 5, (dest >>= 16) & 0xffff); + instructions = 4; + } } return instructions * NativeInstruction::instruction_size; } @@ -279,12 +286,18 @@ address MacroAssembler::target_addr_for_insn(address insn_addr, unsigned insn) { } } else if (Instruction_aarch64::extract(insn, 31, 23) == 0b110100101) { uint32_t *insns = (uint32_t *)insn_addr; - // Move wide constant: movz, movk, movk. See movptr(). - assert(nativeInstruction_at(insns+1)->is_movk(), "wrong insns in patch"); - assert(nativeInstruction_at(insns+2)->is_movk(), "wrong insns in patch"); - return address(uint64_t(Instruction_aarch64::extract(insns[0], 20, 5)) + // Move wide constant: movz, movk, movk [, movk]. See movptr(). + assert(nativeInstruction_at(insns+1)->is_movk(), "wrong insns in patch - 2nd movk missing"); + assert(nativeInstruction_at(insns+2)->is_movk(), "wrong insns in patch - 3rd movk missing"); + uint64_t addr = uint64_t(Instruction_aarch64::extract(insns[0], 20, 5)) + (uint64_t(Instruction_aarch64::extract(insns[1], 20, 5)) << 16) - + (uint64_t(Instruction_aarch64::extract(insns[2], 20, 5)) << 32)); + + (uint64_t(Instruction_aarch64::extract(insns[2], 20, 5)) << 32); + // Allow for getting the target address of a possible adddress. + if (UseTBI) { + assert(nativeInstruction_at(insns+3)->is_movk(), "wrong insns in patch - 4th movk missing."); + addr += uint64_t(Instruction_aarch64::extract(insns[3], 20, 5)) << 48; + } + return address(addr); } else if (Instruction_aarch64::extract(insn, 31, 22) == 0b1011100101 && Instruction_aarch64::extract(insn, 4, 0) == 0b11111) { return 0; @@ -1505,10 +1518,9 @@ void MacroAssembler::mov(Register r, Address dest) { movptr(r, imm64); } -// Move a constant pointer into r. In AArch64 mode the virtual -// address space is 48 bits in size, so we only need three -// instructions to create a patchable instruction sequence that can -// reach anywhere. +// Move a constant pointer into r. In AArch64 mode the virtual address space +// is 48 bits in size or 52 bits. We need three or four instructions to create +// a patchable instruction sequence that can reach anywhere. void MacroAssembler::movptr(Register r, uintptr_t imm64) { #ifndef PRODUCT { @@ -1517,12 +1529,18 @@ void MacroAssembler::movptr(Register r, uintptr_t imm64) { block_comment(buffer); } #endif - assert(imm64 < (1ull << 48), "48-bit overflow in address constant"); + if (!UseTBI) { + assert(imm64 < (1ull << 48), "48-bit overflow in address constant"); + } movz(r, imm64 & 0xffff); imm64 >>= 16; movk(r, imm64 & 0xffff, 16); imm64 >>= 16; movk(r, imm64 & 0xffff, 32); + if (UseTBI) { + imm64 >>= 16; + movk(r, imm64 & 0xffff, 48); + } } // Macro to mov replicated immediate to vector register. diff --git a/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp b/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp index d808e4b5b..117686f8c 100644 --- a/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp @@ -234,6 +234,14 @@ void NativeCall::insert(address code_pos, address entry) { Unimplemented(); } //------------------------------------------------------------------- +// movz, movk, movk. +int NativeMovConstReg::instruction_size = 3 * NativeInstruction::instruction_size; + +void NativeMovConstReg::init_for_tbi() { + // movz, movk, movk, movk. + instruction_size = 4 * NativeInstruction::instruction_size; +} + void NativeMovConstReg::verify() { if (! (nativeInstruction_at(instruction_address())->is_movz() || is_adrp_at(instruction_address()) || @@ -439,8 +447,17 @@ bool NativeInstruction::is_general_jump() { NativeInstruction* inst2 = nativeInstruction_at(addr_at(instruction_size * 2)); if (inst2->is_movk()) { NativeInstruction* inst3 = nativeInstruction_at(addr_at(instruction_size * 3)); - if (inst3->is_blr()) { - return true; + if (!UseTBI) { + if (inst3->is_blr()) { + return true; + } + } else { + if(inst3->is_movk()) { + NativeInstruction* inst4 = nativeInstruction_at(addr_at(instruction_size * 4)); + if (inst4->is_blr()) { + return true; + } + } } } } @@ -496,6 +513,16 @@ void NativeJump::patch_verified_entry(address entry, address verified_entry, add ICache::invalidate_range(verified_entry, instruction_size); } +// movz, movk, movk, br. +int NativeGeneralJump::instruction_size = 4 * NativeInstruction::instruction_size; +int NativeGeneralJump::next_instruction_offset = 4 * NativeInstruction::instruction_size; + +void NativeGeneralJump::init_for_tbi() { + // movz, movk, movk, movk, br. + instruction_size = 5 * NativeInstruction::instruction_size; + next_instruction_offset = 5 * NativeInstruction::instruction_size; +} + void NativeGeneralJump::verify() { } void NativeGeneralJump::insert_unconditional(address code_pos, address entry) { diff --git a/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp b/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp index 75f2797c3..2460c02a1 100644 --- a/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp @@ -271,16 +271,19 @@ inline NativeCall* nativeCall_before(address return_address) { class NativeMovConstReg: public NativeInstruction { public: enum Aarch64_specific_constants { - instruction_size = 3 * 4, // movz, movk, movk. See movptr(). instruction_offset = 0, displacement_offset = 0, }; + static int instruction_size; + + static void init_for_tbi(); + address instruction_address() const { return addr_at(instruction_offset); } address next_instruction_address() const { if (nativeInstruction_at(instruction_address())->is_movz()) - // Assume movz, movk, movk + // Assume movz, movk, movk [, movk]. return addr_at(instruction_size); else if (is_adrp_at(instruction_address())) return addr_at(2*4); @@ -469,12 +472,15 @@ inline NativeJump* nativeJump_at(address address) { class NativeGeneralJump: public NativeJump { public: enum AArch64_specific_constants { - instruction_size = 4 * 4, instruction_offset = 0, - data_offset = 0, - next_instruction_offset = 4 * 4 + data_offset = 0 }; + static int instruction_size; + static int next_instruction_offset; + + static void init_for_tbi(); + address jump_destination() const; void set_jump_destination(address dest); diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp index 31dfb7727..bcd064cfe 100644 --- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp @@ -31,6 +31,10 @@ #include "runtime/vm_version.hpp" #include "utilities/formatBuffer.hpp" #include "utilities/macros.hpp" +#include "code/nativeInst.hpp" +#ifdef COMPILER1 +#include "c1/c1_LIRAssembler.hpp" +#endif // COMPILER1 #include OS_HEADER_INLINE(os) @@ -66,6 +70,16 @@ static SpinWait get_spin_wait_desc() { return SpinWait{}; } +// Configure instruction sizes for nativeInst_aarch64 and c1_LIRAssembler_aarch64 +// based on flag UseTBI. +static void init_instruction_sizes_for_tbi() { + NativeMovConstReg::init_for_tbi(); + NativeGeneralJump::init_for_tbi(); +#ifdef COMPILER1 + LIR_Assembler::init_for_tbi(); +#endif // COMPILER1 +} + void VM_Version::initialize() { _supports_cx8 = true; _supports_atomic_getset4 = true; @@ -415,6 +429,20 @@ void VM_Version::initialize() { UsePopCountInstruction = true; } +#if INCLUDE_JVMCI && defined(AARCH64) + if (UseTBI && EnableJVMCI) { + warning("64-bit Literal Addresses disabled due to EnableJVMCI."); + UseTBI = false; + } +#endif + if (UseTBI && !UseZGC) { + warning("UseTBI only works when UseZGC is on."); + UseTBI = false; + } + if (UseTBI) { + init_instruction_sizes_for_tbi(); + } + #ifdef COMPILER2 if (FLAG_IS_DEFAULT(UseMultiplyToLenIntrinsic)) { UseMultiplyToLenIntrinsic = true; diff --git a/src/hotspot/os/linux/gc/z/zNUMA_linux.cpp b/src/hotspot/os/linux/gc/z/zNUMA_linux.cpp index cfe25549f..cefe5d03d 100644 --- a/src/hotspot/os/linux/gc/z/zNUMA_linux.cpp +++ b/src/hotspot/os/linux/gc/z/zNUMA_linux.cpp @@ -59,6 +59,10 @@ uint32_t ZNUMA::memory_id(uintptr_t addr) { uint32_t id = (uint32_t)-1; +#ifdef AARCH64 + addr = CLEAR_COLOR_BITS(addr); +#endif + if (ZSyscall::get_mempolicy((int*)&id, NULL, 0, (void*)addr, MPOL_F_NODE | MPOL_F_ADDR) == -1) { ZErrno err; fatal("Failed to get NUMA id for memory at " PTR_FORMAT " (%s)", addr, err.to_string()); diff --git a/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp b/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp index 7a1130554..d0ffe93ce 100644 --- a/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp +++ b/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp @@ -120,6 +120,14 @@ ZPhysicalMemoryBacking::ZPhysicalMemoryBacking(size_t max_capacity) : _available(0), _initialized(false) { +#ifdef AARCH64 + if (UseTBI) { + // Successfully initialized + _initialized = true; + return; + } +#endif // AARCH64 + // Create backing file _fd = create_fd(ZFILENAME_HEAP); if (_fd == -1) { @@ -350,7 +358,12 @@ void ZPhysicalMemoryBacking::warn_max_map_count(size_t max_capacity) const { // However, ZGC tends to create the most mappings and dominate the total count. // In the worst cases, ZGC will map each granule three times, i.e. once per heap view. // We speculate that we need another 20% to allow for non-ZGC subsystems to map memory. - const size_t required_max_map_count = (max_capacity / ZGranuleSize) * 3 * 1.2; +#ifdef AARCH64 + const size_t required_max_map_multiple = UseTBI ? 1 : 3; +#else + const size_t required_max_map_multiple = 3; +#endif // AARCH64 + const size_t required_max_map_count = (max_capacity / ZGranuleSize) * required_max_map_multiple * 1.2; if (actual_max_map_count < required_max_map_count) { log_warning_p(gc)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****"); log_warning_p(gc)("The system limit on number of memory mappings per process might be too low for the given"); @@ -363,6 +376,13 @@ void ZPhysicalMemoryBacking::warn_max_map_count(size_t max_capacity) const { } void ZPhysicalMemoryBacking::warn_commit_limits(size_t max_capacity) const { +#ifdef AARCH64 + if (UseTBI) { + // Warn if max map count is too low + warn_max_map_count(max_capacity); + return; + } +#endif // AARCH64 // Warn if available space is too low warn_available_space(max_capacity); @@ -681,6 +701,13 @@ size_t ZPhysicalMemoryBacking::commit_default(size_t offset, size_t length) cons } size_t ZPhysicalMemoryBacking::commit(size_t offset, size_t length) const { +#ifdef AARCH64 + if (UseTBI) { + // do nothing + return length; + } +#endif // AARCH64 + if (ZNUMA::is_enabled() && !ZLargePages::is_explicit()) { // To get granule-level NUMA interleaving when using non-large pages, // we must explicitly interleave the memory at commit/fallocate time. @@ -693,6 +720,12 @@ size_t ZPhysicalMemoryBacking::commit(size_t offset, size_t length) const { size_t ZPhysicalMemoryBacking::uncommit(size_t offset, size_t length) const { log_trace(gc, heap)("Uncommitting memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)", offset / M, (offset + length) / M, length / M); +#ifdef AARCH64 + if (UseTBI) { + // Not yet supported. + return length; + } +#endif // AARCH64 const ZErrno err = fallocate(true /* punch_hole */, offset, length); if (err) { @@ -704,6 +737,29 @@ size_t ZPhysicalMemoryBacking::uncommit(size_t offset, size_t length) const { } void ZPhysicalMemoryBacking::map(uintptr_t addr, size_t size, uintptr_t offset) const { +#ifdef AARCH64 + if (UseTBI) { + int flags = MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE; + if (ZLargePages::is_explicit()) { + flags |= MAP_HUGETLB; + } + const void* const res = mmap((void*) CLEAR_TOP_BYTE(addr), size, PROT_READ | PROT_WRITE, flags, 0, 0); + if (res == MAP_FAILED) { + ZErrno err; + fatal("Failed to map memory (%s)", err.to_string()); + } + + // Advise on use of transparent huge pages before touching it + if (ZLargePages::is_transparent()) { + if (madvise((void*) CLEAR_TOP_BYTE(addr), size, MADV_HUGEPAGE) == -1) { + ZErrno err; + log_error(gc)("Failed to advise use of transparent huge pages (%s)", err.to_string()); + } + } + return; + } +#endif // AARCH64 + const void* const res = mmap((void*)addr, size, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_SHARED, _fd, offset); if (res == MAP_FAILED) { ZErrno err; @@ -712,6 +768,9 @@ void ZPhysicalMemoryBacking::map(uintptr_t addr, size_t size, uintptr_t offset) } void ZPhysicalMemoryBacking::unmap(uintptr_t addr, size_t size) const { +#ifdef AARCH64 + addr = CLEAR_COLOR_BITS(addr); +#endif // AARCH64 // Note that we must keep the address space reservation intact and just detach // the backing memory. For this reason we map a new anonymous, non-accessible // and non-reserved page over the mapping instead of actually unmapping. diff --git a/src/hotspot/share/asm/codeBuffer.cpp b/src/hotspot/share/asm/codeBuffer.cpp index 0012152d4..c89c8ee3b 100644 --- a/src/hotspot/share/asm/codeBuffer.cpp +++ b/src/hotspot/share/asm/codeBuffer.cpp @@ -331,7 +331,8 @@ void CodeSection::relocate(address at, RelocationHolder const& spec, int format) rtype == relocInfo::runtime_call_type || rtype == relocInfo::internal_word_type|| rtype == relocInfo::section_word_type || - rtype == relocInfo::external_word_type, + rtype == relocInfo::external_word_type|| + rtype == relocInfo::barrier_type, "code needs relocation information"); // leave behind an indication that we attempted a relocation DEBUG_ONLY(_locs_start = _locs_limit = (relocInfo*)badAddress); diff --git a/src/hotspot/share/code/relocInfo.hpp b/src/hotspot/share/code/relocInfo.hpp index 55d4ac7c6..57d2126fb 100644 --- a/src/hotspot/share/code/relocInfo.hpp +++ b/src/hotspot/share/code/relocInfo.hpp @@ -268,7 +268,8 @@ class relocInfo { trampoline_stub_type = 13, // stub-entry for trampoline runtime_call_w_cp_type = 14, // Runtime call which may load its target from the constant pool data_prefix_tag = 15, // tag for a prefix (carries data arguments) - type_mask = 15 // A mask which selects only the above values + barrier_type = 16, // GC barrier data + type_mask = 31 // A mask which selects only the above values }; private: @@ -307,12 +308,13 @@ class relocInfo { visitor(poll_return) \ visitor(section_word) \ visitor(trampoline_stub) \ + visitor(barrier) \ public: enum { value_width = sizeof(unsigned short) * BitsPerByte, - type_width = 4, // == log2(type_mask+1) + type_width = 5, // == log2(type_mask+1) nontype_width = value_width - type_width, datalen_width = nontype_width-1, datalen_tag = 1 << datalen_width, // or-ed into _value @@ -994,6 +996,29 @@ class metadata_Relocation : public DataRelocation { // Note: metadata_value transparently converts Universe::non_metadata_word to NULL. }; +class barrier_Relocation : public DataRelocation { + + public: + // The uninitialized value used before the relocation has been patched. + // Code assumes that the unpatched value is zero. + static const int16_t unpatched = 0; + + static RelocationHolder spec() { + RelocationHolder rh = newHolder(); + new(rh) barrier_Relocation(); + return rh; + } + + private: + friend class RelocIterator; + barrier_Relocation() : DataRelocation(relocInfo::barrier_type) { } + + public: + virtual int offset() { ShouldNotReachHere(); return 0; } + virtual address value() { ShouldNotReachHere(); return NULL; } + virtual void set_value(address x) { ShouldNotReachHere(); } + virtual void set_value(address x, intptr_t o) { ShouldNotReachHere(); } +}; class virtual_call_Relocation : public CallRelocation { diff --git a/src/hotspot/share/gc/z/vmStructs_z.cpp b/src/hotspot/share/gc/z/vmStructs_z.cpp index c86d11c81..e68077ccb 100644 --- a/src/hotspot/share/gc/z/vmStructs_z.cpp +++ b/src/hotspot/share/gc/z/vmStructs_z.cpp @@ -27,6 +27,7 @@ ZGlobalsForVMStructs::ZGlobalsForVMStructs() : _ZGlobalPhase(&ZGlobalPhase), _ZGlobalSeqNum(&ZGlobalSeqNum), + _ZAddressBase(&ZAddressBase), _ZAddressOffsetMask(&ZAddressOffsetMask), _ZAddressMetadataMask(&ZAddressMetadataMask), _ZAddressMetadataFinalizable(&ZAddressMetadataFinalizable), diff --git a/src/hotspot/share/gc/z/vmStructs_z.hpp b/src/hotspot/share/gc/z/vmStructs_z.hpp index 3c0eb9f74..f8754e0cb 100644 --- a/src/hotspot/share/gc/z/vmStructs_z.hpp +++ b/src/hotspot/share/gc/z/vmStructs_z.hpp @@ -45,6 +45,7 @@ public: uint32_t* _ZGlobalSeqNum; + uintptr_t* _ZAddressBase; uintptr_t* _ZAddressOffsetMask; uintptr_t* _ZAddressMetadataMask; uintptr_t* _ZAddressMetadataFinalizable; @@ -64,6 +65,7 @@ typedef ZAttachedArray ZAttachedArrayForForwardin static_field(ZGlobalsForVMStructs, _instance_p, ZGlobalsForVMStructs*) \ nonstatic_field(ZGlobalsForVMStructs, _ZGlobalPhase, uint32_t*) \ nonstatic_field(ZGlobalsForVMStructs, _ZGlobalSeqNum, uint32_t*) \ + nonstatic_field(ZGlobalsForVMStructs, _ZAddressBase, uintptr_t*) \ nonstatic_field(ZGlobalsForVMStructs, _ZAddressOffsetMask, uintptr_t*) \ nonstatic_field(ZGlobalsForVMStructs, _ZAddressMetadataMask, uintptr_t*) \ nonstatic_field(ZGlobalsForVMStructs, _ZAddressMetadataFinalizable, uintptr_t*) \ @@ -120,6 +122,7 @@ typedef ZAttachedArray ZAttachedArrayForForwardin declare_constant(ZPageSizeMediumShift) \ declare_constant(ZAddressOffsetShift) \ declare_constant(ZAddressOffsetBits) \ + declare_constant(ZAddressBase) \ declare_constant(ZAddressOffsetMask) \ declare_constant(ZAddressOffsetMax) diff --git a/src/hotspot/share/gc/z/zAddress.cpp b/src/hotspot/share/gc/z/zAddress.cpp index cfa7c04d3..47935d4c1 100644 --- a/src/hotspot/share/gc/z/zAddress.cpp +++ b/src/hotspot/share/gc/z/zAddress.cpp @@ -32,6 +32,12 @@ void ZAddress::set_good_mask(uintptr_t mask) { } void ZAddress::initialize() { +#ifdef AARCH64 + ZAddressBase = ZPlatformAddressBase(); +#else + ZAddressBase = 0; +#endif + ZAddressOffsetBits = ZPlatformAddressOffsetBits(); ZAddressOffsetMask = (((uintptr_t)1 << ZAddressOffsetBits) - 1) << ZAddressOffsetShift; ZAddressOffsetMax = (uintptr_t)1 << ZAddressOffsetBits; diff --git a/src/hotspot/share/gc/z/zAddress.hpp b/src/hotspot/share/gc/z/zAddress.hpp index eddd10418..c9e7449dc 100644 --- a/src/hotspot/share/gc/z/zAddress.hpp +++ b/src/hotspot/share/gc/z/zAddress.hpp @@ -52,6 +52,9 @@ public: static bool is_remapped(uintptr_t value); static bool is_in(uintptr_t value); +#ifdef AARCH64 + static uintptr_t base(uintptr_t value); +#endif static uintptr_t offset(uintptr_t value); static uintptr_t good(uintptr_t value); static uintptr_t good_or_null(uintptr_t value); diff --git a/src/hotspot/share/gc/z/zAddress.inline.hpp b/src/hotspot/share/gc/z/zAddress.inline.hpp index a151e7182..c8b560490 100644 --- a/src/hotspot/share/gc/z/zAddress.inline.hpp +++ b/src/hotspot/share/gc/z/zAddress.inline.hpp @@ -31,6 +31,12 @@ #include "utilities/macros.hpp" #include "utilities/powerOfTwo.hpp" +#ifdef AARCH64 +#define AARCH64_BASE(x) base(x) +#else +#define AARCH64_BASE(x) (x) +#endif + inline bool ZAddress::is_null(uintptr_t value) { return value == 0; } @@ -90,7 +96,11 @@ inline bool ZAddress::is_remapped(uintptr_t value) { inline bool ZAddress::is_in(uintptr_t value) { // Check that exactly one non-offset bit is set +#ifdef AARCH64 + if (!is_power_of_2(value & ~ZAddressOffsetMask & ~ZAddressBase)) { +#else if (!is_power_of_2(value & ~ZAddressOffsetMask)) { +#endif return false; } @@ -98,12 +108,18 @@ inline bool ZAddress::is_in(uintptr_t value) { return value & (ZAddressMetadataMask & ~ZAddressMetadataFinalizable); } +#ifdef AARCH64 +inline uintptr_t ZAddress::base(uintptr_t value) { + return value | ZAddressBase; +} +#endif + inline uintptr_t ZAddress::offset(uintptr_t value) { return value & ZAddressOffsetMask; } inline uintptr_t ZAddress::good(uintptr_t value) { - return offset(value) | ZAddressGoodMask; + return AARCH64_BASE(offset(value) | ZAddressGoodMask); } inline uintptr_t ZAddress::good_or_null(uintptr_t value) { @@ -111,27 +127,29 @@ inline uintptr_t ZAddress::good_or_null(uintptr_t value) { } inline uintptr_t ZAddress::finalizable_good(uintptr_t value) { - return offset(value) | ZAddressMetadataFinalizable | ZAddressGoodMask; + return AARCH64_BASE(offset(value) | ZAddressMetadataFinalizable | ZAddressGoodMask); } inline uintptr_t ZAddress::marked(uintptr_t value) { - return offset(value) | ZAddressMetadataMarked; + return AARCH64_BASE(offset(value) | ZAddressMetadataMarked); } inline uintptr_t ZAddress::marked0(uintptr_t value) { - return offset(value) | ZAddressMetadataMarked0; + return AARCH64_BASE(offset(value) | ZAddressMetadataMarked0); } inline uintptr_t ZAddress::marked1(uintptr_t value) { - return offset(value) | ZAddressMetadataMarked1; + return AARCH64_BASE(offset(value) | ZAddressMetadataMarked1); } inline uintptr_t ZAddress::remapped(uintptr_t value) { - return offset(value) | ZAddressMetadataRemapped; + return AARCH64_BASE(offset(value) | ZAddressMetadataRemapped); } inline uintptr_t ZAddress::remapped_or_null(uintptr_t value) { return is_null(value) ? 0 : remapped(value); } +#undef AARCH64_BASE + #endif // SHARE_GC_Z_ZADDRESS_INLINE_HPP diff --git a/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp b/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp index 9916178cc..e5a2ab09a 100644 --- a/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp +++ b/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp @@ -55,6 +55,13 @@ bool ZBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) { return false; } +#ifdef AARCH64 + if (UseTBI) { + // Heal barriers + ZNMethod::nmethod_patch_barriers(nm); + } +#endif + // Heal oops ZNMethod::nmethod_oops_barrier(nm); diff --git a/src/hotspot/share/gc/z/zGlobals.cpp b/src/hotspot/share/gc/z/zGlobals.cpp index 28200e23b..cc82e9335 100644 --- a/src/hotspot/share/gc/z/zGlobals.cpp +++ b/src/hotspot/share/gc/z/zGlobals.cpp @@ -49,6 +49,7 @@ static uint32_t* ZAddressCalculateBadMaskHighOrderBitsAddr() { uint32_t* ZAddressBadMaskHighOrderBitsAddr = ZAddressCalculateBadMaskHighOrderBitsAddr(); +uintptr_t ZAddressBase; size_t ZAddressOffsetBits; uintptr_t ZAddressOffsetMask; size_t ZAddressOffsetMax; diff --git a/src/hotspot/share/gc/z/zGlobals.hpp b/src/hotspot/share/gc/z/zGlobals.hpp index 300f6489a..783411a82 100644 --- a/src/hotspot/share/gc/z/zGlobals.hpp +++ b/src/hotspot/share/gc/z/zGlobals.hpp @@ -102,6 +102,7 @@ extern uint32_t* ZAddressBadMaskHighOrderBitsAddr; const int ZAddressBadMaskHighOrderBitsOffset = LITTLE_ENDIAN_ONLY(4) BIG_ENDIAN_ONLY(0); // Pointer part of address +extern uintptr_t ZAddressBase; extern size_t ZAddressOffsetBits; const size_t ZAddressOffsetShift = 0; extern uintptr_t ZAddressOffsetMask; diff --git a/src/hotspot/share/gc/z/zMark.cpp b/src/hotspot/share/gc/z/zMark.cpp index edc80925d..8a527107a 100644 --- a/src/hotspot/share/gc/z/zMark.cpp +++ b/src/hotspot/share/gc/z/zMark.cpp @@ -643,6 +643,13 @@ public: } if (ZNMethod::is_armed(nm)) { +#ifdef AARCH64 + if (UseTBI) { + // Heal barriers + ZNMethod::nmethod_patch_barriers(nm); + } +#endif + ZNMethod::nmethod_oops_do_inner(nm, _cl); ZNMethod::disarm(nm); } diff --git a/src/hotspot/share/gc/z/zMarkStackEntry.hpp b/src/hotspot/share/gc/z/zMarkStackEntry.hpp index fbb798cc8..ba66717c1 100644 --- a/src/hotspot/share/gc/z/zMarkStackEntry.hpp +++ b/src/hotspot/share/gc/z/zMarkStackEntry.hpp @@ -78,7 +78,8 @@ private: typedef ZBitField field_follow; typedef ZBitField field_inc_live; typedef ZBitField field_mark; - typedef ZBitField field_object_address; + // Set ValueShift to 2 to avoid the top bits being cleared. + typedef ZBitField field_object_address; typedef ZBitField field_partial_array_length; typedef ZBitField field_partial_array_offset; diff --git a/src/hotspot/share/gc/z/zNMethod.cpp b/src/hotspot/share/gc/z/zNMethod.cpp index 71f510c2e..777ade668 100644 --- a/src/hotspot/share/gc/z/zNMethod.cpp +++ b/src/hotspot/share/gc/z/zNMethod.cpp @@ -28,7 +28,10 @@ #include "gc/shared/barrierSet.hpp" #include "gc/shared/barrierSetNMethod.hpp" #include "gc/shared/suspendibleThreadSet.hpp" +#include "gc/z/zArray.inline.hpp" #include "gc/z/zBarrier.inline.hpp" +#include "gc/z/zBarrierSet.hpp" +#include "gc/z/zBarrierSetAssembler.hpp" #include "gc/z/zGlobals.hpp" #include "gc/z/zLock.inline.hpp" #include "gc/z/zNMethod.hpp" @@ -54,12 +57,23 @@ static void set_gc_data(nmethod* nm, ZNMethodData* data) { } void ZNMethod::attach_gc_data(nmethod* nm) { +#ifdef AARCH64 + ZArray
barriers; +#endif GrowableArray immediate_oops; bool non_immediate_oops = false; - // Find all oop relocations + // Find all barrier and oop relocations RelocIterator iter(nm); while (iter.next()) { +#ifdef AARCH64 + if (UseTBI && iter.type() == relocInfo::barrier_type) { + // Barrier relocation + barrier_Relocation* const reloc = iter.barrier_reloc(); + barriers.push(reloc->addr()); + continue; + } +#endif if (iter.type() != relocInfo::oop_type) { // Not an oop continue; @@ -92,9 +106,16 @@ void ZNMethod::attach_gc_data(nmethod* nm) { ZNMethodDataOops* const new_oops = ZNMethodDataOops::create(immediate_oops, non_immediate_oops); ZNMethodDataOops* const old_oops = data->swap_oops(new_oops); ZNMethodDataOops::destroy(old_oops); + +#ifdef AARCH64 + if (UseTBI) { + // Attach barriers in GC data + data->swap_barriers(&barriers); + } +#endif } -ZReentrantLock* ZNMethod::lock_for_nmethod(nmethod* nm) { +ZReentrantLock* ZNMethod::lock_for_nmethod(const nmethod* nm) { return gc_data(nm)->lock(); } @@ -116,6 +137,20 @@ void ZNMethod::log_register(const nmethod* nm) { oops->immediates_count(), oops->has_non_immediates() ? "Yes" : "No"); +#ifdef AARCH64 + if (UseTBI) { + LogTarget(Trace, gc, nmethod, barrier) log_barriers; + if (log_barriers.is_enabled()) { + // Print nmethod barriers + ZLocker locker(lock_for_nmethod(nm)); + ZArrayIterator
iter(gc_data(nm)->barriers()); + for (address b; iter.next(&b);) { + log_barriers.print(" Barrier: " PTR_FORMAT, p2i(b)); + } + } + } +#endif + LogTarget(Trace, gc, nmethod, oops) log_oops; if (!log_oops.is_enabled()) { return; @@ -164,6 +199,15 @@ void ZNMethod::register_nmethod(nmethod* nm) { log_register(nm); +#ifdef AARCH64 + if (UseTBI) { + ZLocker locker(lock_for_nmethod(nm)); + // Patch nmathod barriers + nmethod_patch_barriers(nm); + } +#endif + + // Register nmethod ZNMethodTable::register_nmethod(nm); // Disarm nmethod entry barrier @@ -206,6 +250,16 @@ void ZNMethod::disarm(nmethod* nm) { bs->disarm(nm); } +#ifdef AARCH64 +void ZNMethod::nmethod_patch_barriers(nmethod* nm) { + ZBarrierSetAssembler* const bs_asm = ZBarrierSet::assembler(); + ZArrayIterator
iter(gc_data(nm)->barriers()); + for (address barrier; iter.next(&barrier);) { + bs_asm->patch_barrier_relocation(barrier); + } +} +#endif + void ZNMethod::nmethod_oops_do(nmethod* nm, OopClosure* cl) { ZLocker locker(ZNMethod::lock_for_nmethod(nm)); if (!nm->is_alive()) { @@ -329,6 +383,13 @@ public: ZLocker locker(ZNMethod::lock_for_nmethod(nm)); if (ZNMethod::is_armed(nm)) { +#ifdef AARCH64 + if (UseTBI) { + // Heal barriers + ZNMethod::nmethod_patch_barriers(nm); + } +#endif + // Heal oops and disarm ZNMethod::nmethod_oops_barrier(nm); ZNMethod::disarm(nm); diff --git a/src/hotspot/share/gc/z/zNMethod.hpp b/src/hotspot/share/gc/z/zNMethod.hpp index 40ac93adb..bfe6af377 100644 --- a/src/hotspot/share/gc/z/zNMethod.hpp +++ b/src/hotspot/share/gc/z/zNMethod.hpp @@ -48,6 +48,10 @@ public: static bool is_armed(nmethod* nm); static void disarm(nmethod* nm); +#ifdef AARCH64 + static void nmethod_patch_barriers(nmethod* nm); +#endif + static void nmethod_oops_do(nmethod* nm, OopClosure* cl); static void nmethod_oops_do_inner(nmethod* nm, OopClosure* cl); @@ -57,7 +61,7 @@ public: static void nmethods_do_end(); static void nmethods_do(NMethodClosure* cl); - static ZReentrantLock* lock_for_nmethod(nmethod* nm); + static ZReentrantLock* lock_for_nmethod(const nmethod* nm); static void unlink(ZWorkers* workers, bool unloading_occurred); static void purge(ZWorkers* workers); diff --git a/src/hotspot/share/gc/z/zNMethodData.cpp b/src/hotspot/share/gc/z/zNMethodData.cpp index c6efbfe66..70d06557f 100644 --- a/src/hotspot/share/gc/z/zNMethodData.cpp +++ b/src/hotspot/share/gc/z/zNMethodData.cpp @@ -86,3 +86,15 @@ ZNMethodDataOops* ZNMethodData::swap_oops(ZNMethodDataOops* new_oops) { _oops = new_oops; return old_oops; } + +#ifdef AARCH64 +const ZArray
* ZNMethodData::barriers() const { + assert(_lock.is_owned(), "Should be owned"); + return &_barriers; +} + +void ZNMethodData::swap_barriers(ZArray
* new_barriers) { + ZLocker locker(&_lock); + _barriers.swap(new_barriers); +} +#endif \ No newline at end of file diff --git a/src/hotspot/share/gc/z/zNMethodData.hpp b/src/hotspot/share/gc/z/zNMethodData.hpp index 7afd60105..1013855c9 100644 --- a/src/hotspot/share/gc/z/zNMethodData.hpp +++ b/src/hotspot/share/gc/z/zNMethodData.hpp @@ -24,6 +24,7 @@ #ifndef SHARE_GC_Z_ZNMETHODDATA_HPP #define SHARE_GC_Z_ZNMETHODDATA_HPP +#include "gc/z/zArray.hpp" #include "gc/z/zAttachedArray.hpp" #include "gc/z/zLock.hpp" #include "memory/allocation.hpp" @@ -57,6 +58,9 @@ class ZNMethodData : public CHeapObj { private: ZReentrantLock _lock; ZNMethodDataOops* volatile _oops; +#ifdef AARCH64 + ZArray
_barriers; +#endif public: ZNMethodData(); @@ -66,6 +70,11 @@ public: ZNMethodDataOops* oops() const; ZNMethodDataOops* swap_oops(ZNMethodDataOops* oops); + +#ifdef AARCH64 + const ZArray
* barriers() const; + void swap_barriers(ZArray
* barriers); +#endif }; #endif // SHARE_GC_Z_ZNMETHODDATA_HPP diff --git a/src/hotspot/share/gc/z/zPhysicalMemory.cpp b/src/hotspot/share/gc/z/zPhysicalMemory.cpp index ad8b762bd..38422d37c 100644 --- a/src/hotspot/share/gc/z/zPhysicalMemory.cpp +++ b/src/hotspot/share/gc/z/zPhysicalMemory.cpp @@ -277,13 +277,21 @@ void ZPhysicalMemoryManager::try_enable_uncommit(size_t min_capacity, size_t max void ZPhysicalMemoryManager::nmt_commit(uintptr_t offset, size_t size) const { // From an NMT point of view we treat the first heap view (marked0) as committed +#ifdef AARCH64 + const uintptr_t addr = UseTBI ? ZAddress::base(offset) : ZAddress::marked0(offset); +#else // AARCH64 const uintptr_t addr = ZAddress::marked0(offset); +#endif // AARCH64 MemTracker::record_virtual_memory_commit((void*)addr, size, CALLER_PC); } void ZPhysicalMemoryManager::nmt_uncommit(uintptr_t offset, size_t size) const { if (MemTracker::enabled()) { +#ifdef AARCH64 + const uintptr_t addr = UseTBI ? ZAddress::base(offset) : ZAddress::marked0(offset); +#else // AARCH64 const uintptr_t addr = ZAddress::marked0(offset); +#endif // AARCH64 Tracker tracker(Tracker::uncommit); tracker.record((address)addr, size); } @@ -291,6 +299,13 @@ void ZPhysicalMemoryManager::nmt_uncommit(uintptr_t offset, size_t size) const { void ZPhysicalMemoryManager::alloc(ZPhysicalMemory& pmem, size_t size) { assert(is_aligned(size, ZGranuleSize), "Invalid size"); +#ifdef AARCH64 + if (UseTBI) { + // We don't use _manager to alloc addresses. + pmem.add_segment(ZPhysicalMemorySegment(0, size, false /* committed */)); + return; + } +#endif // AARCH64 // Allocate segments while (size > 0) { @@ -303,6 +318,13 @@ void ZPhysicalMemoryManager::alloc(ZPhysicalMemory& pmem, size_t size) { } void ZPhysicalMemoryManager::free(const ZPhysicalMemory& pmem) { +#ifdef AARCH64 + if (UseTBI) { + // We don't use _manager to alloc addresses. + return; + } +#endif // AARCH64 + // Free segments for (int i = 0; i < pmem.nsegments(); i++) { const ZPhysicalMemorySegment& segment = pmem.segment(i); @@ -381,6 +403,13 @@ void ZPhysicalMemoryManager::unmap_view(uintptr_t addr, size_t size) const { } void ZPhysicalMemoryManager::pretouch(uintptr_t offset, size_t size) const { +#ifdef AARCH64 + if (UseTBI) { + pretouch_view(ZAddress::base(offset), size); + return; + } +#endif // AARCH64 + if (ZVerifyViews) { // Pre-touch good view pretouch_view(ZAddress::good(offset), size); @@ -394,6 +423,13 @@ void ZPhysicalMemoryManager::pretouch(uintptr_t offset, size_t size) const { void ZPhysicalMemoryManager::map(uintptr_t offset, const ZPhysicalMemory& pmem) const { const size_t size = pmem.size(); +#ifdef AARCH64 + if (UseTBI) { + map_view(ZAddress::base(offset), pmem); + nmt_commit(offset, size); + return; + } +#endif // AARCH64 if (ZVerifyViews) { // Map good view @@ -410,6 +446,12 @@ void ZPhysicalMemoryManager::map(uintptr_t offset, const ZPhysicalMemory& pmem) void ZPhysicalMemoryManager::unmap(uintptr_t offset, size_t size) const { nmt_uncommit(offset, size); +#ifdef AARCH64 + if (UseTBI) { + unmap_view(ZAddress::base(offset), size); + return; + } +#endif // AARCH64 if (ZVerifyViews) { // Unmap good view @@ -423,12 +465,24 @@ void ZPhysicalMemoryManager::unmap(uintptr_t offset, size_t size) const { } void ZPhysicalMemoryManager::debug_map(uintptr_t offset, const ZPhysicalMemory& pmem) const { +#ifdef AARCH64 + if (UseTBI) { + // Does nothing when using VA-masking + return; + } +#endif // AARCH64 // Map good view assert(ZVerifyViews, "Should be enabled"); map_view(ZAddress::good(offset), pmem); } void ZPhysicalMemoryManager::debug_unmap(uintptr_t offset, size_t size) const { +#ifdef AARCH64 + if (UseTBI) { + // Does nothing when using VA-masking + return; + } +#endif // AARCH64 // Unmap good view assert(ZVerifyViews, "Should be enabled"); unmap_view(ZAddress::good(offset), size); diff --git a/src/hotspot/share/gc/z/zVirtualMemory.cpp b/src/hotspot/share/gc/z/zVirtualMemory.cpp index 4f9c9bd4b..cb50ae803 100644 --- a/src/hotspot/share/gc/z/zVirtualMemory.cpp +++ b/src/hotspot/share/gc/z/zVirtualMemory.cpp @@ -104,6 +104,21 @@ size_t ZVirtualMemoryManager::reserve_discontiguous(size_t size) { bool ZVirtualMemoryManager::reserve_contiguous(uintptr_t start, size_t size) { assert(is_aligned(size, ZGranuleSize), "Must be granule aligned"); +#ifdef AARCH64 + if (UseTBI) { + const uintptr_t addr = ZAddress::base(start); + if (!pd_reserve(addr, size)) { + return false; + } + nmt_reserve(addr, size); + + // Make the address range free + _manager.free(start, size); + + return true; + } +#endif // AARCH64 + // Reserve address views const uintptr_t marked0 = ZAddress::marked0(start); const uintptr_t marked1 = ZAddress::marked1(start); diff --git a/src/hotspot/share/jfr/writers/jfrStreamWriterHost.inline.hpp b/src/hotspot/share/jfr/writers/jfrStreamWriterHost.inline.hpp index f8900a13b..356c6ca8b 100644 --- a/src/hotspot/share/jfr/writers/jfrStreamWriterHost.inline.hpp +++ b/src/hotspot/share/jfr/writers/jfrStreamWriterHost.inline.hpp @@ -28,6 +28,7 @@ #include "jfr/jni/jfrJavaSupport.hpp" #include "jfr/writers/jfrStreamWriterHost.hpp" #include "runtime/os.hpp" +#include "runtime/globals.hpp" template StreamWriterHost::StreamWriterHost(typename Adapter::StorageType* storage, Thread* thread) : @@ -74,6 +75,9 @@ inline void StreamWriterHost::write_bytes(void* dest, const void* b template inline void StreamWriterHost::write_bytes(const u1* buf, intptr_t len) { assert(len >= 0, "invariant"); +#ifdef AARCH64 + buf = (const u1*) CLEAR_COLOR_BITS((uintptr_t) buf); +#endif while (len > 0) { const unsigned int nBytes = len > INT_MAX ? INT_MAX : (unsigned int)len; const ssize_t num_written = (ssize_t)os::write(_fd, buf, nBytes); diff --git a/src/hotspot/share/prims/jni.cpp b/src/hotspot/share/prims/jni.cpp index cd0115248..41e946563 100644 --- a/src/hotspot/share/prims/jni.cpp +++ b/src/hotspot/share/prims/jni.cpp @@ -2831,6 +2831,9 @@ JNI_ENTRY(void*, jni_GetPrimitiveArrayCritical(JNIEnv *env, jarray array, jboole type = TypeArrayKlass::cast(a->klass())->element_type(); } void* ret = arrayOop(a)->base(type); +#ifdef AARCH64 + ret = (void*) CLEAR_COLOR_BITS((uintptr_t) ret); +#endif // AARCH64 HOTSPOT_JNI_GETPRIMITIVEARRAYCRITICAL_RETURN(ret); return ret; JNI_END @@ -2893,6 +2896,9 @@ JNI_ENTRY(const jchar*, jni_GetStringCritical(JNIEnv *env, jstring string, jbool } if (isCopy != NULL) *isCopy = JNI_TRUE; } +#ifdef AARCH64 + ret = (jchar*) CLEAR_COLOR_BITS((uintptr_t) ret); +#endif // AARCH64 HOTSPOT_JNI_GETSTRINGCRITICAL_RETURN((uint16_t *) ret); return ret; JNI_END diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZAddress.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZAddress.java index 52d9555e4..4a449d9d4 100644 --- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZAddress.java +++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZAddress.java @@ -51,12 +51,18 @@ class ZAddress { return !is_weak_bad(value); } + static long base(long address) { + // ZAddressBase is a non-zero value only when flag UseTBI is on. + // So nothing changes when the arch is not Aarch64 or UseTBI is off. + return address | ZGlobals.ZAddressBase(); + } + static long offset(Address address) { return as_long(address) & ZGlobals.ZAddressOffsetMask(); } static Address good(Address value) { - return VM.getVM().getDebugger().newAddress(offset(value) | ZGlobals.ZAddressGoodMask()); + return VM.getVM().getDebugger().newAddress(base(offset(value) | ZGlobals.ZAddressGoodMask())); } static Address good_or_null(Address value) { @@ -69,9 +75,14 @@ class ZAddress { static boolean isIn(Address addr) { long value = as_long(addr); - if (!isPowerOf2(value & ~ZGlobals.ZAddressOffsetMask())) { + if (!isPowerOf2(value & ~ZGlobals.ZAddressOffsetMask() & ~ZGlobals.ZAddressBase())) { return false; } return (value & (ZGlobals.ZAddressMetadataMask() & ~ZGlobals.ZAddressMetadataFinalizable())) != 0L; } + + static Address clearTopByte(Address value) { + // (1L << 56) - 1 = 0x 00ff ffff ffff ffff + return VM.getVM().getDebugger().newAddress(as_long(value) & ((1L << 56) - 1)); + } } diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZCollectedHeap.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZCollectedHeap.java index c55370ef7..dd74913dd 100644 --- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZCollectedHeap.java +++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZCollectedHeap.java @@ -92,6 +92,9 @@ public class ZCollectedHeap extends CollectedHeap { if (oopAddress == null) { return null; } + if (ZUtils.getUseTBI()) { + oopAddress = ZAddress.clearTopByte(oopAddress); + } return oopAddress.addOffsetToAsOopHandle(0); } diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobals.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobals.java index e01f7b832..41c9bb37b 100644 --- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobals.java +++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobals.java @@ -98,6 +98,10 @@ public class ZGlobals { return instance().ZGlobalSeqNum(); } + public static long ZAddressBase() { + return instance().ZAddressBase(); + } + public static long ZAddressOffsetMask() { return instance().ZAddressOffsetMask(); } diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobalsForVMStructs.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobalsForVMStructs.java index 28d33f4d9..12054c92a 100644 --- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobalsForVMStructs.java +++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobalsForVMStructs.java @@ -34,6 +34,7 @@ import sun.jvm.hotspot.types.TypeDataBase; class ZGlobalsForVMStructs extends VMObject { private static AddressField ZGlobalPhaseField; private static AddressField ZGlobalSeqNumField; + private static AddressField ZAddressBaseField; private static AddressField ZAddressOffsetMaskField; private static AddressField ZAddressMetadataMaskField; private static AddressField ZAddressMetadataFinalizableField; @@ -52,6 +53,7 @@ class ZGlobalsForVMStructs extends VMObject { ZGlobalPhaseField = type.getAddressField("_ZGlobalPhase"); ZGlobalSeqNumField = type.getAddressField("_ZGlobalSeqNum"); + ZAddressBaseField = type.getAddressField("_ZAddressBase"); ZAddressOffsetMaskField = type.getAddressField("_ZAddressOffsetMask"); ZAddressMetadataMaskField = type.getAddressField("_ZAddressMetadataMask"); ZAddressMetadataFinalizableField = type.getAddressField("_ZAddressMetadataFinalizable"); @@ -74,6 +76,10 @@ class ZGlobalsForVMStructs extends VMObject { return ZGlobalSeqNumField.getValue(addr).getJIntAt(0); } + long ZAddressBase() { + return ZAddressBaseField.getValue(addr).getJLongAt(0); + } + long ZAddressOffsetMask() { return ZAddressOffsetMaskField.getValue(addr).getJLongAt(0); } diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZUtils.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZUtils.java index 2029a71da..65930a006 100644 --- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZUtils.java +++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZUtils.java @@ -37,4 +37,9 @@ class ZUtils { long adjusted = size + mask; return adjusted & ~mask; } + + static boolean getUseTBI() { + VM.Flag flag = VM.getVM().getCommandLineFlag("UseTBI"); + return flag == null ? false : flag.getBool(); + } } -- 2.17.1