commit a935af4ab7f1792fd9b85edf503094221d357ba4 Author: jvmboy Date: Sun Apr 26 19:46:07 2020 +0800 Initial build from OpenJDK 11.0.6 diff --git a/8209375-ZGC-Use-dynamic-base-address-for-mark-stack-.patch b/8209375-ZGC-Use-dynamic-base-address-for-mark-stack-.patch new file mode 100644 index 0000000..dfad78c --- /dev/null +++ b/8209375-ZGC-Use-dynamic-base-address-for-mark-stack-.patch @@ -0,0 +1,184 @@ +From 476ec6be3f75c70c50bd1552c624abca098ddba2 Mon Sep 17 00:00:00 2001 +Date: Wed, 18 Mar 2020 10:25:06 +0000 +Subject: [PATCH] 8209375: ZGC: Use dynamic base address for mark stack space + +Summary: : +LLT: jdk11u/test/hotspot/jtreg/vmTestbase/gc/gctests/SoftReference/soft004/soft004.java +Bug url: https://bugs.openjdk.java.net/browse/JDK-8209375 +--- + src/hotspot/share/gc/z/zGlobals.hpp | 7 +--- + src/hotspot/share/gc/z/zMarkStack.cpp | 74 +++++++++++++++-------------------- + src/hotspot/share/gc/z/zMarkStack.hpp | 1 + + src/hotspot/share/gc/z/z_globals.hpp | 6 +-- + 4 files changed, 38 insertions(+), 50 deletions(-) + +diff --git a/src/hotspot/share/gc/z/zGlobals.hpp b/src/hotspot/share/gc/z/zGlobals.hpp +index 080ea5c0e..0f9e9dcb4 100644 +--- a/src/hotspot/share/gc/z/zGlobals.hpp ++++ b/src/hotspot/share/gc/z/zGlobals.hpp +@@ -117,11 +117,8 @@ extern uintptr_t ZAddressWeakBadMask; + // Marked state + extern uintptr_t ZAddressMetadataMarked; + +-// Address space for mark stack allocations +-const size_t ZMarkStackSpaceSizeShift = 40; // 1TB +-const size_t ZMarkStackSpaceSize = (size_t)1 << ZMarkStackSpaceSizeShift; +-const uintptr_t ZMarkStackSpaceStart = ZAddressSpaceEnd + ZMarkStackSpaceSize; +-const uintptr_t ZMarkStackSpaceEnd = ZMarkStackSpaceStart + ZMarkStackSpaceSize; ++// Mark stack space ++extern uintptr_t ZMarkStackSpaceStart; + const size_t ZMarkStackSpaceExpandSize = (size_t)1 << 25; // 32M + + // Mark stack and magazine sizes +diff --git a/src/hotspot/share/gc/z/zMarkStack.cpp b/src/hotspot/share/gc/z/zMarkStack.cpp +index 52fe51ece..9cc768956 100644 +--- a/src/hotspot/share/gc/z/zMarkStack.cpp ++++ b/src/hotspot/share/gc/z/zMarkStack.cpp +@@ -28,58 +28,44 @@ + #include "gc/z/zMarkStack.inline.hpp" + #include "logging/log.hpp" + #include "runtime/atomic.hpp" ++#include "runtime/os.hpp" + #include "utilities/debug.hpp" + +-#include +-#include ++uintptr_t ZMarkStackSpaceStart; + + ZMarkStackSpace::ZMarkStackSpace() : + _expand_lock(), ++ _start(0), + _top(0), + _end(0) { +- assert(ZMarkStacksMax >= ZMarkStackSpaceExpandSize, "ZMarkStacksMax too small"); +- assert(ZMarkStacksMax <= ZMarkStackSpaceSize, "ZMarkStacksMax too large"); +- ++ assert(ZMarkStackSpaceLimit >= ZMarkStackSpaceExpandSize, "ZMarkStackSpaceLimit too small"); + // Reserve address space +- const void* res = mmap((void*)ZMarkStackSpaceStart, ZMarkStackSpaceSize, +- PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0); +- if (res != (void*)ZMarkStackSpaceStart) { +- log_error(gc, marking)("Failed to reserve address space for marking stacks"); ++ const size_t size = ZMarkStackSpaceLimit; ++ const size_t alignment = (size_t)os::vm_allocation_granularity(); ++ const uintptr_t addr = (uintptr_t)os::reserve_memory(size, NULL, alignment, mtGC); ++ if (addr == 0) { ++ log_error(gc, marking)("Failed to reserve address space for mark stacks"); + return; + } + + // Successfully initialized +- _top = _end = ZMarkStackSpaceStart; +-} ++ _start = _top = _end = addr; + +-bool ZMarkStackSpace::is_initialized() const { +- return _top != 0; ++ // Register mark stack space start ++ ZMarkStackSpaceStart = _start; + } + +-bool ZMarkStackSpace::expand() { +- const size_t max = ZMarkStackSpaceStart + ZMarkStacksMax; +- if (_end + ZMarkStackSpaceExpandSize > max) { +- // Expansion limit reached +- return false; +- } +- +- void* const res = mmap((void*)_end, ZMarkStackSpaceExpandSize, +- PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE|MAP_FIXED, -1, 0); +- if (res == MAP_FAILED) { +- ZErrno err; +- log_error(gc, marking)("Failed to map memory for marking stacks (%s)", err.to_string()); +- return false; +- } +- +- return true; ++bool ZMarkStackSpace::is_initialized() const { ++ return _start != 0; + } + + uintptr_t ZMarkStackSpace::alloc_space(size_t size) { +- uintptr_t top = _top; ++ uintptr_t top = Atomic::load(&_top); + + for (;;) { ++ const uintptr_t end = Atomic::load(&_end); + const uintptr_t new_top = top + size; +- if (new_top > _end) { ++ if (new_top > end) { + // Not enough space left + return 0; + } +@@ -104,24 +90,28 @@ uintptr_t ZMarkStackSpace::expand_and_alloc_space(size_t size) { + return addr; + } + +- // Expand stack space +- if (!expand()) { +- // We currently can't handle the situation where we +- // are running out of mark stack space. +- fatal("Mark stack overflow (allocated " SIZE_FORMAT "M, size " SIZE_FORMAT "M, max " SIZE_FORMAT "M)," +- " use -XX:ZMarkStacksMax=? to increase this limit", +- (_end - ZMarkStackSpaceStart) / M, size / M, ZMarkStacksMax / M); +- return 0; ++ // Check expansion limit ++ const size_t expand_size = ZMarkStackSpaceExpandSize; ++ const size_t old_size = _end - _start; ++ const size_t new_size = old_size + expand_size; ++ if (new_size > ZMarkStackSpaceLimit) { ++ // Expansion limit reached. This is a fatal error since we ++ // currently can't recover from running out of mark stack space. ++ fatal("Mark stack space exhausted. Use -XX:ZMarkStackSpaceLimit= to increase the " ++ "maximum number of bytes allocated for mark stacks. Current limit is " SIZE_FORMAT "M.", ++ ZMarkStackSpaceLimit / M); + } + + log_debug(gc, marking)("Expanding mark stack space: " SIZE_FORMAT "M->" SIZE_FORMAT "M", +- (_end - ZMarkStackSpaceStart) / M, +- (_end - ZMarkStackSpaceStart + ZMarkStackSpaceExpandSize) / M); ++ old_size / M, new_size / M); ++ ++ // Expand ++ os::commit_memory_or_exit((char*)_end, expand_size, false /* executable */, "Mark stack space"); + + // Increment top before end to make sure another + // thread can't steal out newly expanded space. + addr = Atomic::add(size, &_top) - size; +- _end += ZMarkStackSpaceExpandSize; ++ Atomic::add(expand_size, &_end); + + return addr; + } +diff --git a/src/hotspot/share/gc/z/zMarkStack.hpp b/src/hotspot/share/gc/z/zMarkStack.hpp +index b68b9faa3..12f3e4eca 100644 +--- a/src/hotspot/share/gc/z/zMarkStack.hpp ++++ b/src/hotspot/share/gc/z/zMarkStack.hpp +@@ -76,6 +76,7 @@ typedef ZStackList ZMarkStackMagazineList; + class ZMarkStackSpace { + private: + ZLock _expand_lock; ++ uintptr_t _start; + volatile uintptr_t _top; + volatile uintptr_t _end; + +diff --git a/src/hotspot/share/gc/z/z_globals.hpp b/src/hotspot/share/gc/z/z_globals.hpp +index 9e0f8985b..8cee59be7 100644 +--- a/src/hotspot/share/gc/z/z_globals.hpp ++++ b/src/hotspot/share/gc/z/z_globals.hpp +@@ -53,9 +53,9 @@ + "Allow Java threads to stall and wait for GC to complete " \ + "instead of immediately throwing an OutOfMemoryError") \ + \ +- product(size_t, ZMarkStacksMax, NOT_LP64(512*M) LP64_ONLY(8*G), \ +- "Maximum number of bytes allocated for marking stacks") \ +- range(32*M, NOT_LP64(512*M) LP64_ONLY(1024*G)) \ ++ product(size_t, ZMarkStackSpaceLimit, 8*G, \ ++ "Maximum number of bytes allocated for mark stacks") \ ++ range(32*M, 1024*G) \ + \ + product(uint, ZCollectionInterval, 0, \ + "Force GC at a fixed time interval (in seconds)") \ +-- +2.12.3 + diff --git a/8209894-ZGC-Cap-number-of-GC-workers-based-on-heap-s.patch b/8209894-ZGC-Cap-number-of-GC-workers-based-on-heap-s.patch new file mode 100644 index 0000000..ef06ac2 --- /dev/null +++ b/8209894-ZGC-Cap-number-of-GC-workers-based-on-heap-s.patch @@ -0,0 +1,94 @@ +From 7ca249ae82c6b6c60c524781806f9d12ef3f8f98 Mon Sep 17 00:00:00 2001 +Date: Mon, 16 Mar 2020 16:24:43 +0800 +Subject: [PATCH] 8209894: ZGC: Cap number of GC workers based on heap size + +Summary: : +LLT: jdk11u/test/hotspot/jtreg/vmTestbase/nsk/jdi/ObjectReference/disableCollection/disablecollection002/TestDescription.java +Bug url: https://bugs.openjdk.java.net/browse/JDK-8209894 +--- + src/hotspot/share/gc/z/zWorkers.cpp | 23 ++++++++++++++++++----- + src/hotspot/share/gc/z/zWorkers.hpp | 4 +--- + 2 files changed, 19 insertions(+), 8 deletions(-) + +diff --git a/src/hotspot/share/gc/z/zWorkers.cpp b/src/hotspot/share/gc/z/zWorkers.cpp +index 0686ec7af..6a0c2561d 100644 +--- a/src/hotspot/share/gc/z/zWorkers.cpp ++++ b/src/hotspot/share/gc/z/zWorkers.cpp +@@ -1,5 +1,5 @@ + /* +- * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. ++ * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it +@@ -22,14 +22,27 @@ + */ + + #include "precompiled.hpp" ++#include "gc/z/zGlobals.hpp" + #include "gc/z/zTask.hpp" + #include "gc/z/zWorkers.inline.hpp" + #include "runtime/os.hpp" + #include "runtime/mutexLocker.hpp" + #include "runtime/safepoint.hpp" + +-uint ZWorkers::calculate_ncpus(double share_in_percent) { +- return ceil(os::initial_active_processor_count() * share_in_percent / 100.0); ++static uint calculate_nworkers_based_on_ncpus(double cpu_share_in_percent) { ++ return ceil(os::initial_active_processor_count() * cpu_share_in_percent / 100.0); ++} ++ ++static uint calculate_nworkers_based_on_heap_size(double reserve_share_in_percent) { ++ const int nworkers = ((MaxHeapSize * (reserve_share_in_percent / 100.0)) - ZPageSizeMedium) / ZPageSizeSmall; ++ return MAX2(nworkers, 1); ++} ++ ++static uint calculate_nworkers(double cpu_share_in_percent) { ++ // Cap number of workers so that we never use more than 10% of the max heap ++ // for the reserve. This is useful when using small heaps on large machines. ++ return MIN2(calculate_nworkers_based_on_ncpus(cpu_share_in_percent), ++ calculate_nworkers_based_on_heap_size(10.0)); + } + + uint ZWorkers::calculate_nparallel() { +@@ -38,7 +51,7 @@ uint ZWorkers::calculate_nparallel() { + // close to the number of processors tends to lead to over-provisioning and + // scheduling latency issues. Using 60% of the active processors appears to + // be a fairly good balance. +- return calculate_ncpus(60.0); ++ return calculate_nworkers(60.0); + } + + uint ZWorkers::calculate_nconcurrent() { +@@ -48,7 +61,7 @@ uint ZWorkers::calculate_nconcurrent() { + // throughput, while using too few threads will prolong the GC-cycle and + // we then risk being out-run by the application. Using 12.5% of the active + // processors appears to be a fairly good balance. +- return calculate_ncpus(12.5); ++ return calculate_nworkers(12.5); + } + + class ZWorkersWarmupTask : public ZTask { +diff --git a/src/hotspot/share/gc/z/zWorkers.hpp b/src/hotspot/share/gc/z/zWorkers.hpp +index 36a3c61fd..6ce09c447 100644 +--- a/src/hotspot/share/gc/z/zWorkers.hpp ++++ b/src/hotspot/share/gc/z/zWorkers.hpp +@@ -1,5 +1,5 @@ + /* +- * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. ++ * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it +@@ -34,8 +34,6 @@ private: + bool _boost; + WorkGang _workers; + +- static uint calculate_ncpus(double share_in_percent); +- + void run(ZTask* task, uint nworkers); + + public: +-- +2.12.3 + diff --git a/8214345-infinite-recursion-while-checking-super-clas.patch b/8214345-infinite-recursion-while-checking-super-clas.patch new file mode 100644 index 0000000..f1bed5c --- /dev/null +++ b/8214345-infinite-recursion-while-checking-super-clas.patch @@ -0,0 +1,73 @@ +From 557d2ea5560c2d0e9cae7f9a792329dfc5cb9573 Mon Sep 17 00:00:00 2001 +Date: Fri, 29 Nov 2019 18:03:11 +0000 +Subject: [PATCH] 8214345: infinite recursion while checking super class + +Summary: : infinite recursion while checking super class +LLT: ClassBoundCheckingOverflow.java ClassBoundCheckingOverflow.out +Bug url: https://bugs.openjdk.java.net/browse/JDK-8214345 +--- + .../share/classes/com/sun/tools/javac/comp/Check.java | 5 ++++- + .../tools/javac/generics/ClassBoundCheckingOverflow.java | 12 ++++++++++++ + .../tools/javac/generics/ClassBoundCheckingOverflow.out | 3 +++ + 3 files changed, 19 insertions(+), 1 deletion(-) + create mode 100644 test/langtools/tools/javac/generics/ClassBoundCheckingOverflow.java + create mode 100644 test/langtools/tools/javac/generics/ClassBoundCheckingOverflow.out + +diff --git a/src/jdk.compiler/share/classes/com/sun/tools/javac/comp/Check.java b/src/jdk.compiler/share/classes/com/sun/tools/javac/comp/Check.java +index 90cb9d189..b1f4abcb6 100644 +--- a/src/jdk.compiler/share/classes/com/sun/tools/javac/comp/Check.java ++++ b/src/jdk.compiler/share/classes/com/sun/tools/javac/comp/Check.java +@@ -1,5 +1,5 @@ + /* +- * Copyright (c) 1999, 2018, Oracle and/or its affiliates. All rights reserved. ++ * Copyright (c) 1999, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it +@@ -2716,6 +2716,8 @@ public class Check { + if (type.isErroneous()) return; + for (List l = types.interfaces(type); l.nonEmpty(); l = l.tail) { + Type it = l.head; ++ if (type.hasTag(CLASS) && !it.hasTag(CLASS)) continue; // JLS 8.1.5 ++ + Type oldit = seensofar.put(it.tsym, it); + if (oldit != null) { + List oldparams = oldit.allparams(); +@@ -2729,6 +2731,7 @@ public class Check { + checkClassBounds(pos, seensofar, it); + } + Type st = types.supertype(type); ++ if (type.hasTag(CLASS) && !st.hasTag(CLASS)) return; // JLS 8.1.4 + if (st != Type.noType) checkClassBounds(pos, seensofar, st); + } + +diff --git a/test/langtools/tools/javac/generics/ClassBoundCheckingOverflow.java b/test/langtools/tools/javac/generics/ClassBoundCheckingOverflow.java +new file mode 100644 +index 000000000..1aeb7d71a +--- /dev/null ++++ b/test/langtools/tools/javac/generics/ClassBoundCheckingOverflow.java +@@ -0,0 +1,12 @@ ++/* ++ * @test /nodynamiccopyright/ ++ * @bug 8214345 ++ * @summary infinite recursion while checking super class ++ * ++ * @compile/fail/ref=ClassBoundCheckingOverflow.out -XDrawDiagnostics ClassBoundCheckingOverflow.java ++ */ ++ ++public class ClassBoundCheckingOverflow { ++ abstract class InfiniteLoop1> extends E {} ++ abstract class InfiniteLoop2> implements E {} ++} +diff --git a/test/langtools/tools/javac/generics/ClassBoundCheckingOverflow.out b/test/langtools/tools/javac/generics/ClassBoundCheckingOverflow.out +new file mode 100644 +index 000000000..bed6acfd7 +--- /dev/null ++++ b/test/langtools/tools/javac/generics/ClassBoundCheckingOverflow.out +@@ -0,0 +1,3 @@ ++ClassBoundCheckingOverflow.java:10:70: compiler.err.type.found.req: (compiler.misc.type.parameter: E), (compiler.misc.type.req.class) ++ClassBoundCheckingOverflow.java:11:73: compiler.err.type.found.req: (compiler.misc.type.parameter: E), (compiler.misc.type.req.class) ++2 errors +-- +2.12.3 + diff --git a/8214527-AArch64-ZGC-for-Aarch64.patch b/8214527-AArch64-ZGC-for-Aarch64.patch new file mode 100644 index 0000000..7abfe54 --- /dev/null +++ b/8214527-AArch64-ZGC-for-Aarch64.patch @@ -0,0 +1,2691 @@ +From c759704261967595054d7ad928cb2cb13dc53356 Mon Sep 17 00:00:00 2001 +Date: Wed, 25 Dec 2019 14:21:26 +0800 +Subject: [PATCH] 8214527: AArch64: ZGC for Aarch64 + +Summary: : +LLT: jtreg +Bug url: https://bugs.openjdk.java.net/browse/JDK-8214527 +--- + make/autoconf/hotspot.m4 | 3 +- + src/hotspot/cpu/aarch64/aarch64.ad | 486 +++++++++++++++++++++ + .../cpu/aarch64/c1_LIRAssembler_aarch64.cpp | 12 +- + .../gc/shared/barrierSetAssembler_aarch64.hpp | 2 +- + .../shared/modRefBarrierSetAssembler_aarch64.cpp | 4 +- + .../shared/modRefBarrierSetAssembler_aarch64.hpp | 2 +- + .../aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp | 408 +++++++++++++++++ + .../aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp | 92 ++++ + src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp | 19 +- + .../cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp | 8 +- + .../gc/z/zAddress_linux_aarch64.inline.hpp | 31 ++ + .../gc/z/zBackingFile_linux_aarch64.cpp | 426 ++++++++++++++++++ + .../gc/z/zBackingFile_linux_aarch64.hpp | 63 +++ + .../gc/z/zBackingPath_linux_aarch64.cpp | 149 +++++++ + .../gc/z/zBackingPath_linux_aarch64.hpp | 52 +++ + .../linux_aarch64/gc/z/zGlobals_linux_aarch64.cpp | 33 ++ + .../linux_aarch64/gc/z/zGlobals_linux_aarch64.hpp | 88 ++++ + .../gc/z/zLargePages_linux_aarch64.cpp | 38 ++ + .../linux_aarch64/gc/z/zNUMA_linux_aarch64.cpp | 83 ++++ + .../gc/z/zPhysicalMemoryBacking_linux_aarch64.cpp | 270 ++++++++++++ + .../gc/z/zPhysicalMemoryBacking_linux_aarch64.hpp | 65 +++ + .../gc/z/zVirtualMemory_linux_aarch64.cpp | 41 ++ + src/hotspot/share/gc/z/c1/zBarrierSetC1.cpp | 20 +- + 23 files changed, 2368 insertions(+), 27 deletions(-) + create mode 100644 src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp + create mode 100644 src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp + create mode 100644 src/hotspot/os_cpu/linux_aarch64/gc/z/zAddress_linux_aarch64.inline.hpp + create mode 100644 src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.cpp + create mode 100644 src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.hpp + create mode 100644 src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingPath_linux_aarch64.cpp + create mode 100644 src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingPath_linux_aarch64.hpp + create mode 100644 src/hotspot/os_cpu/linux_aarch64/gc/z/zGlobals_linux_aarch64.cpp + create mode 100644 src/hotspot/os_cpu/linux_aarch64/gc/z/zGlobals_linux_aarch64.hpp + create mode 100644 src/hotspot/os_cpu/linux_aarch64/gc/z/zLargePages_linux_aarch64.cpp + create mode 100644 src/hotspot/os_cpu/linux_aarch64/gc/z/zNUMA_linux_aarch64.cpp + create mode 100644 src/hotspot/os_cpu/linux_aarch64/gc/z/zPhysicalMemoryBacking_linux_aarch64.cpp + create mode 100644 src/hotspot/os_cpu/linux_aarch64/gc/z/zPhysicalMemoryBacking_linux_aarch64.hpp + create mode 100644 src/hotspot/os_cpu/linux_aarch64/gc/z/zVirtualMemory_linux_aarch64.cpp + +diff --git a/make/autoconf/hotspot.m4 b/make/autoconf/hotspot.m4 +index 1d4c710be..6bfda33da 100644 +--- a/make/autoconf/hotspot.m4 ++++ b/make/autoconf/hotspot.m4 +@@ -340,7 +340,8 @@ AC_DEFUN_ONCE([HOTSPOT_SETUP_JVM_FEATURES], + + # Only enable ZGC on supported platforms + AC_MSG_CHECKING([if zgc can be built]) +- if test "x$OPENJDK_TARGET_OS" = "xlinux" && test "x$OPENJDK_TARGET_CPU" = "xx86_64"; then ++ if (test "x$OPENJDK_TARGET_OS" = "xlinux" && test "x$OPENJDK_TARGET_CPU" = "xx86_64") || \ ++ (test "x$OPENJDK_TARGET_OS" = "xlinux" && test "x$OPENJDK_TARGET_CPU" = "xaarch64"); then + AC_MSG_RESULT([yes]) + else + DISABLED_JVM_FEATURES="$DISABLED_JVM_FEATURES zgc" +diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad +index bdef2a6d7..d0f3fc33d 100644 +--- a/src/hotspot/cpu/aarch64/aarch64.ad ++++ b/src/hotspot/cpu/aarch64/aarch64.ad +@@ -957,6 +957,146 @@ reg_class v3_reg( + V3, V3_H + ); + ++// Class for 128 bit register v4 ++reg_class v4_reg( ++ V4, V4_H ++); ++ ++// Class for 128 bit register v5 ++reg_class v5_reg( ++ V5, V5_H ++); ++ ++// Class for 128 bit register v6 ++reg_class v6_reg( ++ V6, V6_H ++); ++ ++// Class for 128 bit register v7 ++reg_class v7_reg( ++ V7, V7_H ++); ++ ++// Class for 128 bit register v8 ++reg_class v8_reg( ++ V8, V8_H ++); ++ ++// Class for 128 bit register v9 ++reg_class v9_reg( ++ V9, V9_H ++); ++ ++// Class for 128 bit register v10 ++reg_class v10_reg( ++ V10, V10_H ++); ++ ++// Class for 128 bit register v11 ++reg_class v11_reg( ++ V11, V11_H ++); ++ ++// Class for 128 bit register v12 ++reg_class v12_reg( ++ V12, V12_H ++); ++ ++// Class for 128 bit register v13 ++reg_class v13_reg( ++ V13, V13_H ++); ++ ++// Class for 128 bit register v14 ++reg_class v14_reg( ++ V14, V14_H ++); ++ ++// Class for 128 bit register v15 ++reg_class v15_reg( ++ V15, V15_H ++); ++ ++// Class for 128 bit register v16 ++reg_class v16_reg( ++ V16, V16_H ++); ++ ++// Class for 128 bit register v17 ++reg_class v17_reg( ++ V17, V17_H ++); ++ ++// Class for 128 bit register v18 ++reg_class v18_reg( ++ V18, V18_H ++); ++ ++// Class for 128 bit register v19 ++reg_class v19_reg( ++ V19, V19_H ++); ++ ++// Class for 128 bit register v20 ++reg_class v20_reg( ++ V20, V20_H ++); ++ ++// Class for 128 bit register v21 ++reg_class v21_reg( ++ V21, V21_H ++); ++ ++// Class for 128 bit register v22 ++reg_class v22_reg( ++ V22, V22_H ++); ++ ++// Class for 128 bit register v23 ++reg_class v23_reg( ++ V23, V23_H ++); ++ ++// Class for 128 bit register v24 ++reg_class v24_reg( ++ V24, V24_H ++); ++ ++// Class for 128 bit register v25 ++reg_class v25_reg( ++ V25, V25_H ++); ++ ++// Class for 128 bit register v26 ++reg_class v26_reg( ++ V26, V26_H ++); ++ ++// Class for 128 bit register v27 ++reg_class v27_reg( ++ V27, V27_H ++); ++ ++// Class for 128 bit register v28 ++reg_class v28_reg( ++ V28, V28_H ++); ++ ++// Class for 128 bit register v29 ++reg_class v29_reg( ++ V29, V29_H ++); ++ ++// Class for 128 bit register v30 ++reg_class v30_reg( ++ V30, V30_H ++); ++ ++// Class for 128 bit register v31 ++reg_class v31_reg( ++ V31, V31_H ++); ++ + // Singleton class for condition codes + reg_class int_flags(RFLAGS); + +@@ -988,6 +1128,11 @@ definitions %{ + int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST); + %} + ++source_hpp %{ ++ ++#include "gc/z/c2/zBarrierSetC2.hpp" ++ ++%} + + //----------SOURCE BLOCK------------------------------------------------------- + // This is a block of C++ code which provides values, functions, and +@@ -4732,6 +4877,258 @@ operand vRegD_V3() + interface(REG_INTER); + %} + ++operand vRegD_V4() ++%{ ++ constraint(ALLOC_IN_RC(v4_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V5() ++%{ ++ constraint(ALLOC_IN_RC(v5_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V6() ++%{ ++ constraint(ALLOC_IN_RC(v6_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V7() ++%{ ++ constraint(ALLOC_IN_RC(v7_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V8() ++%{ ++ constraint(ALLOC_IN_RC(v8_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V9() ++%{ ++ constraint(ALLOC_IN_RC(v9_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V10() ++%{ ++ constraint(ALLOC_IN_RC(v10_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V11() ++%{ ++ constraint(ALLOC_IN_RC(v11_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V12() ++%{ ++ constraint(ALLOC_IN_RC(v12_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V13() ++%{ ++ constraint(ALLOC_IN_RC(v13_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V14() ++%{ ++ constraint(ALLOC_IN_RC(v14_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V15() ++%{ ++ constraint(ALLOC_IN_RC(v15_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V16() ++%{ ++ constraint(ALLOC_IN_RC(v16_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V17() ++%{ ++ constraint(ALLOC_IN_RC(v17_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V18() ++%{ ++ constraint(ALLOC_IN_RC(v18_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V19() ++%{ ++ constraint(ALLOC_IN_RC(v19_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V20() ++%{ ++ constraint(ALLOC_IN_RC(v20_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V21() ++%{ ++ constraint(ALLOC_IN_RC(v21_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V22() ++%{ ++ constraint(ALLOC_IN_RC(v22_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V23() ++%{ ++ constraint(ALLOC_IN_RC(v23_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V24() ++%{ ++ constraint(ALLOC_IN_RC(v24_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V25() ++%{ ++ constraint(ALLOC_IN_RC(v25_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V26() ++%{ ++ constraint(ALLOC_IN_RC(v26_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V27() ++%{ ++ constraint(ALLOC_IN_RC(v27_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V28() ++%{ ++ constraint(ALLOC_IN_RC(v28_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V29() ++%{ ++ constraint(ALLOC_IN_RC(v29_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V30() ++%{ ++ constraint(ALLOC_IN_RC(v30_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ ++operand vRegD_V31() ++%{ ++ constraint(ALLOC_IN_RC(v31_reg)); ++ match(RegD); ++ op_cost(0); ++ format %{ %} ++ interface(REG_INTER); ++%} ++ + // Flags register, used as output of signed compare instructions + + // note that on AArch64 we also use this register as the output for +@@ -17077,6 +17474,95 @@ instruct vsrl2L_imm(vecX dst, vecX src, immI shift) %{ + ins_pipe(vshift128_imm); + %} + ++source %{ ++ ++#include "gc/z/zBarrierSetAssembler.hpp" ++ ++static void z_load_barrier_slow_reg(MacroAssembler& _masm, Register dst, ++ Register base, int index, int scale, ++ int disp, bool weak) { ++ const address stub = weak ? ZBarrierSet::assembler()->load_barrier_weak_slow_stub(dst) ++ : ZBarrierSet::assembler()->load_barrier_slow_stub(dst); ++ ++ if (index == -1) { ++ if (disp != 0) { ++ __ lea(dst, Address(base, disp)); ++ } else { ++ __ mov(dst, base); ++ } ++ } else { ++ Register index_reg = as_Register(index); ++ if (disp == 0) { ++ __ lea(dst, Address(base, index_reg, Address::lsl(scale))); ++ } else { ++ __ lea(dst, Address(base, disp)); ++ __ lea(dst, Address(dst, index_reg, Address::lsl(scale))); ++ } ++ } ++ ++ __ far_call(RuntimeAddress(stub)); ++} ++ ++%} ++ ++// ++// Execute ZGC load barrier (strong) slow path ++// ++instruct loadBarrierSlowReg(iRegP dst, memory mem, rFlagsReg cr, ++ vRegD_V0 v0, vRegD_V1 v1, vRegD_V2 v2, vRegD_V3 v3, vRegD_V4 v4, ++ vRegD_V5 v5, vRegD_V6 v6, vRegD_V7 v7, vRegD_V8 v8, vRegD_V9 v9, ++ vRegD_V10 v10, vRegD_V11 v11, vRegD_V12 v12, vRegD_V13 v13, vRegD_V14 v14, ++ vRegD_V15 v15, vRegD_V16 v16, vRegD_V17 v17, vRegD_V18 v18, vRegD_V19 v19, ++ vRegD_V20 v20, vRegD_V21 v21, vRegD_V22 v22, vRegD_V23 v23, vRegD_V24 v24, ++ vRegD_V25 v25, vRegD_V26 v26, vRegD_V27 v27, vRegD_V28 v28, vRegD_V29 v29, ++ vRegD_V30 v30, vRegD_V31 v31) %{ ++ match(Set dst (LoadBarrierSlowReg mem)); ++ //predicate(!n->as_LoadBarrierSlowReg()->is_weak()); ++ ++ effect(DEF dst, KILL cr, ++ KILL v0, KILL v1, KILL v2, KILL v3, KILL v4, KILL v5, KILL v6, KILL v7, ++ KILL v8, KILL v9, KILL v10, KILL v11, KILL v12, KILL v13, KILL v14, ++ KILL v15, KILL v16, KILL v17, KILL v18, KILL v19, KILL v20, KILL v21, ++ KILL v22, KILL v23, KILL v24, KILL v25, KILL v26, KILL v27, KILL v28, ++ KILL v29, KILL v30, KILL v31); ++ ++ format %{"LoadBarrierSlowReg $dst, $mem" %} ++ ins_encode %{ ++ z_load_barrier_slow_reg(_masm, $dst$$Register, $mem$$base$$Register, ++ $mem$$index, $mem$$scale, $mem$$disp, false); ++ %} ++ ins_pipe(pipe_slow); ++%} ++ ++// ++// Execute ZGC load barrier (weak) slow path ++// ++instruct loadBarrierWeakSlowReg(iRegP dst, memory mem, rFlagsReg cr, ++ vRegD_V0 v0, vRegD_V1 v1, vRegD_V2 v2, vRegD_V3 v3, vRegD_V4 v4, ++ vRegD_V5 v5, vRegD_V6 v6, vRegD_V7 v7, vRegD_V8 v8, vRegD_V9 v9, ++ vRegD_V10 v10, vRegD_V11 v11, vRegD_V12 v12, vRegD_V13 v13, vRegD_V14 v14, ++ vRegD_V15 v15, vRegD_V16 v16, vRegD_V17 v17, vRegD_V18 v18, vRegD_V19 v19, ++ vRegD_V20 v20, vRegD_V21 v21, vRegD_V22 v22, vRegD_V23 v23, vRegD_V24 v24, ++ vRegD_V25 v25, vRegD_V26 v26, vRegD_V27 v27, vRegD_V28 v28, vRegD_V29 v29, ++ vRegD_V30 v30, vRegD_V31 v31) %{ ++ match(Set dst (LoadBarrierSlowReg mem)); ++ //predicate(n->as_LoadBarrierSlowReg()->is_weak()); ++ ++ effect(DEF dst, KILL cr, ++ KILL v0, KILL v1, KILL v2, KILL v3, KILL v4, KILL v5, KILL v6, KILL v7, ++ KILL v8, KILL v9, KILL v10, KILL v11, KILL v12, KILL v13, KILL v14, ++ KILL v15, KILL v16, KILL v17, KILL v18, KILL v19, KILL v20, KILL v21, ++ KILL v22, KILL v23, KILL v24, KILL v25, KILL v26, KILL v27, KILL v28, ++ KILL v29, KILL v30, KILL v31); ++ ++ format %{"LoadBarrierWeakSlowReg $dst, $mem" %} ++ ins_encode %{ ++ z_load_barrier_slow_reg(_masm, $dst$$Register, $mem$$base$$Register, ++ $mem$$index, $mem$$scale, $mem$$disp, true); ++ %} ++ ins_pipe(pipe_slow); ++%} ++ + //----------PEEPHOLE RULES----------------------------------------------------- + // These must follow all instruction definitions as they use the names + // defined in the instructions definitions. +diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp +index ac91e87d2..e6601434b 100644 +--- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp ++++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp +@@ -1012,7 +1012,11 @@ void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch + if (UseCompressedOops && !wide) { + __ decode_heap_oop(dest->as_register()); + } +- __ verify_oop(dest->as_register()); ++ ++ if (!UseZGC) { ++ // Load barrier has not yet been applied, so ZGC can't verify the oop here ++ __ verify_oop(dest->as_register()); ++ } + } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) { + if (UseCompressedClassPointers) { + __ decode_klass_not_null(dest->as_register()); +@@ -2811,7 +2815,11 @@ void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) { + + + void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) { +- assert(patch_code == lir_patch_none, "Patch code not supported"); ++ if (patch_code != lir_patch_none) { ++ deoptimize_trap(info); ++ return; ++ } ++ + __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr())); + } + +diff --git a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp +index 6bd6c6b89..68e287517 100644 +--- a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp ++++ b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp +@@ -37,7 +37,7 @@ private: + + public: + virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, +- Register addr, Register count, RegSet saved_regs) {} ++ Register src, Register dst, Register count, RegSet saved_regs) {} + virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, + Register start, Register end, Register tmp, RegSet saved_regs) {} + virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, +diff --git a/src/hotspot/cpu/aarch64/gc/shared/modRefBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shared/modRefBarrierSetAssembler_aarch64.cpp +index a28c50169..badd46d05 100644 +--- a/src/hotspot/cpu/aarch64/gc/shared/modRefBarrierSetAssembler_aarch64.cpp ++++ b/src/hotspot/cpu/aarch64/gc/shared/modRefBarrierSetAssembler_aarch64.cpp +@@ -29,10 +29,10 @@ + #define __ masm-> + + void ModRefBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, +- Register addr, Register count, RegSet saved_regs) { ++ Register src, Register dst, Register count, RegSet saved_regs) { + + if (is_oop) { +- gen_write_ref_array_pre_barrier(masm, decorators, addr, count, saved_regs); ++ gen_write_ref_array_pre_barrier(masm, decorators, dst, count, saved_regs); + } + } + +diff --git a/src/hotspot/cpu/aarch64/gc/shared/modRefBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shared/modRefBarrierSetAssembler_aarch64.hpp +index e145b5d74..00e36b919 100644 +--- a/src/hotspot/cpu/aarch64/gc/shared/modRefBarrierSetAssembler_aarch64.hpp ++++ b/src/hotspot/cpu/aarch64/gc/shared/modRefBarrierSetAssembler_aarch64.hpp +@@ -44,7 +44,7 @@ protected: + + public: + virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, +- Register addr, Register count, RegSet saved_regs); ++ Register src, Register dst, Register count, RegSet saved_regs); + virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, + Register start, Register count, Register tmp, RegSet saved_regs); + virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, +diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp +new file mode 100644 +index 000000000..90b2b4ca7 +--- /dev/null ++++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp +@@ -0,0 +1,408 @@ ++/* ++ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. ++ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++ * ++ * This code is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License version 2 only, as ++ * published by the Free Software Foundation. ++ * ++ * This code is distributed in the hope that it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++ * version 2 for more details (a copy is included in the LICENSE file that ++ * accompanied this code). ++ * ++ * You should have received a copy of the GNU General Public License version ++ * 2 along with this work; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++ * or visit www.oracle.com if you need additional information or have any ++ * questions. ++ */ ++ ++#include "precompiled.hpp" ++#include "asm/macroAssembler.inline.hpp" ++#include "code/codeBlob.hpp" ++#include "gc/z/zBarrier.inline.hpp" ++#include "gc/z/zBarrierSet.hpp" ++#include "gc/z/zBarrierSetAssembler.hpp" ++#include "gc/z/zBarrierSetRuntime.hpp" ++#include "memory/resourceArea.hpp" ++#ifdef COMPILER1 ++#include "c1/c1_LIRAssembler.hpp" ++#include "c1/c1_MacroAssembler.hpp" ++#include "gc/z/c1/zBarrierSetC1.hpp" ++#endif // COMPILER1 ++ ++#include "gc/z/zThreadLocalData.hpp" ++ ++ZBarrierSetAssembler::ZBarrierSetAssembler() : ++ _load_barrier_slow_stub(), ++ _load_barrier_weak_slow_stub() {} ++ ++#ifdef PRODUCT ++#define BLOCK_COMMENT(str) /* nothing */ ++#else ++#define BLOCK_COMMENT(str) __ block_comment(str) ++#endif ++ ++#undef __ ++#define __ masm-> ++ ++void ZBarrierSetAssembler::load_at(MacroAssembler* masm, ++ DecoratorSet decorators, ++ BasicType type, ++ Register dst, ++ Address src, ++ Register tmp1, ++ Register tmp_thread) { ++ if (!ZBarrierSet::barrier_needed(decorators, type)) { ++ // Barrier not needed ++ BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); ++ return; ++ } ++ ++ // rscratch1 can be passed as src or dst, so don't use it. ++ RegSet savedRegs = RegSet::of(rscratch2, rheapbase); ++ ++ Label done; ++ assert_different_registers(rheapbase, rscratch2, dst); ++ assert_different_registers(rheapbase, rscratch2, src.base()); ++ ++ __ push(savedRegs, sp); ++ ++ // Load bad mask into scratch register. ++ __ ldr(rheapbase, address_bad_mask_from_thread(rthread)); ++ __ lea(rscratch2, src); ++ __ ldr(dst, src); ++ ++ // Test reference against bad mask. If mask bad, then we need to fix it up. ++ __ tst(dst, rheapbase); ++ __ br(Assembler::EQ, done); ++ ++ __ enter(); ++ ++ __ push(RegSet::range(r0,r28) - RegSet::of(dst), sp); ++ ++ if (c_rarg0 != dst) { ++ __ mov(c_rarg0, dst); ++ } ++ __ mov(c_rarg1, rscratch2); ++ ++ int step = 4 * wordSize; ++ __ mov(rscratch1, -step); ++ __ sub(sp, sp, step); ++ ++ for (int i = 28; i >= 4; i -= 4) { ++ __ st1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2), ++ as_FloatRegister(i+3), __ T1D, Address(__ post(sp, rscratch1))); ++ } ++ ++ __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2); ++ ++ for (int i = 0; i <= 28; i += 4) { ++ __ ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2), ++ as_FloatRegister(i+3), __ T1D, Address(__ post(sp, step))); ++ } ++ ++ // Make sure dst has the return value. ++ if (dst != r0) { ++ __ mov(dst, r0); ++ } ++ ++ __ pop(RegSet::range(r0,r28) - RegSet::of(dst), sp); ++ __ leave(); ++ ++ __ bind(done); ++ ++ // Restore tmps ++ __ pop(savedRegs, sp); ++} ++ ++#ifdef ASSERT ++ ++void ZBarrierSetAssembler::store_at(MacroAssembler* masm, ++ DecoratorSet decorators, ++ BasicType type, ++ Address dst, ++ Register val, ++ Register tmp1, ++ Register tmp2) { ++ // Verify value ++ if (type == T_OBJECT || type == T_ARRAY) { ++ // Note that src could be noreg, which means we ++ // are storing null and can skip verification. ++ if (val != noreg) { ++ Label done; ++ ++ // tmp1 and tmp2 are often set to noreg. ++ RegSet savedRegs = RegSet::of(rscratch1); ++ __ push(savedRegs, sp); ++ ++ __ ldr(rscratch1, address_bad_mask_from_thread(rthread)); ++ __ tst(val, rscratch1); ++ __ br(Assembler::EQ, done); ++ __ stop("Verify oop store failed"); ++ __ should_not_reach_here(); ++ __ bind(done); ++ __ pop(savedRegs, sp); ++ } ++ } ++ ++ // Store value ++ BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); ++} ++ ++#endif // ASSERT ++ ++void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, ++ DecoratorSet decorators, ++ bool is_oop, ++ Register src, ++ Register dst, ++ Register count, ++ RegSet saved_regs) { ++ if (!is_oop) { ++ // Barrier not needed ++ return; ++ } ++ ++ BLOCK_COMMENT("ZBarrierSetAssembler::arraycopy_prologue {"); ++ ++ assert_different_registers(src, count, rscratch1); ++ ++ __ pusha(); ++ ++ if (count == c_rarg0) { ++ if (src == c_rarg1) { ++ // exactly backwards!! ++ __ mov(rscratch1, c_rarg0); ++ __ mov(c_rarg0, c_rarg1); ++ __ mov(c_rarg1, rscratch1); ++ } else { ++ __ mov(c_rarg1, count); ++ __ mov(c_rarg0, src); ++ } ++ } else { ++ __ mov(c_rarg0, src); ++ __ mov(c_rarg1, count); ++ } ++ ++ __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_array_addr(), 2); ++ ++ __ popa(); ++ BLOCK_COMMENT("} ZBarrierSetAssembler::arraycopy_prologue"); ++} ++ ++void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, ++ Register jni_env, ++ Register robj, ++ Register tmp, ++ Label& slowpath) { ++ BLOCK_COMMENT("ZBarrierSetAssembler::try_resolve_jobject_in_native {"); ++ ++ assert_different_registers(jni_env, robj, tmp); ++ ++ // Resolve jobject ++ BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, robj, tmp, slowpath); ++ ++ // The Address offset is too large to direct load - -784. Our range is +127, -128. ++ __ mov(tmp, (long int)(in_bytes(ZThreadLocalData::address_bad_mask_offset()) - ++ in_bytes(JavaThread::jni_environment_offset()))); ++ // Load address bad mask ++ __ add(tmp, jni_env, tmp); ++ __ ldr(tmp, Address(tmp)); ++ ++ // Check address bad mask ++ __ tst(robj, tmp); ++ __ br(Assembler::NE, slowpath); ++ ++ BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_jobject_in_native"); ++} ++ ++#ifdef COMPILER1 ++ ++#undef __ ++#define __ ce->masm()-> ++ ++void ZBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce, ++ LIR_Opr ref) const { ++ assert_different_registers(rheapbase, rthread, ref->as_register()); ++ ++ __ ldr(rheapbase, address_bad_mask_from_thread(rthread)); ++ __ tst(ref->as_register(), rheapbase); ++} ++ ++void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce, ++ ZLoadBarrierStubC1* stub) const { ++ // Stub entry ++ __ bind(*stub->entry()); ++ ++ Register ref = stub->ref()->as_register(); ++ Register ref_addr = noreg; ++ Register tmp = noreg; ++ ++ if (stub->tmp()->is_valid()) { ++ // Load address into tmp register ++ //ce->leal(stub->ref_addr(), stub->tmp()); ++ ce->leal(stub->ref_addr(), stub->tmp(), lir_patch_none, NULL); ++ ref_addr = tmp = stub->tmp()->as_pointer_register(); ++ } else { ++ // Address already in register ++ ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register(); ++ } ++ ++ assert_different_registers(ref, ref_addr, noreg); ++ ++ // Save r0 unless it is the result or tmp register ++ // Set up SP to accomodate parameters and maybe r0.. ++ if (ref != r0 && tmp != r0) { ++ __ sub(sp, sp, 32); ++ __ str(r0, Address(sp, 16)); ++ } else { ++ __ sub(sp, sp, 16); ++ } ++ ++ // Setup arguments and call runtime stub ++ ce->store_parameter(ref_addr, 1); ++ ce->store_parameter(ref, 0); ++ ++ __ far_call(stub->runtime_stub()); ++ ++ // Verify result ++ __ verify_oop(r0, "Bad oop"); ++ ++ // Move result into place ++ if (ref != r0) { ++ __ mov(ref, r0); ++ } ++ ++ // Restore r0 unless it is the result or tmp register ++ if (ref != r0 && tmp != r0) { ++ __ ldr(r0, Address(sp, 16)); ++ __ add(sp, sp, 32); ++ } else { ++ __ add(sp, sp, 16); ++ } ++ ++ // Stub exit ++ __ b(*stub->continuation()); ++} ++ ++#undef __ ++#define __ sasm-> ++ ++void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm, ++ DecoratorSet decorators) const { ++ __ prologue("zgc_load_barrier stub", false); ++ ++ // We don't use push/pop_clobbered_registers() - we need to pull out the result from r0. ++ for (int i = 0; i < 32; i +=2) { ++ __ stpd(as_FloatRegister(i), as_FloatRegister(i+1), Address(__ pre(sp,-16))); ++ } ++ ++ RegSet saveRegs = RegSet::range(r0,r28) - RegSet::of(r0); ++ __ push(saveRegs, sp); ++ ++ // Setup arguments ++ __ load_parameter(0, c_rarg0); ++ __ load_parameter(1, c_rarg1); ++ ++ __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2); ++ ++ __ pop(saveRegs, sp); ++ ++ for (int i = 30; i >0; i -=2) { ++ __ ldpd(as_FloatRegister(i), as_FloatRegister(i+1), Address(__ post(sp, 16))); ++ } ++ ++ __ epilogue(); ++} ++#endif // COMPILER1 ++ ++#undef __ ++#define __ cgen->assembler()-> ++ ++// Generates a register specific stub for calling ++// ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or ++// ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded(). ++// ++// The raddr register serves as both input and output for this stub. When the stub is ++// called the raddr register contains the object field address (oop*) where the bad oop ++// was loaded from, which caused the slow path to be taken. On return from the stub the ++// raddr register contains the good/healed oop returned from ++// ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or ++// ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded(). ++static address generate_load_barrier_stub(StubCodeGenerator* cgen, Register raddr, DecoratorSet decorators) { ++ // Don't generate stub for invalid registers ++ if (raddr == zr || raddr == r29 || raddr == r30) { ++ return NULL; ++ } ++ ++ // Create stub name ++ char name[64]; ++ const bool weak = (decorators & ON_WEAK_OOP_REF) != 0; ++ os::snprintf(name, sizeof(name), "zgc_load_barrier%s_stub_%s", weak ? "_weak" : "", raddr->name()); ++ ++ __ align(CodeEntryAlignment); ++ StubCodeMark mark(cgen, "StubRoutines", os::strdup(name, mtCode)); ++ address start = __ pc(); ++ ++ // Save live registers ++ RegSet savedRegs = RegSet::range(r0,r18) - RegSet::of(raddr); ++ ++ __ enter(); ++ __ push(savedRegs, sp); ++ ++ // Setup arguments ++ if (raddr != c_rarg1) { ++ __ mov(c_rarg1, raddr); ++ } ++ ++ __ ldr(c_rarg0, Address(raddr)); ++ ++ // Call barrier function ++ __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), c_rarg0, c_rarg1); ++ ++ // Move result returned in r0 to raddr, if needed ++ if (raddr != r0) { ++ __ mov(raddr, r0); ++ } ++ ++ __ pop(savedRegs, sp); ++ __ leave(); ++ __ ret(lr); ++ ++ return start; ++} ++ ++#undef __ ++ ++static void barrier_stubs_init_inner(const char* label, const DecoratorSet decorators, address* stub) { ++ const int nregs = 28; // Exclude FP, XZR, SP from calculation. ++ const int code_size = nregs * 254; // Rough estimate of code size ++ ++ ResourceMark rm; ++ ++ CodeBuffer buf(BufferBlob::create(label, code_size)); ++ StubCodeGenerator cgen(&buf); ++ ++ for (int i = 0; i < nregs; i++) { ++ const Register reg = as_Register(i); ++ stub[i] = generate_load_barrier_stub(&cgen, reg, decorators); ++ } ++} ++ ++void ZBarrierSetAssembler::barrier_stubs_init() { ++ barrier_stubs_init_inner("zgc_load_barrier_stubs", ON_STRONG_OOP_REF, _load_barrier_slow_stub); ++ barrier_stubs_init_inner("zgc_load_barrier_weak_stubs", ON_WEAK_OOP_REF, _load_barrier_weak_slow_stub); ++} ++ ++address ZBarrierSetAssembler::load_barrier_slow_stub(Register reg) { ++ return _load_barrier_slow_stub[reg->encoding()]; ++} ++ ++address ZBarrierSetAssembler::load_barrier_weak_slow_stub(Register reg) { ++ return _load_barrier_weak_slow_stub[reg->encoding()]; ++} +diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp +new file mode 100644 +index 000000000..7e8be01cc +--- /dev/null ++++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp +@@ -0,0 +1,92 @@ ++/* ++ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. ++ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++ * ++ * This code is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License version 2 only, as ++ * published by the Free Software Foundation. ++ * ++ * This code is distributed in the hope that it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++ * version 2 for more details (a copy is included in the LICENSE file that ++ * accompanied this code). ++ * ++ * You should have received a copy of the GNU General Public License version ++ * 2 along with this work; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++ * or visit www.oracle.com if you need additional information or have any ++ * questions. ++ */ ++ ++#ifndef CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP ++#define CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP ++ ++#ifdef COMPILER1 ++class LIR_Assembler; ++class LIR_OprDesc; ++typedef LIR_OprDesc* LIR_Opr; ++class StubAssembler; ++class ZLoadBarrierStubC1; ++#endif // COMPILER1 ++ ++class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase { ++private: ++ address _load_barrier_slow_stub[RegisterImpl::number_of_registers]; ++ address _load_barrier_weak_slow_stub[RegisterImpl::number_of_registers]; ++ ++public: ++ ZBarrierSetAssembler(); ++ ++ virtual void load_at(MacroAssembler* masm, ++ DecoratorSet decorators, ++ BasicType type, ++ Register dst, ++ Address src, ++ Register tmp1, ++ Register tmp_thread); ++ ++#ifdef ASSERT ++ virtual void store_at(MacroAssembler* masm, ++ DecoratorSet decorators, ++ BasicType type, ++ Address dst, ++ Register val, ++ Register tmp1, ++ Register tmp2); ++#endif // ASSERT ++ ++ virtual void arraycopy_prologue(MacroAssembler* masm, ++ DecoratorSet decorators, ++ bool is_oop, ++ Register src, ++ Register dst, ++ Register count, ++ RegSet saved_regs); ++ ++ virtual void try_resolve_jobject_in_native(MacroAssembler* masm, ++ Register jni_env, ++ Register robj, ++ Register tmp, ++ Label& slowpath); ++ ++#ifdef COMPILER1 ++ void generate_c1_load_barrier_test(LIR_Assembler* ce, ++ LIR_Opr ref) const; ++ ++ void generate_c1_load_barrier_stub(LIR_Assembler* ce, ++ ZLoadBarrierStubC1* stub) const; ++ ++ void generate_c1_load_barrier_runtime_stub(StubAssembler* sasm, ++ DecoratorSet decorators) const; ++#endif // COMPILER1 ++ ++ virtual void barrier_stubs_init(); ++ ++ address load_barrier_slow_stub(Register reg); ++ address load_barrier_weak_slow_stub(Register reg); ++}; ++ ++#endif // CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP +diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +index eff23b6ba..ac7eb8480 100644 +--- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp ++++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +@@ -45,6 +45,9 @@ + #ifdef COMPILER2 + #include "opto/runtime.hpp" + #endif ++#if INCLUDE_ZGC ++#include "gc/z/zThreadLocalData.hpp" ++#endif + + // Declaration and definition of StubGenerator (no .hpp file). + // For a more detailed description of the stub routine structure +@@ -549,6 +552,16 @@ class StubGenerator: public StubCodeGenerator { + // make sure object is 'reasonable' + __ cbz(r0, exit); // if obj is NULL it is OK + ++#if INCLUDE_ZGC ++ if (UseZGC) { ++ // Check if mask is good. ++ // verifies that ZAddressBadMask & r0 == 0 ++ __ ldr(c_rarg3, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); ++ __ andr(c_rarg2, r0, c_rarg3); ++ __ cbnz(c_rarg2, error); ++ } ++#endif ++ + // Check if the oop is in the right area of memory + __ mov(c_rarg3, (intptr_t) Universe::verify_oop_mask()); + __ andr(c_rarg2, r0, c_rarg3); +@@ -1333,7 +1346,7 @@ class StubGenerator: public StubCodeGenerator { + } + + BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); +- bs->arraycopy_prologue(_masm, decorators, is_oop, d, count, saved_reg); ++ bs->arraycopy_prologue(_masm, decorators, is_oop, s, d, count, saved_reg); + + if (is_oop) { + // save regs before copy_memory +@@ -1399,7 +1412,7 @@ class StubGenerator: public StubCodeGenerator { + } + + BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); +- bs->arraycopy_prologue(_masm, decorators, is_oop, d, count, saved_regs); ++ bs->arraycopy_prologue(_masm, decorators, is_oop, s, d, count, saved_regs); + + if (is_oop) { + // save regs before copy_memory +@@ -1753,7 +1766,7 @@ class StubGenerator: public StubCodeGenerator { + } + + BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); +- bs->arraycopy_prologue(_masm, decorators, is_oop, to, count, wb_pre_saved_regs); ++ bs->arraycopy_prologue(_masm, decorators, is_oop, from, to, count, wb_pre_saved_regs); + + // save the original count + __ mov(count_save, count); +diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp +index e8b4b7414..3aa3f8579 100644 +--- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp ++++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp +@@ -273,13 +273,13 @@ void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce, + Register ref = stub->ref()->as_register(); + Register ref_addr = noreg; + +- if (stub->ref_addr()->is_register()) { +- // Address already in register +- ref_addr = stub->ref_addr()->as_pointer_register(); +- } else { ++ if (stub->tmp()->is_valid()) { + // Load address into tmp register + ce->leal(stub->ref_addr(), stub->tmp(), stub->patch_code(), stub->patch_info()); + ref_addr = stub->tmp()->as_pointer_register(); ++ } else { ++ // Address already in register ++ ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register(); + } + + assert_different_registers(ref, ref_addr, noreg); +diff --git a/src/hotspot/os_cpu/linux_aarch64/gc/z/zAddress_linux_aarch64.inline.hpp b/src/hotspot/os_cpu/linux_aarch64/gc/z/zAddress_linux_aarch64.inline.hpp +new file mode 100644 +index 000000000..936480cb5 +--- /dev/null ++++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zAddress_linux_aarch64.inline.hpp +@@ -0,0 +1,31 @@ ++/* ++ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. ++ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++ * ++ * This code is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License version 2 only, as ++ * published by the Free Software Foundation. ++ * ++ * This code is distributed in the hope that it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++ * version 2 for more details (a copy is included in the LICENSE file that ++ * accompanied this code). ++ * ++ * You should have received a copy of the GNU General Public License version ++ * 2 along with this work; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++ * or visit www.oracle.com if you need additional information or have any ++ * questions. ++ */ ++ ++#ifndef OS_CPU_LINUX_AARCH64_ZADDRESS_LINUX_AARCH64_INLINE_HPP ++#define OS_CPU_LINUX_AARCH64_ZADDRESS_LINUX_AARCH64_INLINE_HPP ++ ++inline uintptr_t ZAddress::address(uintptr_t value) { ++ return value; ++} ++ ++#endif // OS_CPU_LINUX_AARCH64_ZADDRESS_LINUX_AARCH64_INLINE_HPP +diff --git a/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.cpp +new file mode 100644 +index 000000000..47894b5c8 +--- /dev/null ++++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.cpp +@@ -0,0 +1,426 @@ ++/* ++ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. ++ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++ * ++ * This code is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License version 2 only, as ++ * published by the Free Software Foundation. ++ * ++ * This code is distributed in the hope that it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++ * version 2 for more details (a copy is included in the LICENSE file that ++ * accompanied this code). ++ * ++ * You should have received a copy of the GNU General Public License version ++ * 2 along with this work; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++ * or visit www.oracle.com if you need additional information or have any ++ * questions. ++ */ ++ ++#include "precompiled.hpp" ++#include "gc/z/zArray.inline.hpp" ++#include "gc/z/zBackingFile_linux_aarch64.hpp" ++#include "gc/z/zBackingPath_linux_aarch64.hpp" ++#include "gc/z/zErrno.hpp" ++#include "gc/z/zLargePages.inline.hpp" ++#include "logging/log.hpp" ++#include "runtime/os.hpp" ++#include "utilities/align.hpp" ++#include "utilities/debug.hpp" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++// Filesystem names ++#define ZFILESYSTEM_TMPFS "tmpfs" ++#define ZFILESYSTEM_HUGETLBFS "hugetlbfs" ++ ++// Sysfs file for transparent huge page on tmpfs ++#define ZFILENAME_SHMEM_ENABLED "/sys/kernel/mm/transparent_hugepage/shmem_enabled" ++ ++// Java heap filename ++#define ZFILENAME_HEAP "java_heap" ++ ++// Support for building on older Linux systems ++#ifndef __NR_memfd_create ++#define __NR_memfd_create 319 ++#endif ++#ifndef MFD_CLOEXEC ++#define MFD_CLOEXEC 0x0001U ++#endif ++#ifndef MFD_HUGETLB ++#define MFD_HUGETLB 0x0004U ++#endif ++#ifndef O_CLOEXEC ++#define O_CLOEXEC 02000000 ++#endif ++#ifndef O_TMPFILE ++#define O_TMPFILE (020000000 | O_DIRECTORY) ++#endif ++ ++// Filesystem types, see statfs(2) ++#ifndef TMPFS_MAGIC ++#define TMPFS_MAGIC 0x01021994 ++#endif ++#ifndef HUGETLBFS_MAGIC ++#define HUGETLBFS_MAGIC 0x958458f6 ++#endif ++ ++// Preferred tmpfs mount points, ordered by priority ++static const char* z_preferred_tmpfs_mountpoints[] = { ++ "/dev/shm", ++ "/run/shm", ++ NULL ++}; ++ ++// Preferred hugetlbfs mount points, ordered by priority ++static const char* z_preferred_hugetlbfs_mountpoints[] = { ++ "/dev/hugepages", ++ "/hugepages", ++ NULL ++}; ++ ++static int z_memfd_create(const char *name, unsigned int flags) { ++ return syscall(__NR_memfd_create, name, flags); ++} ++ ++bool ZBackingFile::_hugetlbfs_mmap_retry = true; ++ ++ZBackingFile::ZBackingFile() : ++ _fd(-1), ++ _filesystem(0), ++ _available(0), ++ _initialized(false) { ++ ++ // Create backing file ++ _fd = create_fd(ZFILENAME_HEAP); ++ if (_fd == -1) { ++ return; ++ } ++ ++ // Get filesystem statistics ++ struct statfs statfs_buf; ++ if (fstatfs(_fd, &statfs_buf) == -1) { ++ ZErrno err; ++ log_error(gc, init)("Failed to determine filesystem type for backing file (%s)", ++ err.to_string()); ++ return; ++ } ++ ++ _filesystem = statfs_buf.f_type; ++ _available = statfs_buf.f_bavail * statfs_buf.f_bsize; ++ ++ // Make sure we're on a supported filesystem ++ if (!is_tmpfs() && !is_hugetlbfs()) { ++ log_error(gc, init)("Backing file must be located on a %s or a %s filesystem", ++ ZFILESYSTEM_TMPFS, ZFILESYSTEM_HUGETLBFS); ++ return; ++ } ++ ++ // Make sure the filesystem type matches requested large page type ++ if (ZLargePages::is_transparent() && !is_tmpfs()) { ++ log_error(gc, init)("-XX:+UseTransparentHugePages can only be enable when using a %s filesystem", ++ ZFILESYSTEM_TMPFS); ++ return; ++ } ++ ++ if (ZLargePages::is_transparent() && !tmpfs_supports_transparent_huge_pages()) { ++ log_error(gc, init)("-XX:+UseTransparentHugePages on a %s filesystem not supported by kernel", ++ ZFILESYSTEM_TMPFS); ++ return; ++ } ++ ++ if (ZLargePages::is_explicit() && !is_hugetlbfs()) { ++ log_error(gc, init)("-XX:+UseLargePages (without -XX:+UseTransparentHugePages) can only be enabled when using a %s filesystem", ++ ZFILESYSTEM_HUGETLBFS); ++ return; ++ } ++ ++ if (!ZLargePages::is_explicit() && is_hugetlbfs()) { ++ log_error(gc, init)("-XX:+UseLargePages must be enabled when using a %s filesystem", ++ ZFILESYSTEM_HUGETLBFS); ++ return; ++ } ++ ++ // Successfully initialized ++ _initialized = true; ++} ++ ++int ZBackingFile::create_mem_fd(const char* name) const { ++ // Create file name ++ char filename[PATH_MAX]; ++ snprintf(filename, sizeof(filename), "%s%s", name, ZLargePages::is_explicit() ? ".hugetlb" : ""); ++ ++ // Create file ++ const int extra_flags = ZLargePages::is_explicit() ? MFD_HUGETLB : 0; ++ const int fd = z_memfd_create(filename, MFD_CLOEXEC | extra_flags); ++ if (fd == -1) { ++ ZErrno err; ++ log_debug(gc, init)("Failed to create memfd file (%s)", ++ ((UseLargePages && err == EINVAL) ? "Hugepages not supported" : err.to_string())); ++ return -1; ++ } ++ ++ log_info(gc, init)("Heap backed by file: /memfd:%s", filename); ++ ++ return fd; ++} ++ ++int ZBackingFile::create_file_fd(const char* name) const { ++ const char* const filesystem = ZLargePages::is_explicit() ++ ? ZFILESYSTEM_HUGETLBFS ++ : ZFILESYSTEM_TMPFS; ++ const char** const preferred_mountpoints = ZLargePages::is_explicit() ++ ? z_preferred_hugetlbfs_mountpoints ++ : z_preferred_tmpfs_mountpoints; ++ ++ // Find mountpoint ++ ZBackingPath path(filesystem, preferred_mountpoints); ++ if (path.get() == NULL) { ++ log_error(gc, init)("Use -XX:ZPath to specify the path to a %s filesystem", filesystem); ++ return -1; ++ } ++ ++ // Try to create an anonymous file using the O_TMPFILE flag. Note that this ++ // flag requires kernel >= 3.11. If this fails we fall back to open/unlink. ++ const int fd_anon = os::open(path.get(), O_TMPFILE|O_EXCL|O_RDWR|O_CLOEXEC, S_IRUSR|S_IWUSR); ++ if (fd_anon == -1) { ++ ZErrno err; ++ log_debug(gc, init)("Failed to create anonymous file in %s (%s)", path.get(), ++ (err == EINVAL ? "Not supported" : err.to_string())); ++ } else { ++ // Get inode number for anonymous file ++ struct stat stat_buf; ++ if (fstat(fd_anon, &stat_buf) == -1) { ++ ZErrno err; ++ log_error(gc, init)("Failed to determine inode number for anonymous file (%s)", err.to_string()); ++ return -1; ++ } ++ ++ log_info(gc, init)("Heap backed by file: %s/#" UINT64_FORMAT, path.get(), (uint64_t)stat_buf.st_ino); ++ ++ return fd_anon; ++ } ++ ++ log_debug(gc, init)("Falling back to open/unlink"); ++ ++ // Create file name ++ char filename[PATH_MAX]; ++ snprintf(filename, sizeof(filename), "%s/%s.%d", path.get(), name, os::current_process_id()); ++ ++ // Create file ++ const int fd = os::open(filename, O_CREAT|O_EXCL|O_RDWR|O_CLOEXEC, S_IRUSR|S_IWUSR); ++ if (fd == -1) { ++ ZErrno err; ++ log_error(gc, init)("Failed to create file %s (%s)", filename, err.to_string()); ++ return -1; ++ } ++ ++ // Unlink file ++ if (unlink(filename) == -1) { ++ ZErrno err; ++ log_error(gc, init)("Failed to unlink file %s (%s)", filename, err.to_string()); ++ return -1; ++ } ++ ++ log_info(gc, init)("Heap backed by file: %s", filename); ++ ++ return fd; ++} ++ ++int ZBackingFile::create_fd(const char* name) const { ++ if (ZPath == NULL) { ++ // If the path is not explicitly specified, then we first try to create a memfd file ++ // instead of looking for a tmpfd/hugetlbfs mount point. Note that memfd_create() might ++ // not be supported at all (requires kernel >= 3.17), or it might not support large ++ // pages (requires kernel >= 4.14). If memfd_create() fails, then we try to create a ++ // file on an accessible tmpfs or hugetlbfs mount point. ++ const int fd = create_mem_fd(name); ++ if (fd != -1) { ++ return fd; ++ } ++ ++ log_debug(gc, init)("Falling back to searching for an accessible mount point"); ++ } ++ ++ return create_file_fd(name); ++} ++ ++bool ZBackingFile::is_initialized() const { ++ return _initialized; ++} ++ ++int ZBackingFile::fd() const { ++ return _fd; ++} ++ ++size_t ZBackingFile::available() const { ++ return _available; ++} ++ ++bool ZBackingFile::is_tmpfs() const { ++ return _filesystem == TMPFS_MAGIC; ++} ++ ++bool ZBackingFile::is_hugetlbfs() const { ++ return _filesystem == HUGETLBFS_MAGIC; ++} ++ ++bool ZBackingFile::tmpfs_supports_transparent_huge_pages() const { ++ // If the shmem_enabled file exists and is readable then we ++ // know the kernel supports transparent huge pages for tmpfs. ++ return access(ZFILENAME_SHMEM_ENABLED, R_OK) == 0; ++} ++ ++bool ZBackingFile::try_split_and_expand_tmpfs(size_t offset, size_t length, size_t alignment) const { ++ // Try first smaller part. ++ const size_t offset0 = offset; ++ const size_t length0 = align_up(length / 2, alignment); ++ if (!try_expand_tmpfs(offset0, length0, alignment)) { ++ return false; ++ } ++ ++ // Try second smaller part. ++ const size_t offset1 = offset0 + length0; ++ const size_t length1 = length - length0; ++ if (!try_expand_tmpfs(offset1, length1, alignment)) { ++ return false; ++ } ++ ++ return true; ++} ++ ++bool ZBackingFile::try_expand_tmpfs(size_t offset, size_t length, size_t alignment) const { ++ assert(length > 0, "Invalid length"); ++ assert(is_aligned(length, alignment), "Invalid length"); ++ ++ ZErrno err = posix_fallocate(_fd, offset, length); ++ ++ if (err == EINTR && length > alignment) { ++ // Calling posix_fallocate() with a large length can take a long ++ // time to complete. When running profilers, such as VTune, this ++ // syscall will be constantly interrupted by signals. Expanding ++ // the file in smaller steps avoids this problem. ++ return try_split_and_expand_tmpfs(offset, length, alignment); ++ } ++ ++ if (err) { ++ log_error(gc)("Failed to allocate backing file (%s)", err.to_string()); ++ return false; ++ } ++ ++ return true; ++} ++ ++bool ZBackingFile::try_expand_tmpfs(size_t offset, size_t length) const { ++ assert(is_tmpfs(), "Wrong filesystem"); ++ return try_expand_tmpfs(offset, length, os::vm_page_size()); ++} ++ ++bool ZBackingFile::try_expand_hugetlbfs(size_t offset, size_t length) const { ++ assert(is_hugetlbfs(), "Wrong filesystem"); ++ ++ // Prior to kernel 4.3, hugetlbfs did not support posix_fallocate(). ++ // Instead of posix_fallocate() we can use a well-known workaround, ++ // which involves truncating the file to requested size and then try ++ // to map it to verify that there are enough huge pages available to ++ // back it. ++ while (ftruncate(_fd, offset + length) == -1) { ++ ZErrno err; ++ if (err != EINTR) { ++ log_error(gc)("Failed to truncate backing file (%s)", err.to_string()); ++ return false; ++ } ++ } ++ ++ // If we fail mapping during initialization, i.e. when we are pre-mapping ++ // the heap, then we wait and retry a few times before giving up. Otherwise ++ // there is a risk that running JVMs back-to-back will fail, since there ++ // is a delay between process termination and the huge pages owned by that ++ // process being returned to the huge page pool and made available for new ++ // allocations. ++ void* addr = MAP_FAILED; ++ const int max_attempts = 5; ++ for (int attempt = 1; attempt <= max_attempts; attempt++) { ++ addr = mmap(0, length, PROT_READ|PROT_WRITE, MAP_SHARED, _fd, offset); ++ if (addr != MAP_FAILED || !_hugetlbfs_mmap_retry) { ++ // Mapping was successful or mmap retry is disabled ++ break; ++ } ++ ++ ZErrno err; ++ log_debug(gc)("Failed to map backing file (%s), attempt %d of %d", ++ err.to_string(), attempt, max_attempts); ++ ++ // Wait and retry in one second, in the hope that ++ // huge pages will be available by then. ++ sleep(1); ++ } ++ ++ // Disable mmap retry from now on ++ if (_hugetlbfs_mmap_retry) { ++ _hugetlbfs_mmap_retry = false; ++ } ++ ++ if (addr == MAP_FAILED) { ++ // Not enough huge pages left ++ ZErrno err; ++ log_error(gc)("Failed to map backing file (%s)", err.to_string()); ++ return false; ++ } ++ ++ // Successful mapping, unmap again. From now on the pages we mapped ++ // will be reserved for this file. ++ if (munmap(addr, length) == -1) { ++ ZErrno err; ++ log_error(gc)("Failed to unmap backing file (%s)", err.to_string()); ++ return false; ++ } ++ ++ return true; ++} ++ ++bool ZBackingFile::try_expand_tmpfs_or_hugetlbfs(size_t offset, size_t length, size_t alignment) const { ++ assert(is_aligned(offset, alignment), "Invalid offset"); ++ assert(is_aligned(length, alignment), "Invalid length"); ++ ++ log_debug(gc)("Expanding heap from " SIZE_FORMAT "M to " SIZE_FORMAT "M", offset / M, (offset + length) / M); ++ ++ return is_hugetlbfs() ? try_expand_hugetlbfs(offset, length) : try_expand_tmpfs(offset, length); ++} ++ ++size_t ZBackingFile::try_expand(size_t offset, size_t length, size_t alignment) const { ++ size_t start = offset; ++ size_t end = offset + length; ++ ++ // Try to expand ++ if (try_expand_tmpfs_or_hugetlbfs(start, length, alignment)) { ++ // Success ++ return end; ++ } ++ ++ // Failed, try to expand as much as possible ++ for (;;) { ++ length = align_down((end - start) / 2, alignment); ++ if (length < alignment) { ++ // Done, don't expand more ++ return start; ++ } ++ ++ if (try_expand_tmpfs_or_hugetlbfs(start, length, alignment)) { ++ // Success, try expand more ++ start += length; ++ } else { ++ // Failed, try expand less ++ end -= length; ++ } ++ } ++} +diff --git a/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.hpp +new file mode 100644 +index 000000000..032dbc133 +--- /dev/null ++++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.hpp +@@ -0,0 +1,63 @@ ++/* ++ * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. ++ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++ * ++ * This code is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License version 2 only, as ++ * published by the Free Software Foundation. ++ * ++ * This code is distributed in the hope that it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++ * version 2 for more details (a copy is included in the LICENSE file that ++ * accompanied this code). ++ * ++ * You should have received a copy of the GNU General Public License version ++ * 2 along with this work; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++ * or visit www.oracle.com if you need additional information or have any ++ * questions. ++ */ ++ ++#ifndef OS_CPU_LINUX_AARCH64_ZBACKINGFILE_LINUX_AARCH64_HPP ++#define OS_CPU_LINUX_AARCH64_ZBACKINGFILE_LINUX_AARCH64_HPP ++ ++#include "memory/allocation.hpp" ++ ++class ZBackingFile { ++private: ++ static bool _hugetlbfs_mmap_retry; ++ ++ int _fd; ++ uint64_t _filesystem; ++ size_t _available; ++ bool _initialized; ++ ++ int create_mem_fd(const char* name) const; ++ int create_file_fd(const char* name) const; ++ int create_fd(const char* name) const; ++ ++ bool is_tmpfs() const; ++ bool is_hugetlbfs() const; ++ bool tmpfs_supports_transparent_huge_pages() const; ++ ++ bool try_split_and_expand_tmpfs(size_t offset, size_t length, size_t alignment) const; ++ bool try_expand_tmpfs(size_t offset, size_t length, size_t alignment) const; ++ bool try_expand_tmpfs(size_t offset, size_t length) const; ++ bool try_expand_hugetlbfs(size_t offset, size_t length) const; ++ bool try_expand_tmpfs_or_hugetlbfs(size_t offset, size_t length, size_t alignment) const; ++ ++public: ++ ZBackingFile(); ++ ++ bool is_initialized() const; ++ ++ int fd() const; ++ size_t available() const; ++ ++ size_t try_expand(size_t offset, size_t length, size_t alignment) const; ++}; ++ ++#endif // OS_CPU_LINUX_AARCH64_ZBACKINGFILE_LINUX_AARCH64_HPP +diff --git a/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingPath_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingPath_linux_aarch64.cpp +new file mode 100644 +index 000000000..1adffa32b +--- /dev/null ++++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingPath_linux_aarch64.cpp +@@ -0,0 +1,149 @@ ++/* ++ * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. ++ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++ * ++ * This code is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License version 2 only, as ++ * published by the Free Software Foundation. ++ * ++ * This code is distributed in the hope that it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++ * version 2 for more details (a copy is included in the LICENSE file that ++ * accompanied this code). ++ * ++ * You should have received a copy of the GNU General Public License version ++ * 2 along with this work; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++ * or visit www.oracle.com if you need additional information or have any ++ * questions. ++ */ ++ ++#include "precompiled.hpp" ++#include "gc/z/zArray.inline.hpp" ++#include "gc/z/zBackingPath_linux_aarch64.hpp" ++#include "gc/z/zErrno.hpp" ++#include "logging/log.hpp" ++ ++#include ++#include ++ ++// Mount information, see proc(5) for more details. ++#define PROC_SELF_MOUNTINFO "/proc/self/mountinfo" ++ ++ZBackingPath::ZBackingPath(const char* filesystem, const char** preferred_mountpoints) { ++ if (ZPath != NULL) { ++ // Use specified path ++ _path = strdup(ZPath); ++ } else { ++ // Find suitable path ++ _path = find_mountpoint(filesystem, preferred_mountpoints); ++ } ++} ++ ++ZBackingPath::~ZBackingPath() { ++ free(_path); ++ _path = NULL; ++} ++ ++char* ZBackingPath::get_mountpoint(const char* line, const char* filesystem) const { ++ char* line_mountpoint = NULL; ++ char* line_filesystem = NULL; ++ ++ // Parse line and return a newly allocated string containing the mount point if ++ // the line contains a matching filesystem and the mount point is accessible by ++ // the current user. ++ if (sscanf(line, "%*u %*u %*u:%*u %*s %ms %*[^-]- %ms", &line_mountpoint, &line_filesystem) != 2 || ++ strcmp(line_filesystem, filesystem) != 0 || ++ access(line_mountpoint, R_OK|W_OK|X_OK) != 0) { ++ // Not a matching or accessible filesystem ++ free(line_mountpoint); ++ line_mountpoint = NULL; ++ } ++ ++ free(line_filesystem); ++ ++ return line_mountpoint; ++} ++ ++void ZBackingPath::get_mountpoints(const char* filesystem, ZArray* mountpoints) const { ++ FILE* fd = fopen(PROC_SELF_MOUNTINFO, "r"); ++ if (fd == NULL) { ++ ZErrno err; ++ log_error(gc, init)("Failed to open %s: %s", PROC_SELF_MOUNTINFO, err.to_string()); ++ return; ++ } ++ ++ char* line = NULL; ++ size_t length = 0; ++ ++ while (getline(&line, &length, fd) != -1) { ++ char* const mountpoint = get_mountpoint(line, filesystem); ++ if (mountpoint != NULL) { ++ mountpoints->add(mountpoint); ++ } ++ } ++ ++ free(line); ++ fclose(fd); ++} ++ ++void ZBackingPath::free_mountpoints(ZArray* mountpoints) const { ++ ZArrayIterator iter(mountpoints); ++ for (char* mountpoint; iter.next(&mountpoint);) { ++ free(mountpoint); ++ } ++ mountpoints->clear(); ++} ++ ++char* ZBackingPath::find_preferred_mountpoint(const char* filesystem, ++ ZArray* mountpoints, ++ const char** preferred_mountpoints) const { ++ // Find preferred mount point ++ ZArrayIterator iter1(mountpoints); ++ for (char* mountpoint; iter1.next(&mountpoint);) { ++ for (const char** preferred = preferred_mountpoints; *preferred != NULL; preferred++) { ++ if (!strcmp(mountpoint, *preferred)) { ++ // Preferred mount point found ++ return strdup(mountpoint); ++ } ++ } ++ } ++ ++ // Preferred mount point not found ++ log_error(gc, init)("More than one %s filesystem found:", filesystem); ++ ZArrayIterator iter2(mountpoints); ++ for (char* mountpoint; iter2.next(&mountpoint);) { ++ log_error(gc, init)(" %s", mountpoint); ++ } ++ ++ return NULL; ++} ++ ++char* ZBackingPath::find_mountpoint(const char* filesystem, const char** preferred_mountpoints) const { ++ char* path = NULL; ++ ZArray mountpoints; ++ ++ get_mountpoints(filesystem, &mountpoints); ++ ++ if (mountpoints.size() == 0) { ++ // No mount point found ++ log_error(gc, init)("Failed to find an accessible %s filesystem", filesystem); ++ } else if (mountpoints.size() == 1) { ++ // One mount point found ++ path = strdup(mountpoints.at(0)); ++ } else { ++ // More than one mount point found ++ path = find_preferred_mountpoint(filesystem, &mountpoints, preferred_mountpoints); ++ } ++ ++ free_mountpoints(&mountpoints); ++ ++ return path; ++} ++ ++const char* ZBackingPath::get() const { ++ return _path; ++} +diff --git a/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingPath_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingPath_linux_aarch64.hpp +new file mode 100644 +index 000000000..a03aaf960 +--- /dev/null ++++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingPath_linux_aarch64.hpp +@@ -0,0 +1,52 @@ ++/* ++ * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. ++ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++ * ++ * This code is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License version 2 only, as ++ * published by the Free Software Foundation. ++ * ++ * This code is distributed in the hope that it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++ * version 2 for more details (a copy is included in the LICENSE file that ++ * accompanied this code). ++ * ++ * You should have received a copy of the GNU General Public License version ++ * 2 along with this work; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++ * or visit www.oracle.com if you need additional information or have any ++ * questions. ++ */ ++ ++#ifndef OS_CPU_LINUX_AARCH64_ZBACKINGPATH_LINUX_AARCH64_HPP ++#define OS_CPU_LINUX_AARCH64_ZBACKINGPATH_LINUX_AARCH64_HPP ++ ++#include "gc/z/zArray.hpp" ++#include "memory/allocation.hpp" ++ ++class ZBackingPath : public StackObj { ++private: ++ char* _path; ++ ++ char* get_mountpoint(const char* line, ++ const char* filesystem) const; ++ void get_mountpoints(const char* filesystem, ++ ZArray* mountpoints) const; ++ void free_mountpoints(ZArray* mountpoints) const; ++ char* find_preferred_mountpoint(const char* filesystem, ++ ZArray* mountpoints, ++ const char** preferred_mountpoints) const; ++ char* find_mountpoint(const char* filesystem, ++ const char** preferred_mountpoints) const; ++ ++public: ++ ZBackingPath(const char* filesystem, const char** preferred_mountpoints); ++ ~ZBackingPath(); ++ ++ const char* get() const; ++}; ++ ++#endif // OS_CPU_LINUX_AARCH64_ZBACKINGPATH_LINUX_AARCH64_HPP +diff --git a/src/hotspot/os_cpu/linux_aarch64/gc/z/zGlobals_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/gc/z/zGlobals_linux_aarch64.cpp +new file mode 100644 +index 000000000..5d2480993 +--- /dev/null ++++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zGlobals_linux_aarch64.cpp +@@ -0,0 +1,33 @@ ++/* ++ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. ++ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++ * ++ * This code is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License version 2 only, as ++ * published by the Free Software Foundation. ++ * ++ * This code is distributed in the hope that it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++ * version 2 for more details (a copy is included in the LICENSE file that ++ * accompanied this code). ++ * ++ * You should have received a copy of the GNU General Public License version ++ * 2 along with this work; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++ * or visit www.oracle.com if you need additional information or have any ++ * questions. ++ */ ++ ++#include "precompiled.hpp" ++#include "gc/z/zGlobals.hpp" ++ ++uintptr_t ZAddressReservedStart() { ++ return ZAddressMetadataMarked0; ++} ++ ++uintptr_t ZAddressReservedEnd() { ++ return ZAddressMetadataRemapped + ZAddressOffsetMax; ++} +diff --git a/src/hotspot/os_cpu/linux_aarch64/gc/z/zGlobals_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/gc/z/zGlobals_linux_aarch64.hpp +new file mode 100644 +index 000000000..93c18067b +--- /dev/null ++++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zGlobals_linux_aarch64.hpp +@@ -0,0 +1,88 @@ ++/* ++ * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. ++ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++ * ++ * This code is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License version 2 only, as ++ * published by the Free Software Foundation. ++ * ++ * This code is distributed in the hope that it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++ * version 2 for more details (a copy is included in the LICENSE file that ++ * accompanied this code). ++ * ++ * You should have received a copy of the GNU General Public License version ++ * 2 along with this work; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++ * or visit www.oracle.com if you need additional information or have any ++ * questions. ++ */ ++ ++#ifndef OS_CPU_LINUX_AARCH64_ZGLOBALS_LINUX_AARCH64_HPP ++#define OS_CPU_LINUX_AARCH64_ZGLOBALS_LINUX_AARCH64_HPP ++ ++// ++// Page Allocation Tiers ++// --------------------- ++// ++// Page Type Page Size Object Size Limit Object Alignment ++// ------------------------------------------------------------------ ++// Small 2M <= 265K ++// Medium 32M <= 4M 4K ++// Large X*M > 4M 2M ++// ------------------------------------------------------------------ ++// ++// ++// Address Space & Pointer Layout ++// ------------------------------ ++// ++// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB) ++// . . ++// . . ++// . . ++// +--------------------------------+ 0x0000140000000000 (20TB) ++// | Remapped View | ++// +--------------------------------+ 0x0000100000000000 (16TB) ++// | (Reserved, but unused) | ++// +--------------------------------+ 0x00000c0000000000 (12TB) ++// | Marked1 View | ++// +--------------------------------+ 0x0000080000000000 (8TB) ++// | Marked0 View | ++// +--------------------------------+ 0x0000040000000000 (4TB) ++// . . ++// +--------------------------------+ 0x0000000000000000 ++// ++// ++// 6 4 4 4 4 4 0 ++// 3 7 6 5 2 1 0 ++// +-------------------+-+----+-----------------------------------------------+ ++// |00000000 00000000 0|0|1111|11 11111111 11111111 11111111 11111111 11111111| ++// +-------------------+-+----+-----------------------------------------------+ ++// | | | | ++// | | | * 41-0 Object Offset (42-bits, 4TB address space) ++// | | | ++// | | * 45-42 Metadata Bits (4-bits) 0001 = Marked0 (Address view 4-8TB) ++// | | 0010 = Marked1 (Address view 8-12TB) ++// | | 0100 = Remapped (Address view 16-20TB) ++// | | 1000 = Finalizable (Address view N/A) ++// | | ++// | * 46-46 Unused (1-bit, always zero) ++// | ++// * 63-47 Fixed (17-bits, always zero) ++// ++ ++const size_t ZPlatformPageSizeSmallShift = 21; // 2M ++ ++const size_t ZPlatformAddressOffsetBits = 42; // 4TB ++ ++const uintptr_t ZPlatformAddressMetadataShift = ZPlatformAddressOffsetBits; ++ ++const uintptr_t ZPlatformAddressSpaceStart = (uintptr_t)1 << ZPlatformAddressOffsetBits; ++const uintptr_t ZPlatformAddressSpaceSize = ((uintptr_t)1 << ZPlatformAddressOffsetBits) * 4; ++ ++const size_t ZPlatformCacheLineSize = 64; ++ ++#endif // OS_CPU_LINUX_AARCH64_ZGLOBALS_LINUX_AARCH64_HPP +diff --git a/src/hotspot/os_cpu/linux_aarch64/gc/z/zLargePages_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/gc/z/zLargePages_linux_aarch64.cpp +new file mode 100644 +index 000000000..c79195cd1 +--- /dev/null ++++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zLargePages_linux_aarch64.cpp +@@ -0,0 +1,38 @@ ++/* ++ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. ++ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++ * ++ * This code is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License version 2 only, as ++ * published by the Free Software Foundation. ++ * ++ * This code is distributed in the hope that it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++ * version 2 for more details (a copy is included in the LICENSE file that ++ * accompanied this code). ++ * ++ * You should have received a copy of the GNU General Public License version ++ * 2 along with this work; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++ * or visit www.oracle.com if you need additional information or have any ++ * questions. ++ */ ++ ++#include "precompiled.hpp" ++#include "gc/z/zLargePages.hpp" ++#include "runtime/globals.hpp" ++ ++void ZLargePages::initialize_platform() { ++ if (UseLargePages) { ++ if (UseTransparentHugePages) { ++ _state = Transparent; ++ } else { ++ _state = Explicit; ++ } ++ } else { ++ _state = Disabled; ++ } ++} +diff --git a/src/hotspot/os_cpu/linux_aarch64/gc/z/zNUMA_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/gc/z/zNUMA_linux_aarch64.cpp +new file mode 100644 +index 000000000..10706fac2 +--- /dev/null ++++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zNUMA_linux_aarch64.cpp +@@ -0,0 +1,83 @@ ++/* ++ * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. ++ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++ * ++ * This code is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License version 2 only, as ++ * published by the Free Software Foundation. ++ * ++ * This code is distributed in the hope that it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++ * version 2 for more details (a copy is included in the LICENSE file that ++ * accompanied this code). ++ * ++ * You should have received a copy of the GNU General Public License version ++ * 2 along with this work; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++ * or visit www.oracle.com if you need additional information or have any ++ * questions. ++ */ ++ ++#include "gc/z/zErrno.hpp" ++#include "gc/z/zCPU.hpp" ++#include "gc/z/zNUMA.hpp" ++#include "runtime/os.hpp" ++#include "utilities/debug.hpp" ++ ++#include ++#include ++ ++#ifndef MPOL_F_NODE ++#define MPOL_F_NODE (1<<0) // Return next IL mode instead of node mask ++#endif ++ ++#ifndef MPOL_F_ADDR ++#define MPOL_F_ADDR (1<<1) // Look up VMA using address ++#endif ++ ++static int z_get_mempolicy(uint32_t* mode, const unsigned long *nmask, unsigned long maxnode, uintptr_t addr, int flags) { ++ return syscall(__NR_get_mempolicy, mode, nmask, maxnode, addr, flags); ++} ++ ++void ZNUMA::initialize_platform() { ++ _enabled = UseNUMA; ++} ++ ++uint32_t ZNUMA::count() { ++ if (!_enabled) { ++ // NUMA support not enabled ++ return 1; ++ } ++ ++ return os::Linux::numa_max_node() + 1; ++} ++ ++uint32_t ZNUMA::id() { ++ if (!_enabled) { ++ // NUMA support not enabled ++ return 0; ++ } ++ ++ return os::Linux::get_node_by_cpu(ZCPU::id()); ++} ++ ++uint32_t ZNUMA::memory_id(uintptr_t addr) { ++ if (!_enabled) { ++ // NUMA support not enabled, assume everything belongs to node zero ++ return 0; ++ } ++ ++ uint32_t id = (uint32_t)-1; ++ ++ if (z_get_mempolicy(&id, NULL, 0, addr, MPOL_F_NODE | MPOL_F_ADDR) == -1) { ++ ZErrno err; ++ fatal("Failed to get NUMA id for memory at " PTR_FORMAT " (%s)", addr, err.to_string()); ++ } ++ ++ assert(id < count(), "Invalid NUMA id"); ++ ++ return id; ++} +diff --git a/src/hotspot/os_cpu/linux_aarch64/gc/z/zPhysicalMemoryBacking_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/gc/z/zPhysicalMemoryBacking_linux_aarch64.cpp +new file mode 100644 +index 000000000..55c2a16e0 +--- /dev/null ++++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zPhysicalMemoryBacking_linux_aarch64.cpp +@@ -0,0 +1,270 @@ ++/* ++ * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. ++ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++ * ++ * This code is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License version 2 only, as ++ * published by the Free Software Foundation. ++ * ++ * This code is distributed in the hope that it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++ * version 2 for more details (a copy is included in the LICENSE file that ++ * accompanied this code). ++ * ++ * You should have received a copy of the GNU General Public License version ++ * 2 along with this work; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++ * or visit www.oracle.com if you need additional information or have any ++ * questions. ++ */ ++ ++#include "precompiled.hpp" ++#include "gc/z/zAddress.inline.hpp" ++#include "gc/z/zBackingFile_linux_aarch64.hpp" ++#include "gc/z/zErrno.hpp" ++#include "gc/z/zLargePages.inline.hpp" ++#include "gc/z/zMemory.hpp" ++#include "gc/z/zNUMA.hpp" ++#include "gc/z/zPhysicalMemory.inline.hpp" ++#include "gc/z/zPhysicalMemoryBacking_linux_aarch64.hpp" ++#include "logging/log.hpp" ++#include "runtime/os.hpp" ++#include "utilities/align.hpp" ++#include "utilities/debug.hpp" ++ ++#include ++#include ++#include ++ ++// Support for building on older Linux systems ++#ifndef MADV_HUGEPAGE ++#define MADV_HUGEPAGE 14 ++#endif ++ ++// Proc file entry for max map mount ++#define ZFILENAME_PROC_MAX_MAP_COUNT "/proc/sys/vm/max_map_count" ++ ++ZPhysicalMemoryBacking::ZPhysicalMemoryBacking(size_t max_capacity, size_t granule_size) : ++ _manager(), ++ _file(), ++ _granule_size(granule_size) { ++ ++ if (!_file.is_initialized()) { ++ return; ++ } ++ ++ // Check and warn if max map count is too low ++ check_max_map_count(max_capacity, granule_size); ++ ++ // Check and warn if available space on filesystem is too low ++ check_available_space_on_filesystem(max_capacity); ++} ++ ++void ZPhysicalMemoryBacking::check_max_map_count(size_t max_capacity, size_t granule_size) const { ++ const char* const filename = ZFILENAME_PROC_MAX_MAP_COUNT; ++ FILE* const file = fopen(filename, "r"); ++ if (file == NULL) { ++ // Failed to open file, skip check ++ log_debug(gc, init)("Failed to open %s", filename); ++ return; ++ } ++ ++ size_t actual_max_map_count = 0; ++ const int result = fscanf(file, SIZE_FORMAT, &actual_max_map_count); ++ fclose(file); ++ if (result != 1) { ++ // Failed to read file, skip check ++ log_debug(gc, init)("Failed to read %s", filename); ++ return; ++ } ++ ++ // The required max map count is impossible to calculate exactly since subsystems ++ // other than ZGC are also creating memory mappings, and we have no control over that. ++ // However, ZGC tends to create the most mappings and dominate the total count. ++ // In the worst cases, ZGC will map each granule three times, i.e. once per heap view. ++ // We speculate that we need another 20% to allow for non-ZGC subsystems to map memory. ++ const size_t required_max_map_count = (max_capacity / granule_size) * 3 * 1.2; ++ if (actual_max_map_count < required_max_map_count) { ++ log_warning(gc, init)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****"); ++ log_warning(gc, init)("The system limit on number of memory mappings per process might be too low " ++ "for the given"); ++ log_warning(gc, init)("max Java heap size (" SIZE_FORMAT "M). Please adjust %s to allow for at", ++ max_capacity / M, filename); ++ log_warning(gc, init)("least " SIZE_FORMAT " mappings (current limit is " SIZE_FORMAT "). Continuing " ++ "execution with the current", required_max_map_count, actual_max_map_count); ++ log_warning(gc, init)("limit could lead to a fatal error, due to failure to map memory."); ++ } ++} ++ ++void ZPhysicalMemoryBacking::check_available_space_on_filesystem(size_t max_capacity) const { ++ // Note that the available space on a tmpfs or a hugetlbfs filesystem ++ // will be zero if no size limit was specified when it was mounted. ++ const size_t available = _file.available(); ++ if (available == 0) { ++ // No size limit set, skip check ++ log_info(gc, init)("Available space on backing filesystem: N/A"); ++ return; ++ } ++ ++ log_info(gc, init)("Available space on backing filesystem: " SIZE_FORMAT "M", ++ available / M); ++ ++ // Warn if the filesystem doesn't currently have enough space available to hold ++ // the max heap size. The max heap size will be capped if we later hit this limit ++ // when trying to expand the heap. ++ if (available < max_capacity) { ++ log_warning(gc, init)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****"); ++ log_warning(gc, init)("Not enough space available on the backing filesystem to hold the current " ++ "max Java heap"); ++ log_warning(gc, init)("size (" SIZE_FORMAT "M). Please adjust the size of the backing filesystem " ++ "accordingly (available", max_capacity / M); ++ log_warning(gc, init)("space is currently " SIZE_FORMAT "M). Continuing execution with the current " ++ "filesystem size could", available / M); ++ log_warning(gc, init)("lead to a premature OutOfMemoryError being thrown, due to failure to map " ++ "memory."); ++ } ++} ++ ++bool ZPhysicalMemoryBacking::is_initialized() const { ++ return _file.is_initialized(); ++} ++ ++size_t ZPhysicalMemoryBacking::try_expand(size_t old_capacity, size_t new_capacity) { ++ assert(old_capacity < new_capacity, "Invalid old/new capacity"); ++ ++ const size_t capacity = _file.try_expand(old_capacity, new_capacity - old_capacity, _granule_size); ++ if (capacity > old_capacity) { ++ // Add expanded capacity to free list ++ _manager.free(old_capacity, capacity - old_capacity); ++ } ++ ++ return capacity; ++} ++ ++ZPhysicalMemory ZPhysicalMemoryBacking::alloc(size_t size) { ++ assert(is_aligned(size, _granule_size), "Invalid size"); ++ ++ ZPhysicalMemory pmem; ++ ++ // Allocate segments ++ for (size_t allocated = 0; allocated < size; allocated += _granule_size) { ++ const uintptr_t start = _manager.alloc_from_front(_granule_size); ++ assert(start != UINTPTR_MAX, "Allocation should never fail"); ++ pmem.add_segment(ZPhysicalMemorySegment(start, _granule_size)); ++ } ++ ++ return pmem; ++} ++ ++void ZPhysicalMemoryBacking::free(ZPhysicalMemory pmem) { ++ const size_t nsegments = pmem.nsegments(); ++ ++ // Free segments ++ for (size_t i = 0; i < nsegments; i++) { ++ const ZPhysicalMemorySegment segment = pmem.segment(i); ++ _manager.free(segment.start(), segment.size()); ++ } ++} ++ ++void ZPhysicalMemoryBacking::map_failed(ZErrno err) const { ++ if (err == ENOMEM) { ++ fatal("Failed to map memory. Please check the system limit on number of " ++ "memory mappings allowed per process (see %s)", ZFILENAME_PROC_MAX_MAP_COUNT); ++ } else { ++ fatal("Failed to map memory (%s)", err.to_string()); ++ } ++} ++ ++void ZPhysicalMemoryBacking::advise_view(uintptr_t addr, size_t size) const { ++ if (madvise((void*)addr, size, MADV_HUGEPAGE) == -1) { ++ ZErrno err; ++ log_error(gc)("Failed to advise use of transparent huge pages (%s)", err.to_string()); ++ } ++} ++ ++void ZPhysicalMemoryBacking::pretouch_view(uintptr_t addr, size_t size) const { ++ const size_t page_size = ZLargePages::is_explicit() ? os::large_page_size() : os::vm_page_size(); ++ os::pretouch_memory((void*)addr, (void*)(addr + size), page_size); ++} ++ ++void ZPhysicalMemoryBacking::map_view(ZPhysicalMemory pmem, uintptr_t addr, bool pretouch) const { ++ const size_t nsegments = pmem.nsegments(); ++ ++ // Map segments ++ for (size_t i = 0; i < nsegments; i++) { ++ const ZPhysicalMemorySegment segment = pmem.segment(i); ++ const size_t size = segment.size(); ++ const void* const res = mmap((void*)addr, size, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_SHARED, _file.fd(), segment.start()); ++ if (res == MAP_FAILED) { ++ ZErrno err; ++ map_failed(err); ++ } ++ ++ // Advise on use of transparent huge pages before touching it ++ if (ZLargePages::is_transparent()) { ++ advise_view(addr, size); ++ } ++ ++ // NUMA interleave memory before touching it ++ ZNUMA::memory_interleave(addr, size); ++ ++ if (pretouch) { ++ pretouch_view(addr, size); ++ } ++ ++ addr += size; ++ } ++} ++ ++void ZPhysicalMemoryBacking::unmap_view(ZPhysicalMemory pmem, uintptr_t addr) const { ++ // Note that we must keep the address space reservation intact and just detach ++ // the backing memory. For this reason we map a new anonymous, non-accessible ++ // and non-reserved page over the mapping instead of actually unmapping. ++ const size_t size = pmem.size(); ++ const void* const res = mmap((void*)addr, size, PROT_NONE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0); ++ if (res == MAP_FAILED) { ++ ZErrno err; ++ map_failed(err); ++ } ++} ++ ++uintptr_t ZPhysicalMemoryBacking::nmt_address(uintptr_t offset) const { ++ // From an NMT point of view we treat the first heap mapping (marked0) as committed ++ return ZAddress::marked0(offset); ++} ++ ++void ZPhysicalMemoryBacking::map(ZPhysicalMemory pmem, uintptr_t offset) const { ++ if (ZUnmapBadViews) { ++ // Only map the good view, for debugging only ++ map_view(pmem, ZAddress::good(offset), AlwaysPreTouch); ++ } else { ++ // Map all views ++ map_view(pmem, ZAddress::marked0(offset), AlwaysPreTouch); ++ map_view(pmem, ZAddress::marked1(offset), AlwaysPreTouch); ++ map_view(pmem, ZAddress::remapped(offset), AlwaysPreTouch); ++ } ++} ++ ++void ZPhysicalMemoryBacking::unmap(ZPhysicalMemory pmem, uintptr_t offset) const { ++ if (ZUnmapBadViews) { ++ // Only map the good view, for debugging only ++ unmap_view(pmem, ZAddress::good(offset)); ++ } else { ++ // Unmap all views ++ unmap_view(pmem, ZAddress::marked0(offset)); ++ unmap_view(pmem, ZAddress::marked1(offset)); ++ unmap_view(pmem, ZAddress::remapped(offset)); ++ } ++} ++ ++void ZPhysicalMemoryBacking::flip(ZPhysicalMemory pmem, uintptr_t offset) const { ++ assert(ZUnmapBadViews, "Should be enabled"); ++ const uintptr_t addr_good = ZAddress::good(offset); ++ const uintptr_t addr_bad = ZAddress::is_marked(ZAddressGoodMask) ? ZAddress::remapped(offset) : ZAddress::marked(offset); ++ // Map/Unmap views ++ map_view(pmem, addr_good, false /* pretouch */); ++ unmap_view(pmem, addr_bad); ++} +diff --git a/src/hotspot/os_cpu/linux_aarch64/gc/z/zPhysicalMemoryBacking_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/gc/z/zPhysicalMemoryBacking_linux_aarch64.hpp +new file mode 100644 +index 000000000..c55b7b17c +--- /dev/null ++++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zPhysicalMemoryBacking_linux_aarch64.hpp +@@ -0,0 +1,65 @@ ++/* ++ * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. ++ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++ * ++ * This code is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License version 2 only, as ++ * published by the Free Software Foundation. ++ * ++ * This code is distributed in the hope that it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++ * version 2 for more details (a copy is included in the LICENSE file that ++ * accompanied this code). ++ * ++ * You should have received a copy of the GNU General Public License version ++ * 2 along with this work; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++ * or visit www.oracle.com if you need additional information or have any ++ * questions. ++ */ ++ ++#ifndef OS_CPU_LINUX_AARCH64_ZPHYSICALMEMORYBACKING_LINUX_AARCH64_HPP ++#define OS_CPU_LINUX_AARCH64_ZPHYSICALMEMORYBACKING_LINUX_AARCH64_HPP ++ ++#include "gc/z/zBackingFile_linux_aarch64.hpp" ++#include "gc/z/zMemory.hpp" ++ ++class ZErrno; ++class ZPhysicalMemory; ++ ++class ZPhysicalMemoryBacking { ++private: ++ ZMemoryManager _manager; ++ ZBackingFile _file; ++ const size_t _granule_size; ++ ++ void check_max_map_count(size_t max_capacity, size_t granule_size) const; ++ void check_available_space_on_filesystem(size_t max_capacity) const; ++ void map_failed(ZErrno err) const; ++ ++ void advise_view(uintptr_t addr, size_t size) const; ++ void pretouch_view(uintptr_t addr, size_t size) const; ++ void map_view(ZPhysicalMemory pmem, uintptr_t addr, bool pretouch) const; ++ void unmap_view(ZPhysicalMemory pmem, uintptr_t addr) const; ++ ++public: ++ ZPhysicalMemoryBacking(size_t max_capacity, size_t granule_size); ++ ++ bool is_initialized() const; ++ ++ size_t try_expand(size_t old_capacity, size_t new_capacity); ++ ++ ZPhysicalMemory alloc(size_t size); ++ void free(ZPhysicalMemory pmem); ++ ++ uintptr_t nmt_address(uintptr_t offset) const; ++ ++ void map(ZPhysicalMemory pmem, uintptr_t offset) const; ++ void unmap(ZPhysicalMemory pmem, uintptr_t offset) const; ++ void flip(ZPhysicalMemory pmem, uintptr_t offset) const; ++}; ++ ++#endif // OS_CPU_LINUX_AARCH64_ZPHYSICALMEMORYBACKING_LINUX_AARCH64_HPP +diff --git a/src/hotspot/os_cpu/linux_aarch64/gc/z/zVirtualMemory_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/gc/z/zVirtualMemory_linux_aarch64.cpp +new file mode 100644 +index 000000000..68df40191 +--- /dev/null ++++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zVirtualMemory_linux_aarch64.cpp +@@ -0,0 +1,41 @@ ++/* ++ * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. ++ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++ * ++ * This code is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License version 2 only, as ++ * published by the Free Software Foundation. ++ * ++ * This code is distributed in the hope that it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++ * version 2 for more details (a copy is included in the LICENSE file that ++ * accompanied this code). ++ * ++ * You should have received a copy of the GNU General Public License version ++ * 2 along with this work; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++ * or visit www.oracle.com if you need additional information or have any ++ * questions. ++ */ ++ ++#include "precompiled.hpp" ++#include "gc/z/zVirtualMemory.hpp" ++#include "logging/log.hpp" ++ ++#include ++#include ++ ++bool ZVirtualMemoryManager::reserve(uintptr_t start, size_t size) { ++ // Reserve address space ++ const uintptr_t actual_start = (uintptr_t)mmap((void*)start, size, PROT_NONE, ++ MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0); ++ if (actual_start != start) { ++ log_error(gc)("Failed to reserve address space for Java heap"); ++ return false; ++ } ++ ++ return true; ++} +diff --git a/src/hotspot/share/gc/z/c1/zBarrierSetC1.cpp b/src/hotspot/share/gc/z/c1/zBarrierSetC1.cpp +index a1b43005c..f6f48268d 100644 +--- a/src/hotspot/share/gc/z/c1/zBarrierSetC1.cpp ++++ b/src/hotspot/share/gc/z/c1/zBarrierSetC1.cpp +@@ -39,21 +39,15 @@ ZLoadBarrierStubC1::ZLoadBarrierStubC1(LIRAccess& access, LIR_Opr ref, address r + _patch_info(access.patch_emit_info()), + _runtime_stub(runtime_stub) { + ++ assert(_ref->is_register(), "Must be a register"); ++ assert(_ref_addr->is_address(), "Must be an address"); ++ + // Allocate tmp register if needed +- if (!_ref_addr->is_register()) { +- assert(_ref_addr->is_address(), "Must be an address"); +- if (_ref_addr->as_address_ptr()->index()->is_valid() || +- _ref_addr->as_address_ptr()->disp() != 0) { +- // Has index or displacement, need tmp register to load address into +- _tmp = access.gen()->new_pointer_register(); +- } else { +- // No index or displacement, address available in base register +- _ref_addr = _ref_addr->as_address_ptr()->base(); +- } ++ if (_ref_addr->as_address_ptr()->index()->is_valid() || ++ _ref_addr->as_address_ptr()->disp() != 0) { ++ // Has index or displacement, need tmp register to load address into ++ _tmp = access.gen()->new_pointer_register(); + } +- +- assert(_ref->is_register(), "Must be a register"); +- assert(_ref_addr->is_register() != _tmp->is_register(), "Only one should be a register"); + } + + DecoratorSet ZLoadBarrierStubC1::decorators() const { +-- +2.12.3 + diff --git a/8217856-ZGC-Break-out-C2-matching-rules-into-separat.patch b/8217856-ZGC-Break-out-C2-matching-rules-into-separat.patch new file mode 100644 index 0000000..1fa22ee --- /dev/null +++ b/8217856-ZGC-Break-out-C2-matching-rules-into-separat.patch @@ -0,0 +1,924 @@ +From 12c28e19bf70b610a723828ce8395902381436fb Mon Sep 17 00:00:00 2001 +Date: Tue, 18 Feb 2020 18:03:42 +0800 +Subject: [PATCH] 8217856: ZGC: Break out C2 matching rules into separate AD + file + +Summary: : +LLT: jtreg +Bug url: https://bugs.openjdk.java.net/browse/JDK-8217856 +--- + make/hotspot/gensrc/GensrcAdlc.gmk | 6 + + src/hotspot/cpu/aarch64/aarch64.ad | 244 --------------------------- + src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad | 268 ++++++++++++++++++++++++++++++ + src/hotspot/cpu/x86/gc/z/z_x86_64.ad | 168 +++++++++++++++++++ + src/hotspot/cpu/x86/x86_64.ad | 151 ----------------- + 5 files changed, 442 insertions(+), 395 deletions(-) + create mode 100644 src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad + create mode 100644 src/hotspot/cpu/x86/gc/z/z_x86_64.ad + +diff --git a/make/hotspot/gensrc/GensrcAdlc.gmk b/make/hotspot/gensrc/GensrcAdlc.gmk +index 92b030b69..1eaed25d9 100644 +--- a/make/hotspot/gensrc/GensrcAdlc.gmk ++++ b/make/hotspot/gensrc/GensrcAdlc.gmk +@@ -136,6 +136,12 @@ ifeq ($(call check-jvm-feature, compiler2), true) + $d/os_cpu/$(HOTSPOT_TARGET_OS)_$(HOTSPOT_TARGET_CPU_ARCH)/$(HOTSPOT_TARGET_OS)_$(HOTSPOT_TARGET_CPU_ARCH).ad \ + ))) + ++ ifeq ($(call check-jvm-feature, zgc), true) ++ AD_SRC_FILES += $(call uniq, $(wildcard $(foreach d, $(AD_SRC_ROOTS), \ ++ $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/z/z_$(HOTSPOT_TARGET_CPU).ad \ ++ ))) ++ endif ++ + SINGLE_AD_SRCFILE := $(ADLC_SUPPORT_DIR)/all-ad-src.ad + + INSERT_FILENAME_AWK_SCRIPT := \ +diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad +index 93dd78e51..3cca2da5b 100644 +--- a/src/hotspot/cpu/aarch64/aarch64.ad ++++ b/src/hotspot/cpu/aarch64/aarch64.ad +@@ -1128,13 +1128,6 @@ definitions %{ + int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST); + %} + +-source_hpp %{ +- +-#include "gc/z/c2/zBarrierSetC2.hpp" +-#include "gc/z/zThreadLocalData.hpp" +- +-%} +- + //----------SOURCE BLOCK------------------------------------------------------- + // This is a block of C++ code which provides values, functions, and + // definitions necessary in the rest of the architecture description +@@ -17487,243 +17480,6 @@ instruct vsrl2L_imm(vecX dst, vecX src, immI shift) %{ + ins_pipe(vshift128_imm); + %} + +-source %{ +- +-static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) { +- ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, weak); +- __ ldr(tmp, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); +- __ andr(tmp, tmp, ref); +- __ cbnz(tmp, *stub->entry()); +- __ bind(*stub->continuation()); +-} +- +-static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) { +- ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, false /* weak */); +- __ b(*stub->entry()); +- __ bind(*stub->continuation()); +-} +- +-%} +- +-// Load Pointer +-instruct zLoadP(iRegPNoSp dst, memory mem, rFlagsReg cr) +-%{ +- match(Set dst (LoadP mem)); +- predicate(UseZGC && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() == ZLoadBarrierStrong)); +- effect(TEMP dst, KILL cr); +- +- ins_cost(4 * INSN_COST); +- +- format %{ "ldr $dst, $mem" %} +- +- ins_encode %{ +- const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); +- __ ldr($dst$$Register, ref_addr); +- if (barrier_data() != ZLoadBarrierElided) { +- z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, false /* weak */); +- } +- %} +- +- ins_pipe(iload_reg_mem); +-%} +- +-// Load Weak Pointer +-instruct zLoadWeakP(iRegPNoSp dst, memory mem, rFlagsReg cr) +-%{ +- match(Set dst (LoadP mem)); +- predicate(UseZGC && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() == ZLoadBarrierWeak)); +- effect(TEMP dst, KILL cr); +- +- ins_cost(4 * INSN_COST); +- +- format %{ "ldr $dst, $mem" %} +- +- ins_encode %{ +- const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); +- __ ldr($dst$$Register, ref_addr); +- z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, true /* weak */); +- %} +- +- ins_pipe(iload_reg_mem); +-%} +- +-// Load Pointer Volatile +-instruct zLoadPVolatile(iRegPNoSp dst, indirect mem /* sync_memory */, rFlagsReg cr) +-%{ +- match(Set dst (LoadP mem)); +- predicate(UseZGC && needs_acquiring_load(n) && n->as_Load()->barrier_data() == ZLoadBarrierStrong); +- effect(TEMP dst, KILL cr); +- +- ins_cost(VOLATILE_REF_COST); +- +- format %{ "ldar $dst, $mem\t" %} +- +- ins_encode %{ +- __ ldar($dst$$Register, $mem$$Register); +- if (barrier_data() != ZLoadBarrierElided) { +- z_load_barrier(_masm, this, Address($mem$$Register), $dst$$Register, rscratch2 /* tmp */, false /* weak */); +- } +- %} +- +- ins_pipe(pipe_serial); +-%} +- +-instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ +- match(Set res (CompareAndSwapP mem (Binary oldval newval))); +- match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); +- predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); +- effect(KILL cr, TEMP_DEF res); +- +- ins_cost(2 * VOLATILE_REF_COST); +- +- format %{ "cmpxchg $mem, $oldval, $newval\n\t" +- "cset $res, EQ" %} +- +- ins_encode %{ +- guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); +- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, +- false /* acquire */, true /* release */, false /* weak */, rscratch2); +- __ cset($res$$Register, Assembler::EQ); +- if (barrier_data() != ZLoadBarrierElided) { +- Label good; +- __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); +- __ andr(rscratch1, rscratch1, rscratch2); +- __ cbz(rscratch1, good); +- z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */); +- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, +- false /* acquire */, true /* release */, false /* weak */, rscratch2); +- __ cset($res$$Register, Assembler::EQ); +- __ bind(good); +- } +- %} +- +- ins_pipe(pipe_slow); +-%} +- +-instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ +- match(Set res (CompareAndSwapP mem (Binary oldval newval))); +- match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); +- predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)); +- effect(KILL cr, TEMP_DEF res); +- +- ins_cost(2 * VOLATILE_REF_COST); +- +- format %{ "cmpxchg $mem, $oldval, $newval\n\t" +- "cset $res, EQ" %} +- +- ins_encode %{ +- guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); +- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, +- true /* acquire */, true /* release */, false /* weak */, rscratch2); +- __ cset($res$$Register, Assembler::EQ); +- if (barrier_data() != ZLoadBarrierElided) { +- Label good; +- __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); +- __ andr(rscratch1, rscratch1, rscratch2); +- __ cbz(rscratch1, good); +- z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */ ); +- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, +- true /* acquire */, true /* release */, false /* weak */, rscratch2); +- __ cset($res$$Register, Assembler::EQ); +- __ bind(good); +- } +- %} +- +- ins_pipe(pipe_slow); +-%} +- +-instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ +- match(Set res (CompareAndExchangeP mem (Binary oldval newval))); +- predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); +- effect(TEMP_DEF res, KILL cr); +- +- ins_cost(2 * VOLATILE_REF_COST); +- +- format %{ "cmpxchg $res = $mem, $oldval, $newval" %} +- +- ins_encode %{ +- guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); +- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, +- false /* acquire */, true /* release */, false /* weak */, $res$$Register); +- if (barrier_data() != ZLoadBarrierElided) { +- Label good; +- __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); +- __ andr(rscratch1, rscratch1, $res$$Register); +- __ cbz(rscratch1, good); +- z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */); +- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, +- false /* acquire */, true /* release */, false /* weak */, $res$$Register); +- __ bind(good); +- } +- %} +- +- ins_pipe(pipe_slow); +-%} +- +-instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ +- match(Set res (CompareAndExchangeP mem (Binary oldval newval))); +- predicate(UseZGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); +- effect(TEMP_DEF res, KILL cr); +- +- ins_cost(2 * VOLATILE_REF_COST); +- +- format %{ "cmpxchg $res = $mem, $oldval, $newval" %} +- +- ins_encode %{ +- guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); +- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, +- true /* acquire */, true /* release */, false /* weak */, $res$$Register); +- if (barrier_data() != ZLoadBarrierElided) { +- Label good; +- __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); +- __ andr(rscratch1, rscratch1, $res$$Register); +- __ cbz(rscratch1, good); +- z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */); +- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, +- true /* acquire */, true /* release */, false /* weak */, $res$$Register); +- __ bind(good); +- } +- %} +- +- ins_pipe(pipe_slow); +-%} +- +-instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{ +- match(Set prev (GetAndSetP mem newv)); +- predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); +- effect(TEMP_DEF prev, KILL cr); +- +- ins_cost(2 * VOLATILE_REF_COST); +- +- format %{ "atomic_xchg $prev, $newv, [$mem]" %} +- +- ins_encode %{ +- __ atomic_xchg($prev$$Register, $newv$$Register, $mem$$Register); +- if (barrier_data() != ZLoadBarrierElided) { +- z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, false /* weak */); +- } +- %} +- +- ins_pipe(pipe_serial); +-%} +- +-instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{ +- match(Set prev (GetAndSetP mem newv)); +- predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)); +- effect(TEMP_DEF prev, KILL cr); +- +- ins_cost(VOLATILE_REF_COST); +- +- format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %} +- +- ins_encode %{ +- __ atomic_xchgal($prev$$Register, $newv$$Register, $mem$$Register); +- if (barrier_data() != ZLoadBarrierElided) { +- z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, false /* weak */); +- } +- %} +- ins_pipe(pipe_serial); +-%} + + //----------PEEPHOLE RULES----------------------------------------------------- + // These must follow all instruction definitions as they use the names +diff --git a/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad b/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad +new file mode 100644 +index 000000000..50cc6f924 +--- /dev/null ++++ b/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad +@@ -0,0 +1,268 @@ ++// ++// Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. ++// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++// ++// This code is free software; you can redistribute it and/or modify it ++// under the terms of the GNU General Public License version 2 only, as ++// published by the Free Software Foundation. ++// ++// This code is distributed in the hope that it will be useful, but WITHOUT ++// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++// version 2 for more details (a copy is included in the LICENSE file that ++// accompanied this code). ++// ++// You should have received a copy of the GNU General Public License version ++// 2 along with this work; if not, write to the Free Software Foundation, ++// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++// ++// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++// or visit www.oracle.com if you need additional information or have any ++// questions. ++// ++ ++source_hpp %{ ++ ++#include "gc/z/c2/zBarrierSetC2.hpp" ++#include "gc/z/zThreadLocalData.hpp" ++ ++%} ++ ++source %{ ++ ++static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) { ++ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, weak); ++ __ ldr(tmp, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); ++ __ andr(tmp, tmp, ref); ++ __ cbnz(tmp, *stub->entry()); ++ __ bind(*stub->continuation()); ++} ++ ++static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) { ++ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, false /* weak */); ++ __ b(*stub->entry()); ++ __ bind(*stub->continuation()); ++} ++ ++%} ++ ++// Load Pointer ++instruct zLoadP(iRegPNoSp dst, memory mem, rFlagsReg cr) ++%{ ++ match(Set dst (LoadP mem)); ++ predicate(UseZGC && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() == ZLoadBarrierStrong)); ++ effect(TEMP dst, KILL cr); ++ ++ ins_cost(4 * INSN_COST); ++ ++ format %{ "ldr $dst, $mem" %} ++ ++ ins_encode %{ ++ const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); ++ __ ldr($dst$$Register, ref_addr); ++ if (barrier_data() != ZLoadBarrierElided) { ++ z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, false /* weak */); ++ } ++ %} ++ ++ ins_pipe(iload_reg_mem); ++%} ++ ++// Load Weak Pointer ++instruct zLoadWeakP(iRegPNoSp dst, memory mem, rFlagsReg cr) ++%{ ++ match(Set dst (LoadP mem)); ++ predicate(UseZGC && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() == ZLoadBarrierWeak)); ++ effect(TEMP dst, KILL cr); ++ ++ ins_cost(4 * INSN_COST); ++ ++ format %{ "ldr $dst, $mem" %} ++ ++ ins_encode %{ ++ const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); ++ __ ldr($dst$$Register, ref_addr); ++ z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, true /* weak */); ++ %} ++ ++ ins_pipe(iload_reg_mem); ++%} ++ ++// Load Pointer Volatile ++instruct zLoadPVolatile(iRegPNoSp dst, indirect mem /* sync_memory */, rFlagsReg cr) ++%{ ++ match(Set dst (LoadP mem)); ++ predicate(UseZGC && needs_acquiring_load(n) && n->as_Load()->barrier_data() == ZLoadBarrierStrong); ++ effect(TEMP dst, KILL cr); ++ ++ ins_cost(VOLATILE_REF_COST); ++ ++ format %{ "ldar $dst, $mem\t" %} ++ ++ ins_encode %{ ++ __ ldar($dst$$Register, $mem$$Register); ++ if (barrier_data() != ZLoadBarrierElided) { ++ z_load_barrier(_masm, this, Address($mem$$Register), $dst$$Register, rscratch2 /* tmp */, false /* weak */); ++ } ++ %} ++ ++ ins_pipe(pipe_serial); ++%} ++ ++instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ ++ match(Set res (CompareAndSwapP mem (Binary oldval newval))); ++ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); ++ predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); ++ effect(KILL cr, TEMP_DEF res); ++ ++ ins_cost(2 * VOLATILE_REF_COST); ++ ++ format %{ "cmpxchg $mem, $oldval, $newval\n\t" ++ "cset $res, EQ" %} ++ ++ ins_encode %{ ++ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ false /* acquire */, true /* release */, false /* weak */, rscratch2); ++ __ cset($res$$Register, Assembler::EQ); ++ if (barrier_data() != ZLoadBarrierElided) { ++ Label good; ++ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); ++ __ andr(rscratch1, rscratch1, rscratch2); ++ __ cbz(rscratch1, good); ++ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ false /* acquire */, true /* release */, false /* weak */, rscratch2); ++ __ cset($res$$Register, Assembler::EQ); ++ __ bind(good); ++ } ++ %} ++ ++ ins_pipe(pipe_slow); ++%} ++ ++instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ ++ match(Set res (CompareAndSwapP mem (Binary oldval newval))); ++ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); ++ predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)); ++ effect(KILL cr, TEMP_DEF res); ++ ++ ins_cost(2 * VOLATILE_REF_COST); ++ ++ format %{ "cmpxchg $mem, $oldval, $newval\n\t" ++ "cset $res, EQ" %} ++ ++ ins_encode %{ ++ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ true /* acquire */, true /* release */, false /* weak */, rscratch2); ++ __ cset($res$$Register, Assembler::EQ); ++ if (barrier_data() != ZLoadBarrierElided) { ++ Label good; ++ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); ++ __ andr(rscratch1, rscratch1, rscratch2); ++ __ cbz(rscratch1, good); ++ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */ ); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ true /* acquire */, true /* release */, false /* weak */, rscratch2); ++ __ cset($res$$Register, Assembler::EQ); ++ __ bind(good); ++ } ++ %} ++ ++ ins_pipe(pipe_slow); ++%} ++ ++instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ ++ match(Set res (CompareAndExchangeP mem (Binary oldval newval))); ++ predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); ++ effect(TEMP_DEF res, KILL cr); ++ ++ ins_cost(2 * VOLATILE_REF_COST); ++ ++ format %{ "cmpxchg $res = $mem, $oldval, $newval" %} ++ ++ ins_encode %{ ++ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ false /* acquire */, true /* release */, false /* weak */, $res$$Register); ++ if (barrier_data() != ZLoadBarrierElided) { ++ Label good; ++ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); ++ __ andr(rscratch1, rscratch1, $res$$Register); ++ __ cbz(rscratch1, good); ++ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ false /* acquire */, true /* release */, false /* weak */, $res$$Register); ++ __ bind(good); ++ } ++ %} ++ ++ ins_pipe(pipe_slow); ++%} ++ ++instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ ++ match(Set res (CompareAndExchangeP mem (Binary oldval newval))); ++ predicate(UseZGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); ++ effect(TEMP_DEF res, KILL cr); ++ ++ ins_cost(2 * VOLATILE_REF_COST); ++ ++ format %{ "cmpxchg $res = $mem, $oldval, $newval" %} ++ ++ ins_encode %{ ++ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ true /* acquire */, true /* release */, false /* weak */, $res$$Register); ++ if (barrier_data() != ZLoadBarrierElided) { ++ Label good; ++ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); ++ __ andr(rscratch1, rscratch1, $res$$Register); ++ __ cbz(rscratch1, good); ++ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ true /* acquire */, true /* release */, false /* weak */, $res$$Register); ++ __ bind(good); ++ } ++ %} ++ ++ ins_pipe(pipe_slow); ++%} ++ ++instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{ ++ match(Set prev (GetAndSetP mem newv)); ++ predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); ++ effect(TEMP_DEF prev, KILL cr); ++ ++ ins_cost(2 * VOLATILE_REF_COST); ++ ++ format %{ "atomic_xchg $prev, $newv, [$mem]" %} ++ ++ ins_encode %{ ++ __ atomic_xchg($prev$$Register, $newv$$Register, $mem$$Register); ++ if (barrier_data() != ZLoadBarrierElided) { ++ z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, false /* weak */); ++ } ++ %} ++ ++ ins_pipe(pipe_serial); ++%} ++ ++instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{ ++ match(Set prev (GetAndSetP mem newv)); ++ predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)); ++ effect(TEMP_DEF prev, KILL cr); ++ ++ ins_cost(VOLATILE_REF_COST); ++ ++ format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %} ++ ++ ins_encode %{ ++ __ atomic_xchgal($prev$$Register, $newv$$Register, $mem$$Register); ++ if (barrier_data() != ZLoadBarrierElided) { ++ z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, false /* weak */); ++ } ++ %} ++ ins_pipe(pipe_serial); ++%} ++ +diff --git a/src/hotspot/cpu/x86/gc/z/z_x86_64.ad b/src/hotspot/cpu/x86/gc/z/z_x86_64.ad +new file mode 100644 +index 000000000..38c2e926b +--- /dev/null ++++ b/src/hotspot/cpu/x86/gc/z/z_x86_64.ad +@@ -0,0 +1,168 @@ ++// ++// Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. ++// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++// ++// This code is free software; you can redistribute it and/or modify it ++// under the terms of the GNU General Public License version 2 only, as ++// published by the Free Software Foundation. ++// ++// This code is distributed in the hope that it will be useful, but WITHOUT ++// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++// version 2 for more details (a copy is included in the LICENSE file that ++// accompanied this code). ++// ++// You should have received a copy of the GNU General Public License version ++// 2 along with this work; if not, write to the Free Software Foundation, ++// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++// ++// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++// or visit www.oracle.com if you need additional information or have any ++// questions. ++// ++ ++source_hpp %{ ++ ++#include "gc/z/c2/zBarrierSetC2.hpp" ++#include "gc/z/zThreadLocalData.hpp" ++ ++%} ++ ++source %{ ++ ++static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) { ++ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, weak); ++ __ testptr(ref, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); ++ __ jcc(Assembler::notZero, *stub->entry()); ++ __ bind(*stub->continuation()); ++} ++ ++static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) { ++ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, false /* weak */); ++ __ jmp(*stub->entry()); ++ __ bind(*stub->continuation()); ++} ++ ++%} ++ ++// Load Pointer ++instruct zLoadP(rRegP dst, memory mem, rFlagsReg cr) ++%{ ++ predicate(UseZGC && n->as_Load()->barrier_data() == ZLoadBarrierStrong); ++ match(Set dst (LoadP mem)); ++ effect(KILL cr, TEMP dst); ++ ++ ins_cost(125); ++ ++ format %{ "movq $dst, $mem" %} ++ ++ ins_encode %{ ++ __ movptr($dst$$Register, $mem$$Address); ++ if (barrier_data() != ZLoadBarrierElided) { ++ z_load_barrier(_masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, false /* weak */); ++ } ++ %} ++ ++ ins_pipe(ialu_reg_mem); ++%} ++ ++// Load Weak Pointer ++instruct zLoadWeakP(rRegP dst, memory mem, rFlagsReg cr) ++%{ ++ predicate(UseZGC && n->as_Load()->barrier_data() == ZLoadBarrierWeak); ++ match(Set dst (LoadP mem)); ++ effect(KILL cr, TEMP dst); ++ ++ ins_cost(125); ++ ++ format %{ "movq $dst, $mem" %} ++ ++ ins_encode %{ ++ __ movptr($dst$$Register, $mem$$Address); ++ z_load_barrier(_masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, true /* weak */); ++ %} ++ ++ ins_pipe(ialu_reg_mem); ++%} ++ ++instruct zCompareAndExchangeP(memory mem, rax_RegP oldval, rRegP newval, rRegP tmp, rFlagsReg cr) %{ ++ match(Set oldval (CompareAndExchangeP mem (Binary oldval newval))); ++ predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); ++ effect(KILL cr, TEMP tmp); ++ ++ format %{ "lock\n\t" ++ "cmpxchgq $newval, $mem" %} ++ ++ ins_encode %{ ++ if (barrier_data() != ZLoadBarrierElided) { ++ __ movptr($tmp$$Register, $oldval$$Register); ++ } ++ __ lock(); ++ __ cmpxchgptr($newval$$Register, $mem$$Address); ++ if (barrier_data() != ZLoadBarrierElided) { ++ Label good; ++ __ testptr($oldval$$Register, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); ++ __ jcc(Assembler::zero, good); ++ z_load_barrier_slow_path(_masm, this, $mem$$Address, $oldval$$Register, $tmp$$Register); ++ __ movptr($oldval$$Register, $tmp$$Register); ++ __ lock(); ++ __ cmpxchgptr($newval$$Register, $mem$$Address); ++ __ bind(good); ++ } ++ %} ++ ++ ins_pipe(pipe_cmpxchg); ++%} ++ ++instruct zCompareAndSwapP(rRegI res, memory mem, rRegP newval, rRegP tmp, rFlagsReg cr, rax_RegP oldval) %{ ++ match(Set res (CompareAndSwapP mem (Binary oldval newval))); ++ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); ++ predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); ++ effect(KILL cr, KILL oldval, TEMP tmp); ++ ++ format %{ "lock\n\t" ++ "cmpxchgq $newval, $mem\n\t" ++ "sete $res\n\t" ++ "movzbl $res, $res" %} ++ ++ ins_encode %{ ++ if (barrier_data() != ZLoadBarrierElided) { ++ __ movptr($tmp$$Register, $oldval$$Register); ++ } ++ __ lock(); ++ __ cmpxchgptr($newval$$Register, $mem$$Address); ++ if (barrier_data() != ZLoadBarrierElided) { ++ Label good; ++ __ testptr($oldval$$Register, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); ++ __ jcc(Assembler::zero, good); ++ z_load_barrier_slow_path(_masm, this, $mem$$Address, $oldval$$Register, $tmp$$Register); ++ __ movptr($oldval$$Register, $tmp$$Register); ++ __ lock(); ++ __ cmpxchgptr($newval$$Register, $mem$$Address); ++ __ bind(good); ++ __ cmpptr($tmp$$Register, $oldval$$Register); ++ } ++ __ setb(Assembler::equal, $res$$Register); ++ __ movzbl($res$$Register, $res$$Register); ++ %} ++ ++ ins_pipe(pipe_cmpxchg); ++%} ++ ++instruct zXChgP(memory mem, rRegP newval, rFlagsReg cr) %{ ++ match(Set newval (GetAndSetP mem newval)); ++ predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); ++ effect(KILL cr); ++ ++ format %{ "xchgq $newval, $mem" %} ++ ++ ins_encode %{ ++ __ xchgptr($newval$$Register, $mem$$Address); ++ if (barrier_data() != ZLoadBarrierElided) { ++ z_load_barrier(_masm, this, Address(noreg, 0), $newval$$Register, noreg /* tmp */, false /* weak */); ++ } ++ %} ++ ++ ins_pipe(pipe_cmpxchg); ++%} ++ +diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad +index f795fb823..7879547ce 100644 +--- a/src/hotspot/cpu/x86/x86_64.ad ++++ b/src/hotspot/cpu/x86/x86_64.ad +@@ -538,19 +538,6 @@ reg_class int_rdi_reg(RDI); + + %} + +-source_hpp %{ +- +-#include "gc/z/c2/zBarrierSetC2.hpp" +-#include "gc/z/zThreadLocalData.hpp" +- +-%} +- +-source_hpp %{ +-#if INCLUDE_ZGC +-#include "gc/z/zBarrierSetAssembler.hpp" +-#endif +-%} +- + //----------SOURCE BLOCK------------------------------------------------------- + // This is a block of C++ code which provides values, functions, and + // definitions necessary in the rest of the architecture description +@@ -1801,19 +1788,6 @@ const RegMask Matcher::method_handle_invoke_SP_save_mask() { + return NO_REG_mask(); + } + +-static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) { +- ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, weak); +- __ testptr(ref, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); +- __ jcc(Assembler::notZero, *stub->entry()); +- __ bind(*stub->continuation()); +-} +- +-static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) { +- ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, false /* weak */); +- __ jmp(*stub->entry()); +- __ bind(*stub->continuation()); +-} +- + %} + + //----------ENCODING BLOCK----------------------------------------------------- +@@ -12520,131 +12494,6 @@ instruct RethrowException() + ins_pipe(pipe_jmp); + %} + +-// +-// Execute ZGC load barrier (strong) slow path +-// +- +-// Load Pointer +-instruct zLoadP(rRegP dst, memory mem, rFlagsReg cr) +-%{ +- predicate(UseZGC && n->as_Load()->barrier_data() == ZLoadBarrierStrong); +- match(Set dst (LoadP mem)); +- effect(KILL cr, TEMP dst); +- +- ins_cost(125); +- +- format %{ "movq $dst, $mem" %} +- +- ins_encode %{ +- __ movptr($dst$$Register, $mem$$Address); +- if (barrier_data() != ZLoadBarrierElided) { +- z_load_barrier(_masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, false /* weak */); +- } +- %} +- +- ins_pipe(ialu_reg_mem); +-%} +- +-// Load Weak Pointer +-instruct zLoadWeakP(rRegP dst, memory mem, rFlagsReg cr) +-%{ +- predicate(UseZGC && n->as_Load()->barrier_data() == ZLoadBarrierWeak); +- match(Set dst (LoadP mem)); +- effect(KILL cr, TEMP dst); +- +- ins_cost(125); +- +- format %{ "movq $dst, $mem" %} +- ins_encode %{ +- __ movptr($dst$$Register, $mem$$Address); +- z_load_barrier(_masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, true /* weak */); +- %} +- +- ins_pipe(ialu_reg_mem); +-%} +- +-instruct zCompareAndExchangeP(memory mem, rax_RegP oldval, rRegP newval, rRegP tmp, rFlagsReg cr) %{ +- match(Set oldval (CompareAndExchangeP mem (Binary oldval newval))); +- predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); +- effect(KILL cr, TEMP tmp); +- +- format %{ "lock\n\t" +- "cmpxchgq $newval, $mem" %} +- +- ins_encode %{ +- if (barrier_data() != ZLoadBarrierElided) { +- __ movptr($tmp$$Register, $oldval$$Register); +- } +- __ lock(); +- __ cmpxchgptr($newval$$Register, $mem$$Address); +- if (barrier_data() != ZLoadBarrierElided) { +- Label good; +- __ testptr($oldval$$Register, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); +- __ jcc(Assembler::zero, good); +- z_load_barrier_slow_path(_masm, this, $mem$$Address, $oldval$$Register, $tmp$$Register); +- __ movptr($oldval$$Register, $tmp$$Register); +- __ lock(); +- __ cmpxchgptr($newval$$Register, $mem$$Address); +- __ bind(good); +- } +- %} +- +- ins_pipe(pipe_cmpxchg); +-%} +- +- +-instruct zCompareAndSwapP(rRegI res, memory mem, rRegP newval, rRegP tmp, rFlagsReg cr, rax_RegP oldval) %{ +- match(Set res (CompareAndSwapP mem (Binary oldval newval))); +- match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); +- predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); +- effect(KILL cr, KILL oldval, TEMP tmp); +- +- format %{ "lock\n\t" +- "cmpxchgq $newval, $mem\n\t" +- "sete $res\n\t" +- "movzbl $res, $res" %} +- +- ins_encode %{ +- if (barrier_data() != ZLoadBarrierElided) { +- __ movptr($tmp$$Register, $oldval$$Register); +- } +- __ lock(); +- __ cmpxchgptr($newval$$Register, $mem$$Address); +- if (barrier_data() != ZLoadBarrierElided) { +- Label good; +- __ testptr($oldval$$Register, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); +- __ jcc(Assembler::zero, good); +- z_load_barrier_slow_path(_masm, this, $mem$$Address, $oldval$$Register, $tmp$$Register); +- __ movptr($oldval$$Register, $tmp$$Register); +- __ lock(); +- __ cmpxchgptr($newval$$Register, $mem$$Address); +- __ bind(good); +- __ cmpptr($tmp$$Register, $oldval$$Register); +- } +- __ setb(Assembler::equal, $res$$Register); +- __ movzbl($res$$Register, $res$$Register); +- %} +- +- ins_pipe(pipe_cmpxchg); +-%} +- +-instruct zXChgP(memory mem, rRegP newval, rFlagsReg cr) %{ +- match(Set newval (GetAndSetP mem newval)); +- predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); +- effect(KILL cr); +- +- format %{ "xchgq $newval, $mem" %} +- +- ins_encode %{ +- __ xchgptr($newval$$Register, $mem$$Address); +- if (barrier_data() != ZLoadBarrierElided) { +- z_load_barrier(_masm, this, Address(noreg, 0), $newval$$Register, noreg /* tmp */, false /* weak */); +- } +- %} +- +- ins_pipe(pipe_cmpxchg); +-%} +- + // ============================================================================ + // This name is KNOWN by the ADLC and cannot be changed. + // The ADLC forces a 'TypeRawPtr::BOTTOM' output type +-- +2.12.3 + diff --git a/8224675-Late-GC-barrier-insertion-for-ZGC.patch b/8224675-Late-GC-barrier-insertion-for-ZGC.patch new file mode 100644 index 0000000..b981c79 --- /dev/null +++ b/8224675-Late-GC-barrier-insertion-for-ZGC.patch @@ -0,0 +1,3343 @@ +From e26d6291e9ee09b805f9ee73b3a9a87335aea4c8 Mon Sep 17 00:00:00 2001 +Date: Mon, 6 Jan 2020 10:49:35 +0800 +Subject: [PATCH] 8224675: Late GC barrier insertion for ZGC + +Summary: : +LLT: jtreg +Bug url: https://bugs.openjdk.java.net/browse/JDK-8224675 +--- + src/hotspot/cpu/aarch64/aarch64.ad | 60 +- + .../aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp | 1 - + src/hotspot/cpu/x86/x86_64.ad | 77 +- + src/hotspot/share/adlc/formssel.cpp | 9 +- + src/hotspot/share/compiler/compilerDirectives.hpp | 2 +- + src/hotspot/share/gc/shared/c2/barrierSetC2.hpp | 4 +- + src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp | 1768 ++++++++++---------- + src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp | 116 +- + src/hotspot/share/gc/z/zHeap.cpp | 26 - + src/hotspot/share/gc/z/z_globals.hpp | 3 - + src/hotspot/share/opto/classes.hpp | 5 +- + src/hotspot/share/opto/compile.cpp | 43 +- + src/hotspot/share/opto/compile.hpp | 23 +- + src/hotspot/share/opto/escape.cpp | 8 - + src/hotspot/share/opto/idealGraphPrinter.cpp | 18 +- + src/hotspot/share/opto/idealGraphPrinter.hpp | 14 +- + src/hotspot/share/opto/lcm.cpp | 1 - + src/hotspot/share/opto/loopnode.cpp | 43 +- + src/hotspot/share/opto/loopnode.hpp | 6 +- + src/hotspot/share/opto/loopopts.cpp | 6 - + src/hotspot/share/opto/matcher.cpp | 27 + + src/hotspot/share/opto/memnode.cpp | 21 +- + src/hotspot/share/opto/memnode.hpp | 11 +- + src/hotspot/share/opto/node.cpp | 13 +- + src/hotspot/share/opto/node.hpp | 7 +- + src/hotspot/share/opto/phaseX.cpp | 3 - + src/hotspot/share/opto/phasetype.hpp | 6 + + src/hotspot/share/opto/vectornode.cpp | 1 - + src/hotspot/share/runtime/stackValue.cpp | 8 +- + src/hotspot/share/utilities/growableArray.hpp | 37 + + 30 files changed, 1259 insertions(+), 1108 deletions(-) + +diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad +index d0f3fc33d..3b898f548 100644 +--- a/src/hotspot/cpu/aarch64/aarch64.ad ++++ b/src/hotspot/cpu/aarch64/aarch64.ad +@@ -17517,7 +17517,7 @@ instruct loadBarrierSlowReg(iRegP dst, memory mem, rFlagsReg cr, + vRegD_V25 v25, vRegD_V26 v26, vRegD_V27 v27, vRegD_V28 v28, vRegD_V29 v29, + vRegD_V30 v30, vRegD_V31 v31) %{ + match(Set dst (LoadBarrierSlowReg mem)); +- //predicate(!n->as_LoadBarrierSlowReg()->is_weak()); ++ predicate(!n->as_LoadBarrierSlowReg()->is_weak()); + + effect(DEF dst, KILL cr, + KILL v0, KILL v1, KILL v2, KILL v3, KILL v4, KILL v5, KILL v6, KILL v7, +@@ -17546,7 +17546,7 @@ instruct loadBarrierWeakSlowReg(iRegP dst, memory mem, rFlagsReg cr, + vRegD_V25 v25, vRegD_V26 v26, vRegD_V27 v27, vRegD_V28 v28, vRegD_V29 v29, + vRegD_V30 v30, vRegD_V31 v31) %{ + match(Set dst (LoadBarrierSlowReg mem)); +- //predicate(n->as_LoadBarrierSlowReg()->is_weak()); ++ predicate(n->as_LoadBarrierSlowReg()->is_weak()); + + effect(DEF dst, KILL cr, + KILL v0, KILL v1, KILL v2, KILL v3, KILL v4, KILL v5, KILL v6, KILL v7, +@@ -17563,6 +17563,62 @@ instruct loadBarrierWeakSlowReg(iRegP dst, memory mem, rFlagsReg cr, + ins_pipe(pipe_slow); + %} + ++// Specialized versions of compareAndExchangeP that adds a keepalive that is consumed ++// but doesn't affect output. ++ ++instruct z_compareAndExchangeP(iRegPNoSp res, indirect mem, ++ iRegP oldval, iRegP newval, iRegP keepalive, ++ rFlagsReg cr) %{ ++ match(Set res (ZCompareAndExchangeP (Binary mem keepalive) (Binary oldval newval))); ++ ins_cost(2 * VOLATILE_REF_COST); ++ effect(TEMP_DEF res, KILL cr); ++ format %{ ++ "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval" ++ %} ++ ins_encode %{ ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, ++ Assembler::xword, /*acquire*/ false, /*release*/ true, ++ /*weak*/ false, $res$$Register); ++ %} ++ ins_pipe(pipe_slow); ++%} ++ ++instruct z_compareAndSwapP(iRegINoSp res, ++ indirect mem, ++ iRegP oldval, iRegP newval, iRegP keepalive, ++ rFlagsReg cr) %{ ++ ++ match(Set res (ZCompareAndSwapP (Binary mem keepalive) (Binary oldval newval))); ++ match(Set res (ZWeakCompareAndSwapP (Binary mem keepalive) (Binary oldval newval))); ++ ++ ins_cost(2 * VOLATILE_REF_COST); ++ ++ effect(KILL cr); ++ ++ format %{ ++ "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval" ++ "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" ++ %} ++ ++ ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval), ++ aarch64_enc_cset_eq(res)); ++ ++ ins_pipe(pipe_slow); ++%} ++ ++ ++instruct z_get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev, ++ iRegP keepalive) %{ ++ match(Set prev (ZGetAndSetP mem (Binary newv keepalive))); ++ ++ ins_cost(2 * VOLATILE_REF_COST); ++ format %{ "atomic_xchg $prev, $newv, [$mem]" %} ++ ins_encode %{ ++ __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base)); ++ %} ++ ins_pipe(pipe_serial); ++%} ++ + //----------PEEPHOLE RULES----------------------------------------------------- + // These must follow all instruction definitions as they use the names + // defined in the instructions definitions. +diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp +index 90b2b4ca7..8e169ace4 100644 +--- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp ++++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp +@@ -245,7 +245,6 @@ void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce, + + if (stub->tmp()->is_valid()) { + // Load address into tmp register +- //ce->leal(stub->ref_addr(), stub->tmp()); + ce->leal(stub->ref_addr(), stub->tmp(), lir_patch_none, NULL); + ref_addr = tmp = stub->tmp()->as_pointer_register(); + } else { +diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad +index 9d9eda84b..97c396875 100644 +--- a/src/hotspot/cpu/x86/x86_64.ad ++++ b/src/hotspot/cpu/x86/x86_64.ad +@@ -539,6 +539,12 @@ reg_class int_rdi_reg(RDI); + %} + + source_hpp %{ ++ ++#include "gc/z/c2/zBarrierSetC2.hpp" ++ ++%} ++ ++source_hpp %{ + #if INCLUDE_ZGC + #include "gc/z/zBarrierSetAssembler.hpp" + #endif +@@ -12630,7 +12636,7 @@ instruct RethrowException() + instruct loadBarrierSlowRegNoVec(rRegP dst, memory mem, rFlagsReg cr) %{ + + match(Set dst (LoadBarrierSlowReg mem)); +- predicate(MaxVectorSize < 16); ++ predicate(MaxVectorSize < 16 && !n->as_LoadBarrierSlowReg()->is_weak()); + + effect(DEF dst, KILL cr); + +@@ -12661,7 +12667,7 @@ instruct loadBarrierSlowRegXmmAndYmm(rRegP dst, memory mem, rFlagsReg cr, + rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{ + + match(Set dst (LoadBarrierSlowReg mem)); +- predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16)); ++ predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16) && !n->as_LoadBarrierSlowReg()->is_weak()); + + effect(DEF dst, KILL cr, + KILL x0, KILL x1, KILL x2, KILL x3, +@@ -12700,7 +12706,7 @@ instruct loadBarrierSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr, + rxmm28 x28, rxmm29 x29, rxmm30 x30, rxmm31 x31) %{ + + match(Set dst (LoadBarrierSlowReg mem)); +- predicate((UseAVX == 3) && (MaxVectorSize >= 16)); ++ predicate((UseAVX == 3) && (MaxVectorSize >= 16) && !n->as_LoadBarrierSlowReg()->is_weak()); + + effect(DEF dst, KILL cr, + KILL x0, KILL x1, KILL x2, KILL x3, +@@ -12739,7 +12745,7 @@ instruct loadBarrierSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr, + instruct loadBarrierWeakSlowRegNoVec(rRegP dst, memory mem, rFlagsReg cr) %{ + + match(Set dst (LoadBarrierSlowReg mem)); +- predicate(MaxVectorSize < 16); ++ predicate(MaxVectorSize < 16 && n->as_LoadBarrierSlowReg()->is_weak()); + + effect(DEF dst, KILL cr); + +@@ -12769,8 +12775,8 @@ instruct loadBarrierWeakSlowRegXmmAndYmm(rRegP dst, memory mem, rFlagsReg cr, + rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11, + rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{ + +- match(Set dst (LoadBarrierWeakSlowReg mem)); +- predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16)); ++ match(Set dst (LoadBarrierSlowReg mem)); ++ predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16) && n->as_LoadBarrierSlowReg()->is_weak()); + + effect(DEF dst, KILL cr, + KILL x0, KILL x1, KILL x2, KILL x3, +@@ -12808,8 +12814,8 @@ instruct loadBarrierWeakSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr, + rxmm24 x24, rxmm25 x25, rxmm26 x26, rxmm27 x27, + rxmm28 x28, rxmm29 x29, rxmm30 x30, rxmm31 x31) %{ + +- match(Set dst (LoadBarrierWeakSlowReg mem)); +- predicate((UseAVX == 3) && (MaxVectorSize >= 16)); ++ match(Set dst (LoadBarrierSlowReg mem)); ++ predicate((UseAVX == 3) && (MaxVectorSize >= 16) && n->as_LoadBarrierSlowReg()->is_weak()); + + effect(DEF dst, KILL cr, + KILL x0, KILL x1, KILL x2, KILL x3, +@@ -12840,6 +12846,61 @@ instruct loadBarrierWeakSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr, + ins_pipe(pipe_slow); + %} + ++// Specialized versions of compareAndExchangeP that adds a keepalive that is consumed ++// but doesn't affect output. ++ ++instruct z_compareAndExchangeP( ++ memory mem_ptr, ++ rax_RegP oldval, rRegP newval, rRegP keepalive, ++ rFlagsReg cr) %{ ++ predicate(VM_Version::supports_cx8()); ++ match(Set oldval (ZCompareAndExchangeP (Binary mem_ptr keepalive) (Binary oldval newval))); ++ effect(KILL cr); ++ ++ format %{ "cmpxchgq $mem_ptr,$newval\t# " ++ "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %} ++ opcode(0x0F, 0xB1); ++ ins_encode(lock_prefix, ++ REX_reg_mem_wide(newval, mem_ptr), ++ OpcP, OpcS, ++ reg_mem(newval, mem_ptr) // lock cmpxchg ++ ); ++ ins_pipe( pipe_cmpxchg ); ++%} ++ ++instruct z_compareAndSwapP(rRegI res, ++ memory mem_ptr, ++ rax_RegP oldval, rRegP newval, rRegP keepalive, ++ rFlagsReg cr) %{ ++ predicate(VM_Version::supports_cx8()); ++ match(Set res (ZCompareAndSwapP (Binary mem_ptr keepalive) (Binary oldval newval))); ++ match(Set res (ZWeakCompareAndSwapP (Binary mem_ptr keepalive) (Binary oldval newval))); ++ effect(KILL cr, KILL oldval); ++ ++ format %{ "cmpxchgq $mem_ptr,$newval\t# " ++ "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" ++ "sete $res\n\t" ++ "movzbl $res, $res" %} ++ opcode(0x0F, 0xB1); ++ ins_encode(lock_prefix, ++ REX_reg_mem_wide(newval, mem_ptr), ++ OpcP, OpcS, ++ reg_mem(newval, mem_ptr), ++ REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete ++ REX_reg_breg(res, res), // movzbl ++ Opcode(0xF), Opcode(0xB6), reg_reg(res, res)); ++ ins_pipe( pipe_cmpxchg ); ++%} ++ ++instruct z_xchgP( memory mem, rRegP newval, rRegP keepalive) %{ ++ match(Set newval (ZGetAndSetP mem (Binary newval keepalive))); ++ format %{ "XCHGQ $newval,[$mem]" %} ++ ins_encode %{ ++ __ xchgq($newval$$Register, $mem$$Address); ++ %} ++ ins_pipe( pipe_cmpxchg ); ++%} ++ + // ============================================================================ + // This name is KNOWN by the ADLC and cannot be changed. + // The ADLC forces a 'TypeRawPtr::BOTTOM' output type +diff --git a/src/hotspot/share/adlc/formssel.cpp b/src/hotspot/share/adlc/formssel.cpp +index 466880d19..2845cf6d8 100644 +--- a/src/hotspot/share/adlc/formssel.cpp ++++ b/src/hotspot/share/adlc/formssel.cpp +@@ -775,8 +775,9 @@ bool InstructForm::captures_bottom_type(FormDict &globals) const { + !strcmp(_matrule->_rChild->_opType,"GetAndSetP") || + !strcmp(_matrule->_rChild->_opType,"GetAndSetN") || + #if INCLUDE_ZGC ++ !strcmp(_matrule->_rChild->_opType,"ZGetAndSetP") || ++ !strcmp(_matrule->_rChild->_opType,"ZCompareAndExchangeP") || + !strcmp(_matrule->_rChild->_opType,"LoadBarrierSlowReg") || +- !strcmp(_matrule->_rChild->_opType,"LoadBarrierWeakSlowReg") || + #endif + !strcmp(_matrule->_rChild->_opType,"CompareAndExchangeP") || + !strcmp(_matrule->_rChild->_opType,"CompareAndExchangeN"))) return true; +@@ -3503,10 +3504,12 @@ int MatchNode::needs_ideal_memory_edge(FormDict &globals) const { + "WeakCompareAndSwapB", "WeakCompareAndSwapS", "WeakCompareAndSwapI", "WeakCompareAndSwapL", "WeakCompareAndSwapP", "WeakCompareAndSwapN", + "CompareAndExchangeB", "CompareAndExchangeS", "CompareAndExchangeI", "CompareAndExchangeL", "CompareAndExchangeP", "CompareAndExchangeN", + "StoreCM", +- "ClearArray", + "GetAndSetB", "GetAndSetS", "GetAndAddI", "GetAndSetI", "GetAndSetP", + "GetAndAddB", "GetAndAddS", "GetAndAddL", "GetAndSetL", "GetAndSetN", +- "LoadBarrierSlowReg", "LoadBarrierWeakSlowReg" ++#if INCLUDE_ZGC ++ "LoadBarrierSlowReg", "ZGetAndSetP", "ZCompareAndSwapP", "ZCompareAndExchangeP", "ZWeakCompareAndSwapP", ++#endif ++ "ClearArray" + }; + int cnt = sizeof(needs_ideal_memory_list)/sizeof(char*); + if( strcmp(_opType,"PrefetchAllocation")==0 ) +diff --git a/src/hotspot/share/compiler/compilerDirectives.hpp b/src/hotspot/share/compiler/compilerDirectives.hpp +index c06d6b899..5c9fc98e1 100644 +--- a/src/hotspot/share/compiler/compilerDirectives.hpp ++++ b/src/hotspot/share/compiler/compilerDirectives.hpp +@@ -67,7 +67,7 @@ NOT_PRODUCT(cflags(TraceOptoOutput, bool, TraceOptoOutput, TraceOptoOutput)) + cflags(CloneMapDebug, bool, false, CloneMapDebug) \ + cflags(IGVPrintLevel, intx, PrintIdealGraphLevel, IGVPrintLevel) \ + cflags(MaxNodeLimit, intx, MaxNodeLimit, MaxNodeLimit) \ +-ZGC_ONLY(cflags(ZOptimizeLoadBarriers, bool, ZOptimizeLoadBarriers, ZOptimizeLoadBarriers)) ++ZGC_ONLY(cflags(ZTraceLoadBarriers, bool, false, ZTraceLoadBarriers)) + #else + #define compilerdirectives_c2_flags(cflags) + #endif +diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp +index 8baf4d9de..eea74674f 100644 +--- a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp ++++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp +@@ -208,15 +208,17 @@ public: + virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const { } + virtual void enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const {} + virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful) const {} +- virtual void add_users_to_worklist(Unique_Node_List* worklist) const {} + + // Allow barrier sets to have shared state that is preserved across a compilation unit. + // This could for example comprise macro nodes to be expanded during macro expansion. + virtual void* create_barrier_state(Arena* comp_arena) const { return NULL; } ++ virtual void optimize_loops(PhaseIdealLoop* phase, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const { } ++ virtual void barrier_insertion_phase(Compile* C, PhaseIterGVN &igvn) const { } + // If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be + // expanded later, then now is the time to do so. + virtual bool expand_macro_nodes(PhaseMacroExpand* macro) const { return false; } + virtual void verify_gc_barriers(bool post_parse) const {} ++ virtual bool needs_anti_dependence_check(const Node* node) const { return true; } + }; + + #endif // SHARE_GC_SHARED_C2_BARRIERSETC2_HPP +diff --git a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp +index f7458e6ce..1c77a4f63 100644 +--- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp ++++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp +@@ -22,14 +22,21 @@ + */ + + #include "precompiled.hpp" +-#include "opto/compile.hpp" + #include "opto/castnode.hpp" ++#include "opto/compile.hpp" + #include "opto/graphKit.hpp" +-#include "opto/idealKit.hpp" + #include "opto/loopnode.hpp" ++#include "opto/machnode.hpp" + #include "opto/macro.hpp" ++#include "opto/memnode.hpp" ++#include "opto/movenode.hpp" + #include "opto/node.hpp" ++#include "opto/phase.hpp" ++#include "opto/phaseX.hpp" ++#include "opto/rootnode.hpp" + #include "opto/type.hpp" ++#include "utilities/copy.hpp" ++#include "utilities/growableArray.hpp" + #include "utilities/macros.hpp" + #include "gc/z/zBarrierSet.hpp" + #include "gc/z/c2/zBarrierSetC2.hpp" +@@ -83,7 +90,7 @@ bool ZBarrierSetC2::is_gc_barrier_node(Node* node) const { + if (node->is_Phi()) { + PhiNode* phi = node->as_Phi(); + Node* n = phi->in(1); +- if (n != NULL && (n->is_LoadBarrierSlowReg() || n->is_LoadBarrierWeakSlowReg())) { ++ if (n != NULL && (n->is_LoadBarrierSlowReg())) { + return true; + } + } +@@ -120,50 +127,19 @@ void ZBarrierSetC2::enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* + } + } + +-void ZBarrierSetC2::find_dominating_barriers(PhaseIterGVN& igvn) { +- // Look for dominating barriers on the same address only once all +- // other loop opts are over: loop opts may cause a safepoint to be +- // inserted between a barrier and its dominating barrier. +- Compile* C = Compile::current(); +- ZBarrierSetC2* bs = (ZBarrierSetC2*)BarrierSet::barrier_set()->barrier_set_c2(); +- ZBarrierSetC2State* s = bs->state(); +- if (s->load_barrier_count() >= 2) { +- Compile::TracePhase tp("idealLoop", &C->timers[Phase::_t_idealLoop]); +- PhaseIdealLoop ideal_loop(igvn, true, false, true); +- if (C->major_progress()) C->print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2); +- } +-} +- +-void ZBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const { +- // Permanent temporary workaround +- // Loadbarriers may have non-obvious dead uses keeping them alive during parsing. The use is +- // removed by RemoveUseless (after parsing, before optimize) but the barriers won't be added to +- // the worklist. Unless we add them explicitly they are not guaranteed to end up there. +- ZBarrierSetC2State* s = state(); ++static bool load_require_barrier(LoadNode* load) { return ((load->barrier_data() & RequireBarrier) != 0); } ++static bool load_has_weak_barrier(LoadNode* load) { return ((load->barrier_data() & WeakBarrier) != 0); } ++static bool load_has_expanded_barrier(LoadNode* load) { return ((load->barrier_data() & ExpandedBarrier) != 0); } ++static void load_set_expanded_barrier(LoadNode* load) { return load->set_barrier_data(ExpandedBarrier); } + +- for (int i = 0; i < s->load_barrier_count(); i++) { +- LoadBarrierNode* n = s->load_barrier_node(i); +- worklist->push(n); ++static void load_set_barrier(LoadNode* load, bool weak) { ++ if (weak) { ++ load->set_barrier_data(WeakBarrier); ++ } else { ++ load->set_barrier_data(RequireBarrier); + } + } + +-const TypeFunc* ZBarrierSetC2::load_barrier_Type() const { +- const Type** fields; +- +- // Create input types (domain) +- fields = TypeTuple::fields(2); +- fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; +- fields[TypeFunc::Parms+1] = TypeOopPtr::BOTTOM; +- const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields); +- +- // Create result type (range) +- fields = TypeTuple::fields(1); +- fields[TypeFunc::Parms+0] = TypeInstPtr::BOTTOM; +- const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields); +- +- return TypeFunc::make(domain, range); +-} +- + // == LoadBarrierNode == + + LoadBarrierNode::LoadBarrierNode(Compile* C, +@@ -171,13 +147,9 @@ LoadBarrierNode::LoadBarrierNode(Compile* C, + Node* mem, + Node* val, + Node* adr, +- bool weak, +- bool writeback, +- bool oop_reload_allowed) : ++ bool weak) : + MultiNode(Number_of_Inputs), +- _weak(weak), +- _writeback(writeback), +- _oop_reload_allowed(oop_reload_allowed) { ++ _weak(weak) { + init_req(Control, c); + init_req(Memory, mem); + init_req(Oop, val); +@@ -216,8 +188,8 @@ const Type *LoadBarrierNode::Value(PhaseGVN *phase) const { + const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*))); + const Type* val_t = phase->type(in(Oop)); + floadbarrier[Control] = Type::CONTROL; +- floadbarrier[Memory] = Type::MEMORY; +- floadbarrier[Oop] = val_t; ++ floadbarrier[Memory] = Type::MEMORY; ++ floadbarrier[Oop] = val_t; + return TypeTuple::make(Number_of_Outputs, floadbarrier); + } + +@@ -237,6 +209,11 @@ bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node + } + + LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) { ++ if (is_weak()) { ++ // Weak barriers can't be eliminated ++ return NULL; ++ } ++ + Node* val = in(LoadBarrierNode::Oop); + if (in(Similar)->is_Proj() && in(Similar)->in(0)->is_LoadBarrier()) { + LoadBarrierNode* lb = in(Similar)->in(0)->as_LoadBarrier(); +@@ -265,7 +242,7 @@ LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, + } + } + +- if (ZVerifyLoadBarriers || can_be_eliminated()) { ++ if (can_be_eliminated()) { + return NULL; + } + +@@ -315,7 +292,7 @@ LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, + } + if (ok) { + assert(dom_found, ""); +- return u->as_LoadBarrier();; ++ return u->as_LoadBarrier(); + } + break; + } +@@ -327,6 +304,7 @@ LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, + + void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const { + // Change to that barrier may affect a dominated barrier so re-push those ++ assert(!is_weak(), "sanity"); + Node* val = in(LoadBarrierNode::Oop); + + for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { +@@ -354,13 +332,9 @@ void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const { + } + + Node *LoadBarrierNode::Identity(PhaseGVN *phase) { +- if (!phase->C->directive()->ZOptimizeLoadBarriersOption) { +- return this; +- } +- +- bool redundant_addr = false; + LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false); + if (dominating_barrier != NULL) { ++ assert(!is_weak(), "Weak barriers cant be eliminated"); + assert(dominating_barrier->in(Oop) == in(Oop), ""); + return dominating_barrier; + } +@@ -373,33 +347,31 @@ Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { + return this; + } + +- Node* val = in(Oop); +- Node* mem = in(Memory); +- Node* ctrl = in(Control); +- Node* adr = in(Address); +- assert(val->Opcode() != Op_LoadN, ""); ++ Node *val = in(Oop); ++ Node *mem = in(Memory); ++ Node *ctrl = in(Control); ++ assert(val->Opcode() != Op_DecodeN, ""); + + if (mem->is_MergeMem()) { +- Node* new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw); ++ Node *new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw); + set_req(Memory, new_mem); + if (mem->outcnt() == 0 && can_reshape) { + phase->is_IterGVN()->_worklist.push(mem); + } +- + return this; + } + +- bool optimizeLoadBarriers = phase->C->directive()->ZOptimizeLoadBarriersOption; +- LoadBarrierNode* dominating_barrier = optimizeLoadBarriers ? has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress()) : NULL; +- if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) { +- assert(in(Address) == dominating_barrier->in(Address), ""); +- set_req(Similar, dominating_barrier->proj_out(Oop)); +- return this; ++ LoadBarrierNode *dominating_barrier = NULL; ++ if (!is_weak()) { ++ dominating_barrier = has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress()); ++ if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) { ++ assert(in(Address) == dominating_barrier->in(Address), ""); ++ set_req(Similar, dominating_barrier->proj_out(Oop)); ++ return this; ++ } + } + +- bool eliminate = (optimizeLoadBarriers && !(val->is_Phi() || val->Opcode() == Op_LoadP || val->Opcode() == Op_GetAndSetP || val->is_DecodeN())) || +- (can_reshape && (dominating_barrier != NULL || !has_true_uses())); +- ++ bool eliminate = can_reshape && (dominating_barrier != NULL || !has_true_uses()); + if (eliminate) { + if (can_reshape) { + PhaseIterGVN* igvn = phase->is_IterGVN(); +@@ -414,13 +386,13 @@ Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { + fix_similar_in_uses(igvn); + if (out_res != NULL) { + if (dominating_barrier != NULL) { ++ assert(!is_weak(), "Sanity"); + igvn->replace_node(out_res, dominating_barrier->proj_out(Oop)); + } else { + igvn->replace_node(out_res, val); + } + } + } +- + return new ConINode(TypeInt::ZERO); + } + +@@ -431,7 +403,7 @@ Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { + return this; + } + +- if (can_reshape) { ++ if (can_reshape && !is_weak()) { + // If this barrier is linked through the Similar edge by a + // dominated barrier and both barriers have the same Oop field, + // the dominated barrier can go away, so push it for reprocessing. +@@ -445,6 +417,7 @@ Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { + Node* u = out_res->fast_out(i); + if (u->is_LoadBarrier() && u->in(Similar) == out_res && + (u->in(Oop) == val || !u->in(Similar)->is_top())) { ++ assert(!u->as_LoadBarrier()->is_weak(), "Sanity"); + igvn->_worklist.push(u); + } + } +@@ -478,213 +451,17 @@ void LoadBarrierNode::fix_similar_in_uses(PhaseIterGVN* igvn) { + + bool LoadBarrierNode::has_true_uses() const { + Node* out_res = proj_out_or_null(Oop); +- if (out_res == NULL) { +- return false; +- } +- +- for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { +- Node* u = out_res->fast_out(i); +- if (!u->is_LoadBarrier() || u->in(Similar) != out_res) { +- return true; ++ if (out_res != NULL) { ++ for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { ++ Node *u = out_res->fast_out(i); ++ if (!u->is_LoadBarrier() || u->in(Similar) != out_res) { ++ return true; ++ } + } + } +- + return false; + } + +-// == Accesses == +- +-Node* ZBarrierSetC2::make_cas_loadbarrier(C2AtomicAccess& access) const { +- assert(!UseCompressedOops, "Not allowed"); +- CompareAndSwapNode* cas = (CompareAndSwapNode*)access.raw_access(); +- PhaseGVN& gvn = access.kit()->gvn(); +- Compile* C = Compile::current(); +- GraphKit* kit = access.kit(); +- +- Node* in_ctrl = cas->in(MemNode::Control); +- Node* in_mem = cas->in(MemNode::Memory); +- Node* in_adr = cas->in(MemNode::Address); +- Node* in_val = cas->in(MemNode::ValueIn); +- Node* in_expected = cas->in(LoadStoreConditionalNode::ExpectedIn); +- +- float likely = PROB_LIKELY(0.999); +- +- const TypePtr *adr_type = gvn.type(in_adr)->isa_ptr(); +- Compile::AliasType* alias_type = C->alias_type(adr_type); +- int alias_idx = C->get_alias_index(adr_type); +- +- // Outer check - true: continue, false: load and check +- Node* region = new RegionNode(3); +- Node* phi = new PhiNode(region, TypeInt::BOOL); +- Node* phi_mem = new PhiNode(region, Type::MEMORY, adr_type); +- +- // Inner check - is the healed ref equal to the expected +- Node* region2 = new RegionNode(3); +- Node* phi2 = new PhiNode(region2, TypeInt::BOOL); +- Node* phi_mem2 = new PhiNode(region2, Type::MEMORY, adr_type); +- +- // CAS node returns 0 or 1 +- Node* cmp = gvn.transform(new CmpINode(cas, kit->intcon(0))); +- Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool(); +- IfNode* iff = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If(); +- Node* then = gvn.transform(new IfTrueNode(iff)); +- Node* elsen = gvn.transform(new IfFalseNode(iff)); +- +- Node* scmemproj1 = gvn.transform(new SCMemProjNode(cas)); +- +- kit->set_memory(scmemproj1, alias_idx); +- phi_mem->init_req(1, scmemproj1); +- phi_mem2->init_req(2, scmemproj1); +- +- // CAS fail - reload and heal oop +- Node* reload = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered); +- Node* barrier = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false)); +- Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control)); +- Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop)); +- +- // Check load +- Node* tmpX = gvn.transform(new CastP2XNode(NULL, barrierdata)); +- Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected)); +- Node* cmp2 = gvn.transform(new CmpXNode(tmpX, in_expX)); +- Node *bol2 = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool(); +- IfNode* iff2 = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If(); +- Node* then2 = gvn.transform(new IfTrueNode(iff2)); +- Node* elsen2 = gvn.transform(new IfFalseNode(iff2)); +- +- // redo CAS +- Node* cas2 = gvn.transform(new CompareAndSwapPNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, cas->order())); +- Node* scmemproj2 = gvn.transform(new SCMemProjNode(cas2)); +- kit->set_control(elsen2); +- kit->set_memory(scmemproj2, alias_idx); +- +- // Merge inner flow - check if healed oop was equal too expected. +- region2->set_req(1, kit->control()); +- region2->set_req(2, then2); +- phi2->set_req(1, cas2); +- phi2->set_req(2, kit->intcon(0)); +- phi_mem2->init_req(1, scmemproj2); +- kit->set_memory(phi_mem2, alias_idx); +- +- // Merge outer flow - then check if first CAS succeeded +- region->set_req(1, then); +- region->set_req(2, region2); +- phi->set_req(1, kit->intcon(1)); +- phi->set_req(2, phi2); +- phi_mem->init_req(2, phi_mem2); +- kit->set_memory(phi_mem, alias_idx); +- +- gvn.transform(region2); +- gvn.transform(phi2); +- gvn.transform(phi_mem2); +- gvn.transform(region); +- gvn.transform(phi); +- gvn.transform(phi_mem); +- +- kit->set_control(region); +- kit->insert_mem_bar(Op_MemBarCPUOrder); +- +- return phi; +-} +- +-Node* ZBarrierSetC2::make_cmpx_loadbarrier(C2AtomicAccess& access) const { +- CompareAndExchangePNode* cmpx = (CompareAndExchangePNode*)access.raw_access(); +- GraphKit* kit = access.kit(); +- PhaseGVN& gvn = kit->gvn(); +- Compile* C = Compile::current(); +- +- Node* in_ctrl = cmpx->in(MemNode::Control); +- Node* in_mem = cmpx->in(MemNode::Memory); +- Node* in_adr = cmpx->in(MemNode::Address); +- Node* in_val = cmpx->in(MemNode::ValueIn); +- Node* in_expected = cmpx->in(LoadStoreConditionalNode::ExpectedIn); +- +- float likely = PROB_LIKELY(0.999); +- +- const TypePtr *adr_type = cmpx->get_ptr_type(); +- Compile::AliasType* alias_type = C->alias_type(adr_type); +- int alias_idx = C->get_alias_index(adr_type); +- +- // Outer check - true: continue, false: load and check +- Node* region = new RegionNode(3); +- Node* phi = new PhiNode(region, adr_type); +- +- // Inner check - is the healed ref equal to the expected +- Node* region2 = new RegionNode(3); +- Node* phi2 = new PhiNode(region2, adr_type); +- +- // Check if cmpx succeeded +- Node* cmp = gvn.transform(new CmpPNode(cmpx, in_expected)); +- Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::eq))->as_Bool(); +- IfNode* iff = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If(); +- Node* then = gvn.transform(new IfTrueNode(iff)); +- Node* elsen = gvn.transform(new IfFalseNode(iff)); +- +- Node* scmemproj1 = gvn.transform(new SCMemProjNode(cmpx)); +- kit->set_memory(scmemproj1, alias_idx); +- +- // CAS fail - reload and heal oop +- Node* reload = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered); +- Node* barrier = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false)); +- Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control)); +- Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop)); +- +- // Check load +- Node* tmpX = gvn.transform(new CastP2XNode(NULL, barrierdata)); +- Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected)); +- Node* cmp2 = gvn.transform(new CmpXNode(tmpX, in_expX)); +- Node *bol2 = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool(); +- IfNode* iff2 = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If(); +- Node* then2 = gvn.transform(new IfTrueNode(iff2)); +- Node* elsen2 = gvn.transform(new IfFalseNode(iff2)); +- +- // Redo CAS +- Node* cmpx2 = gvn.transform(new CompareAndExchangePNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, adr_type, cmpx->get_ptr_type(), cmpx->order())); +- Node* scmemproj2 = gvn.transform(new SCMemProjNode(cmpx2)); +- kit->set_control(elsen2); +- kit->set_memory(scmemproj2, alias_idx); +- +- // Merge inner flow - check if healed oop was equal too expected. +- region2->set_req(1, kit->control()); +- region2->set_req(2, then2); +- phi2->set_req(1, cmpx2); +- phi2->set_req(2, barrierdata); +- +- // Merge outer flow - then check if first cas succeeded +- region->set_req(1, then); +- region->set_req(2, region2); +- phi->set_req(1, cmpx); +- phi->set_req(2, phi2); +- +- gvn.transform(region2); +- gvn.transform(phi2); +- gvn.transform(region); +- gvn.transform(phi); +- +- kit->set_control(region); +- kit->set_memory(in_mem, alias_idx); +- kit->insert_mem_bar(Op_MemBarCPUOrder); +- +- return phi; +-} +- +-Node* ZBarrierSetC2::load_barrier(GraphKit* kit, Node* val, Node* adr, bool weak, bool writeback, bool oop_reload_allowed) const { +- PhaseGVN& gvn = kit->gvn(); +- Node* barrier = new LoadBarrierNode(Compile::current(), kit->control(), kit->memory(TypeRawPtr::BOTTOM), val, adr, weak, writeback, oop_reload_allowed); +- Node* transformed_barrier = gvn.transform(barrier); +- +- if (transformed_barrier->is_LoadBarrier()) { +- if (barrier == transformed_barrier) { +- kit->set_control(gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control))); +- } +- Node* result = gvn.transform(new ProjNode(transformed_barrier, LoadBarrierNode::Oop)); +- assert(is_gc_barrier_node(result), "sanity"); +- assert(step_over_gc_barrier(result) == val, "sanity"); +- return result; +- } else { +- return val; +- } +-} +- + static bool barrier_needed(C2Access access) { + return ZBarrierSet::barrier_needed(access.decorators(), access.type()); + } +@@ -696,157 +473,48 @@ Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) co + } + + bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0; +- +- GraphKit* kit = access.kit(); +- PhaseGVN& gvn = kit->gvn(); +- Node* adr = access.addr().node(); +- Node* heap_base_oop = access.base(); +- bool unsafe = (access.decorators() & C2_UNSAFE_ACCESS) != 0; +- if (unsafe) { +- if (!ZVerifyLoadBarriers) { +- p = load_barrier(kit, p, adr); +- } else { +- if (!TypePtr::NULL_PTR->higher_equal(gvn.type(heap_base_oop))) { +- p = load_barrier(kit, p, adr); +- } else { +- IdealKit ideal(kit); +- IdealVariable res(ideal); +-#define __ ideal. +- __ declarations_done(); +- __ set(res, p); +- __ if_then(heap_base_oop, BoolTest::ne, kit->null(), PROB_UNLIKELY(0.999)); { +- kit->sync_kit(ideal); +- p = load_barrier(kit, p, adr); +- __ set(res, p); +- __ sync_kit(kit); +- } __ end_if(); +- kit->final_sync(ideal); +- p = __ value(res); +-#undef __ +- } +- } +- return p; +- } else { +- return load_barrier(access.kit(), p, access.addr().node(), weak, true, true); ++ if (p->isa_Load()) { ++ load_set_barrier(p->as_Load(), weak); + } ++ return p; + } + + Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val, + Node* new_val, const Type* val_type) const { + Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type); +- if (!barrier_needed(access)) { +- return result; ++ LoadStoreNode* lsn = result->as_LoadStore(); ++ if (barrier_needed(access)) { ++ lsn->set_has_barrier(); + } +- +- access.set_needs_pinning(false); +- return make_cmpx_loadbarrier(access); ++ return lsn; + } + + Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val, + Node* new_val, const Type* value_type) const { + Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); +- if (!barrier_needed(access)) { +- return result; ++ LoadStoreNode* lsn = result->as_LoadStore(); ++ if (barrier_needed(access)) { ++ lsn->set_has_barrier(); + } ++ return lsn; + +- Node* load_store = access.raw_access(); +- bool weak_cas = (access.decorators() & C2_WEAK_CMPXCHG) != 0; +- bool expected_is_null = (expected_val->get_ptr_type() == TypePtr::NULL_PTR); +- +- if (!expected_is_null) { +- if (weak_cas) { +- access.set_needs_pinning(false); +- load_store = make_cas_loadbarrier(access); +- } else { +- access.set_needs_pinning(false); +- load_store = make_cas_loadbarrier(access); +- } +- } +- +- return load_store; + } + + Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const { + Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type); +- if (!barrier_needed(access)) { +- return result; ++ LoadStoreNode* lsn = result->as_LoadStore(); ++ if (barrier_needed(access)) { ++ lsn->set_has_barrier(); + } +- +- Node* load_store = access.raw_access(); +- Node* adr = access.addr().node(); +- +- return load_barrier(access.kit(), load_store, adr, false, false, false); ++ return lsn; + } + + // == Macro Expansion == + ++// Optimized, low spill, loadbarrier variant using stub specialized on register used + void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const { +- Node* in_ctrl = barrier->in(LoadBarrierNode::Control); +- Node* in_mem = barrier->in(LoadBarrierNode::Memory); +- Node* in_val = barrier->in(LoadBarrierNode::Oop); +- Node* in_adr = barrier->in(LoadBarrierNode::Address); +- +- Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control); +- Node* out_res = barrier->proj_out(LoadBarrierNode::Oop); +- +- PhaseIterGVN &igvn = phase->igvn(); +- +- if (ZVerifyLoadBarriers) { +- igvn.replace_node(out_res, in_val); +- igvn.replace_node(out_ctrl, in_ctrl); +- return; +- } +- +- if (barrier->can_be_eliminated()) { +- // Clone and pin the load for this barrier below the dominating +- // barrier: the load cannot be allowed to float above the +- // dominating barrier +- Node* load = in_val; +- +- if (load->is_Load()) { +- Node* new_load = load->clone(); +- Node* addp = new_load->in(MemNode::Address); +- assert(addp->is_AddP() || addp->is_Phi() || addp->is_Load(), "bad address"); +- Node* cast = new CastPPNode(addp, igvn.type(addp), true); +- Node* ctrl = NULL; +- Node* similar = barrier->in(LoadBarrierNode::Similar); +- if (similar->is_Phi()) { +- // already expanded +- ctrl = similar->in(0); +- } else { +- assert(similar->is_Proj() && similar->in(0)->is_LoadBarrier(), "unexpected graph shape"); +- ctrl = similar->in(0)->as_LoadBarrier()->proj_out(LoadBarrierNode::Control); +- } +- assert(ctrl != NULL, "bad control"); +- cast->set_req(0, ctrl); +- igvn.transform(cast); +- new_load->set_req(MemNode::Address, cast); +- igvn.transform(new_load); +- +- igvn.replace_node(out_res, new_load); +- igvn.replace_node(out_ctrl, in_ctrl); +- return; +- } +- // cannot eliminate +- } +- +- // There are two cases that require the basic loadbarrier +- // 1) When the writeback of a healed oop must be avoided (swap) +- // 2) When we must guarantee that no reload of is done (swap, cas, cmpx) +- if (!barrier->is_writeback()) { +- assert(!barrier->oop_reload_allowed(), "writeback barriers should be marked as requires oop"); +- } +- +- if (!barrier->oop_reload_allowed()) { +- expand_loadbarrier_basic(phase, barrier); +- } else { +- expand_loadbarrier_optimized(phase, barrier); +- } +-} +- +-// Basic loadbarrier using conventional argument passing +-void ZBarrierSetC2::expand_loadbarrier_basic(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const { + PhaseIterGVN &igvn = phase->igvn(); ++ float unlikely = PROB_UNLIKELY(0.999); + + Node* in_ctrl = barrier->in(LoadBarrierNode::Control); + Node* in_mem = barrier->in(LoadBarrierNode::Memory); +@@ -856,102 +524,8 @@ void ZBarrierSetC2::expand_loadbarrier_basic(PhaseMacroExpand* phase, LoadBarrie + Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control); + Node* out_res = barrier->proj_out(LoadBarrierNode::Oop); + +- float unlikely = PROB_UNLIKELY(0.999); +- const Type* in_val_maybe_null_t = igvn.type(in_val); +- +- Node* jthread = igvn.transform(new ThreadLocalNode()); +- Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset())); +- Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered)); +- Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val)); +- Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask)); +- Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type()))); +- Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool(); +- IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If(); +- Node* then = igvn.transform(new IfTrueNode(iff)); +- Node* elsen = igvn.transform(new IfFalseNode(iff)); +- +- Node* result_region; +- Node* result_val; +- +- result_region = new RegionNode(3); +- result_val = new PhiNode(result_region, TypeInstPtr::BOTTOM); +- +- result_region->set_req(1, elsen); +- Node* res = igvn.transform(new CastPPNode(in_val, in_val_maybe_null_t)); +- res->init_req(0, elsen); +- result_val->set_req(1, res); +- +- const TypeFunc *tf = load_barrier_Type(); +- Node* call; +- if (barrier->is_weak()) { +- call = new CallLeafNode(tf, +- ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr(), +- "ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded", +- TypeRawPtr::BOTTOM); +- } else { +- call = new CallLeafNode(tf, +- ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(), +- "ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded", +- TypeRawPtr::BOTTOM); +- } +- +- call->init_req(TypeFunc::Control, then); +- call->init_req(TypeFunc::I_O , phase->top()); +- call->init_req(TypeFunc::Memory , in_mem); +- call->init_req(TypeFunc::FramePtr, phase->top()); +- call->init_req(TypeFunc::ReturnAdr, phase->top()); +- call->init_req(TypeFunc::Parms+0, in_val); +- if (barrier->is_writeback()) { +- call->init_req(TypeFunc::Parms+1, in_adr); +- } else { +- // When slow path is called with a null address, the healed oop will not be written back +- call->init_req(TypeFunc::Parms+1, igvn.zerocon(T_OBJECT)); +- } +- call = igvn.transform(call); +- +- Node* ctrl = igvn.transform(new ProjNode(call, TypeFunc::Control)); +- res = igvn.transform(new ProjNode(call, TypeFunc::Parms)); +- res = igvn.transform(new CheckCastPPNode(ctrl, res, in_val_maybe_null_t)); +- +- result_region->set_req(2, ctrl); +- result_val->set_req(2, res); +- +- result_region = igvn.transform(result_region); +- result_val = igvn.transform(result_val); +- +- if (out_ctrl != NULL) { // Added if cond +- igvn.replace_node(out_ctrl, result_region); +- } +- igvn.replace_node(out_res, result_val); +-} +- +-// Optimized, low spill, loadbarrier variant using stub specialized on register used +-void ZBarrierSetC2::expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const { +- PhaseIterGVN &igvn = phase->igvn(); +-#ifdef PRINT_NODE_TRAVERSALS +- Node* preceding_barrier_node = barrier->in(LoadBarrierNode::Oop); +-#endif +- +- Node* in_ctrl = barrier->in(LoadBarrierNode::Control); +- Node* in_mem = barrier->in(LoadBarrierNode::Memory); +- Node* in_val = barrier->in(LoadBarrierNode::Oop); +- Node* in_adr = barrier->in(LoadBarrierNode::Address); +- +- Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control); +- Node* out_res = barrier->proj_out(LoadBarrierNode::Oop); +- + assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null"); + +-#ifdef PRINT_NODE_TRAVERSALS +- tty->print("\n\n\nBefore barrier optimization:\n"); +- traverse(barrier, out_ctrl, out_res, -1); +- +- tty->print("\nBefore barrier optimization: preceding_barrier_node\n"); +- traverse(preceding_barrier_node, out_ctrl, out_res, -1); +-#endif +- +- float unlikely = PROB_UNLIKELY(0.999); +- + Node* jthread = igvn.transform(new ThreadLocalNode()); + Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset())); + Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, +@@ -965,17 +539,9 @@ void ZBarrierSetC2::expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBa + Node* then = igvn.transform(new IfTrueNode(iff)); + Node* elsen = igvn.transform(new IfFalseNode(iff)); + +- Node* slow_path_surrogate; +- if (!barrier->is_weak()) { +- slow_path_surrogate = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(), +- (const TypePtr*) in_val->bottom_type(), MemNode::unordered)); +- } else { +- slow_path_surrogate = igvn.transform(new LoadBarrierWeakSlowRegNode(then, in_mem, in_adr, in_val->adr_type(), +- (const TypePtr*) in_val->bottom_type(), MemNode::unordered)); +- } ++ Node* new_loadp = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(), ++ (const TypePtr*) in_val->bottom_type(), MemNode::unordered, barrier->is_weak())); + +- Node *new_loadp; +- new_loadp = slow_path_surrogate; + // Create the final region/phi pair to converge cntl/data paths to downstream code + Node* result_region = igvn.transform(new RegionNode(3)); + result_region->set_req(1, then); +@@ -985,29 +551,17 @@ void ZBarrierSetC2::expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBa + result_phi->set_req(1, new_loadp); + result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop)); + +- // Finally, connect the original outputs to the barrier region and phi to complete the expansion/substitution +- // igvn.replace_node(out_ctrl, result_region); +- if (out_ctrl != NULL) { // added if cond +- igvn.replace_node(out_ctrl, result_region); +- } ++ igvn.replace_node(out_ctrl, result_region); + igvn.replace_node(out_res, result_phi); + + assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!"); + +-#ifdef PRINT_NODE_TRAVERSALS +- tty->print("\nAfter barrier optimization: old out_ctrl\n"); +- traverse(out_ctrl, out_ctrl, out_res, -1); +- tty->print("\nAfter barrier optimization: old out_res\n"); +- traverse(out_res, out_ctrl, out_res, -1); +- tty->print("\nAfter barrier optimization: old barrier\n"); +- traverse(barrier, out_ctrl, out_res, -1); +- tty->print("\nAfter barrier optimization: preceding_barrier_node\n"); +- traverse(preceding_barrier_node, result_region, result_phi, -1); +-#endif +- ++ igvn.remove_dead_node(barrier); ++ igvn.remove_dead_node(out_ctrl); ++ igvn.remove_dead_node(out_res); + assert(is_gc_barrier_node(result_phi), "sanity"); + assert(step_over_gc_barrier(result_phi) == in_val, "sanity"); +- ++ phase->C->print_method(PHASE_BEFORE_MACRO_EXPANSION, 4, barrier->_idx); + return; + } + +@@ -1053,373 +607,6 @@ bool ZBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const { + return false; + } + +-// == Loop optimization == +- +-static bool replace_with_dominating_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) { +- PhaseIterGVN &igvn = phase->igvn(); +- Compile* C = Compile::current(); +- +- LoadBarrierNode* lb2 = lb->has_dominating_barrier(phase, false, last_round); +- if (lb2 != NULL) { +- if (lb->in(LoadBarrierNode::Oop) != lb2->in(LoadBarrierNode::Oop)) { +- assert(lb->in(LoadBarrierNode::Address) == lb2->in(LoadBarrierNode::Address), ""); +- igvn.replace_input_of(lb, LoadBarrierNode::Similar, lb2->proj_out(LoadBarrierNode::Oop)); +- C->set_major_progress(); +- } else { +- // That transformation may cause the Similar edge on dominated load barriers to be invalid +- lb->fix_similar_in_uses(&igvn); +- +- Node* val = lb->proj_out(LoadBarrierNode::Oop); +- assert(lb2->has_true_uses(), ""); +- assert(lb2->in(LoadBarrierNode::Oop) == lb->in(LoadBarrierNode::Oop), ""); +- +- phase->lazy_update(lb, lb->in(LoadBarrierNode::Control)); +- phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control)); +- igvn.replace_node(val, lb2->proj_out(LoadBarrierNode::Oop)); +- +- return true; +- } +- } +- return false; +-} +- +-static Node* find_dominating_memory(PhaseIdealLoop* phase, Node* mem, Node* dom, int i) { +- assert(dom->is_Region() || i == -1, ""); +- Node* m = mem; +- while(phase->is_dominator(dom, phase->has_ctrl(m) ? phase->get_ctrl(m) : m->in(0))) { +- if (m->is_Mem()) { +- assert(m->as_Mem()->adr_type() == TypeRawPtr::BOTTOM, ""); +- m = m->in(MemNode::Memory); +- } else if (m->is_MergeMem()) { +- m = m->as_MergeMem()->memory_at(Compile::AliasIdxRaw); +- } else if (m->is_Phi()) { +- if (m->in(0) == dom && i != -1) { +- m = m->in(i); +- break; +- } else { +- m = m->in(LoopNode::EntryControl); +- } +- } else if (m->is_Proj()) { +- m = m->in(0); +- } else if (m->is_SafePoint() || m->is_MemBar()) { +- m = m->in(TypeFunc::Memory); +- } else { +-#ifdef ASSERT +- m->dump(); +-#endif +- ShouldNotReachHere(); +- } +- } +- return m; +-} +- +-static LoadBarrierNode* clone_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* ctl, Node* mem, Node* oop_in) { +- PhaseIterGVN &igvn = phase->igvn(); +- Compile* C = Compile::current(); +- Node* the_clone = lb->clone(); +- the_clone->set_req(LoadBarrierNode::Control, ctl); +- the_clone->set_req(LoadBarrierNode::Memory, mem); +- if (oop_in != NULL) { +- the_clone->set_req(LoadBarrierNode::Oop, oop_in); +- } +- +- LoadBarrierNode* new_lb = the_clone->as_LoadBarrier(); +- igvn.register_new_node_with_optimizer(new_lb); +- IdealLoopTree *loop = phase->get_loop(new_lb->in(0)); +- phase->set_ctrl(new_lb, new_lb->in(0)); +- phase->set_loop(new_lb, loop); +- phase->set_idom(new_lb, new_lb->in(0), phase->dom_depth(new_lb->in(0))+1); +- if (!loop->_child) { +- loop->_body.push(new_lb); +- } +- +- Node* proj_ctl = new ProjNode(new_lb, LoadBarrierNode::Control); +- igvn.register_new_node_with_optimizer(proj_ctl); +- phase->set_ctrl(proj_ctl, proj_ctl->in(0)); +- phase->set_loop(proj_ctl, loop); +- phase->set_idom(proj_ctl, new_lb, phase->dom_depth(new_lb)+1); +- if (!loop->_child) { +- loop->_body.push(proj_ctl); +- } +- +- Node* proj_oop = new ProjNode(new_lb, LoadBarrierNode::Oop); +- phase->register_new_node(proj_oop, new_lb); +- +- if (!new_lb->in(LoadBarrierNode::Similar)->is_top()) { +- LoadBarrierNode* similar = new_lb->in(LoadBarrierNode::Similar)->in(0)->as_LoadBarrier(); +- if (!phase->is_dominator(similar, ctl)) { +- igvn.replace_input_of(new_lb, LoadBarrierNode::Similar, C->top()); +- } +- } +- +- return new_lb; +-} +- +-static void replace_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* new_val) { +- PhaseIterGVN &igvn = phase->igvn(); +- Node* val = lb->proj_out(LoadBarrierNode::Oop); +- igvn.replace_node(val, new_val); +- phase->lazy_update(lb, lb->in(LoadBarrierNode::Control)); +- phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control)); +-} +- +-static bool split_barrier_thru_phi(PhaseIdealLoop* phase, LoadBarrierNode* lb) { +- PhaseIterGVN &igvn = phase->igvn(); +- Compile* C = Compile::current(); +- +- if (lb->in(LoadBarrierNode::Oop)->is_Phi()) { +- Node* oop_phi = lb->in(LoadBarrierNode::Oop); +- +- if ((oop_phi->req() != 3) || (oop_phi->in(2) == oop_phi)) { +- // Ignore phis with only one input +- return false; +- } +- +- if (phase->is_dominator(phase->get_ctrl(lb->in(LoadBarrierNode::Address)), +- oop_phi->in(0)) && phase->get_ctrl(lb->in(LoadBarrierNode::Address)) != oop_phi->in(0)) { +- // That transformation may cause the Similar edge on dominated load barriers to be invalid +- lb->fix_similar_in_uses(&igvn); +- +- RegionNode* region = oop_phi->in(0)->as_Region(); +- +- int backedge = LoopNode::LoopBackControl; +- if (region->is_Loop() && region->in(backedge)->is_Proj() && region->in(backedge)->in(0)->is_If()) { +- Node* c = region->in(backedge)->in(0)->in(0); +- assert(c->unique_ctrl_out() == region->in(backedge)->in(0), ""); +- Node* oop = lb->in(LoadBarrierNode::Oop)->in(backedge); +- Node* oop_c = phase->has_ctrl(oop) ? phase->get_ctrl(oop) : oop; +- if (!phase->is_dominator(oop_c, c)) { +- return false; +- } +- } +- +- // If the node on the backedge above the phi is the node itself - we have a self loop. +- // Don't clone - this will be folded later. +- if (oop_phi->in(LoopNode::LoopBackControl) == lb->proj_out(LoadBarrierNode::Oop)) { +- return false; +- } +- +- bool is_strip_mined = region->is_CountedLoop() && region->as_CountedLoop()->is_strip_mined(); +- Node *phi = oop_phi->clone(); +- +- for (uint i = 1; i < region->req(); i++) { +- Node* ctrl = region->in(i); +- if (ctrl != C->top()) { +- assert(!phase->is_dominator(ctrl, region) || region->is_Loop(), ""); +- +- Node* mem = lb->in(LoadBarrierNode::Memory); +- Node* m = find_dominating_memory(phase, mem, region, i); +- +- if (region->is_Loop() && i == LoopNode::LoopBackControl && ctrl->is_Proj() && ctrl->in(0)->is_If()) { +- ctrl = ctrl->in(0)->in(0); +- } else if (region->is_Loop() && is_strip_mined) { +- // If this is a strip mined loop, control must move above OuterStripMinedLoop +- assert(i == LoopNode::EntryControl, "check"); +- assert(ctrl->is_OuterStripMinedLoop(), "sanity"); +- ctrl = ctrl->as_OuterStripMinedLoop()->in(LoopNode::EntryControl); +- } +- +- LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, ctrl, m, lb->in(LoadBarrierNode::Oop)->in(i)); +- Node* out_ctrl = new_lb->proj_out(LoadBarrierNode::Control); +- +- if (is_strip_mined && (i == LoopNode::EntryControl)) { +- assert(region->in(i)->is_OuterStripMinedLoop(), ""); +- igvn.replace_input_of(region->in(i), i, out_ctrl); +- phase->set_idom(region->in(i), out_ctrl, phase->dom_depth(out_ctrl)); +- } else if (ctrl == region->in(i)) { +- igvn.replace_input_of(region, i, out_ctrl); +- // Only update the idom if is the loop entry we are updating +- // - A loop backedge doesn't change the idom +- if (region->is_Loop() && i == LoopNode::EntryControl) { +- phase->set_idom(region, out_ctrl, phase->dom_depth(out_ctrl)); +- } +- } else { +- Node* iff = region->in(i)->in(0); +- igvn.replace_input_of(iff, 0, out_ctrl); +- phase->set_idom(iff, out_ctrl, phase->dom_depth(out_ctrl)+1); +- } +- phi->set_req(i, new_lb->proj_out(LoadBarrierNode::Oop)); +- } +- } +- phase->register_new_node(phi, region); +- replace_barrier(phase, lb, phi); +- +- if (region->is_Loop()) { +- // Load barrier moved to the back edge of the Loop may now +- // have a safepoint on the path to the barrier on the Similar +- // edge +- igvn.replace_input_of(phi->in(LoopNode::LoopBackControl)->in(0), LoadBarrierNode::Similar, C->top()); +- Node* head = region->in(LoopNode::EntryControl); +- phase->set_idom(region, head, phase->dom_depth(head)+1); +- phase->recompute_dom_depth(); +- if (head->is_CountedLoop() && head->as_CountedLoop()->is_main_loop()) { +- head->as_CountedLoop()->set_normal_loop(); +- } +- } +- +- return true; +- } +- } +- +- return false; +-} +- +-static bool move_out_of_loop(PhaseIdealLoop* phase, LoadBarrierNode* lb) { +- PhaseIterGVN &igvn = phase->igvn(); +- IdealLoopTree *lb_loop = phase->get_loop(lb->in(0)); +- if (lb_loop != phase->ltree_root() && !lb_loop->_irreducible) { +- Node* oop_ctrl = phase->get_ctrl(lb->in(LoadBarrierNode::Oop)); +- IdealLoopTree *oop_loop = phase->get_loop(oop_ctrl); +- IdealLoopTree* adr_loop = phase->get_loop(phase->get_ctrl(lb->in(LoadBarrierNode::Address))); +- if (!lb_loop->is_member(oop_loop) && !lb_loop->is_member(adr_loop)) { +- // That transformation may cause the Similar edge on dominated load barriers to be invalid +- lb->fix_similar_in_uses(&igvn); +- +- Node* head = lb_loop->_head; +- assert(head->is_Loop(), ""); +- +- if (phase->is_dominator(head, oop_ctrl)) { +- assert(oop_ctrl->Opcode() == Op_CProj && oop_ctrl->in(0)->Opcode() == Op_NeverBranch, ""); +- assert(lb_loop->is_member(phase->get_loop(oop_ctrl->in(0)->in(0))), ""); +- return false; +- } +- +- if (head->is_CountedLoop()) { +- CountedLoopNode* cloop = head->as_CountedLoop(); +- if (cloop->is_main_loop()) { +- cloop->set_normal_loop(); +- } +- // When we are moving barrier out of a counted loop, +- // make sure we move it all the way out of the strip mined outer loop. +- if (cloop->is_strip_mined()) { +- head = cloop->outer_loop(); +- } +- } +- +- Node* mem = lb->in(LoadBarrierNode::Memory); +- Node* m = find_dominating_memory(phase, mem, head, -1); +- +- LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, head->in(LoopNode::EntryControl), m, NULL); +- +- assert(phase->idom(head) == head->in(LoopNode::EntryControl), ""); +- Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control); +- igvn.replace_input_of(head, LoopNode::EntryControl, proj_ctl); +- phase->set_idom(head, proj_ctl, phase->dom_depth(proj_ctl) + 1); +- +- replace_barrier(phase, lb, new_lb->proj_out(LoadBarrierNode::Oop)); +- +- phase->recompute_dom_depth(); +- +- return true; +- } +- } +- +- return false; +-} +- +-static bool common_barriers(PhaseIdealLoop* phase, LoadBarrierNode* lb) { +- PhaseIterGVN &igvn = phase->igvn(); +- Node* in_val = lb->in(LoadBarrierNode::Oop); +- for (DUIterator_Fast imax, i = in_val->fast_outs(imax); i < imax; i++) { +- Node* u = in_val->fast_out(i); +- if (u != lb && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) { +- Node* this_ctrl = lb->in(LoadBarrierNode::Control); +- Node* other_ctrl = u->in(LoadBarrierNode::Control); +- +- Node* lca = phase->dom_lca(this_ctrl, other_ctrl); +- Node* proj1 = NULL; +- Node* proj2 = NULL; +- bool ok = (lb->in(LoadBarrierNode::Address) == u->in(LoadBarrierNode::Address)); +- +- while (this_ctrl != lca && ok) { +- if (this_ctrl->in(0) != NULL && +- this_ctrl->in(0)->is_MultiBranch()) { +- if (this_ctrl->in(0)->in(0) == lca) { +- assert(proj1 == NULL, ""); +- assert(this_ctrl->is_Proj(), ""); +- proj1 = this_ctrl; +- } else if (!(this_ctrl->in(0)->is_If() && this_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) { +- ok = false; +- } +- } +- this_ctrl = phase->idom(this_ctrl); +- } +- while (other_ctrl != lca && ok) { +- if (other_ctrl->in(0) != NULL && +- other_ctrl->in(0)->is_MultiBranch()) { +- if (other_ctrl->in(0)->in(0) == lca) { +- assert(other_ctrl->is_Proj(), ""); +- assert(proj2 == NULL, ""); +- proj2 = other_ctrl; +- } else if (!(other_ctrl->in(0)->is_If() && other_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) { +- ok = false; +- } +- } +- other_ctrl = phase->idom(other_ctrl); +- } +- assert(proj1 == NULL || proj2 == NULL || proj1->in(0) == proj2->in(0), ""); +- if (ok && proj1 && proj2 && proj1 != proj2 && proj1->in(0)->is_If()) { +- // That transformation may cause the Similar edge on dominated load barriers to be invalid +- lb->fix_similar_in_uses(&igvn); +- u->as_LoadBarrier()->fix_similar_in_uses(&igvn); +- +- Node* split = lca->unique_ctrl_out(); +- assert(split->in(0) == lca, ""); +- +- Node* mem = lb->in(LoadBarrierNode::Memory); +- Node* m = find_dominating_memory(phase, mem, split, -1); +- LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, lca, m, NULL); +- +- Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control); +- igvn.replace_input_of(split, 0, new_lb->proj_out(LoadBarrierNode::Control)); +- phase->set_idom(split, proj_ctl, phase->dom_depth(proj_ctl)+1); +- +- Node* proj_oop = new_lb->proj_out(LoadBarrierNode::Oop); +- replace_barrier(phase, lb, proj_oop); +- replace_barrier(phase, u->as_LoadBarrier(), proj_oop); +- +- phase->recompute_dom_depth(); +- +- return true; +- } +- } +- } +- +- return false; +-} +- +-static void optimize_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) { +- Compile* C = Compile::current(); +- +- if (!C->directive()->ZOptimizeLoadBarriersOption) { +- return; +- } +- +- if (lb->has_true_uses()) { +- if (replace_with_dominating_barrier(phase, lb, last_round)) { +- return; +- } +- +- if (split_barrier_thru_phi(phase, lb)) { +- return; +- } +- +- if (move_out_of_loop(phase, lb)) { +- return; +- } +- +- if (common_barriers(phase, lb)) { +- return; +- } +- } +-} +- +-void ZBarrierSetC2::loop_optimize_gc_barrier(PhaseIdealLoop* phase, Node* node, bool last_round) { +- if (node->is_LoadBarrier()) { +- optimize_load_barrier(phase, node->as_LoadBarrier(), last_round); +- } +-} +- + Node* ZBarrierSetC2::step_over_gc_barrier(Node* c) const { + Node* node = c; + +@@ -1437,7 +624,7 @@ Node* ZBarrierSetC2::step_over_gc_barrier(Node* c) const { + if (node->is_Phi()) { + PhiNode* phi = node->as_Phi(); + Node* n = phi->in(1); +- if (n != NULL && (n->is_LoadBarrierSlowReg() || n->is_LoadBarrierWeakSlowReg())) { ++ if (n != NULL && (n->is_LoadBarrierSlowReg())) { + assert(c == node, "projections from step 1 should only be seen before macro expansion"); + return phi->in(2); + } +@@ -1553,3 +740,798 @@ void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const { + } + + #endif ++ ++static void call_catch_cleanup_one(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl); ++ ++// This code is cloning all uses of a load that is between a call and the catch blocks, ++// to each use. ++ ++static bool fixup_uses_in_catch(PhaseIdealLoop *phase, Node *start_ctrl, Node *node) { ++ ++ if (!phase->has_ctrl(node)) { ++ // This node is floating - doesn't need to be cloned. ++ assert(node != start_ctrl, "check"); ++ return false; ++ } ++ ++ Node* ctrl = phase->get_ctrl(node); ++ if (ctrl != start_ctrl) { ++ // We are in a successor block - the node is ok. ++ return false; // Unwind ++ } ++ ++ // Process successor nodes ++ int outcnt = node->outcnt(); ++ for (int i = 0; i < outcnt; i++) { ++ Node* n = node->raw_out(0); ++ assert(!n->is_LoadBarrier(), "Sanity"); ++ // Calling recursively, visiting leafs first ++ fixup_uses_in_catch(phase, start_ctrl, n); ++ } ++ ++ // Now all successors are outside ++ // - Clone this node to both successors ++ int no_succs = node->outcnt(); ++ assert(!node->is_Store(), "Stores not expected here"); ++ ++ // In some very rare cases a load that doesn't need a barrier will end up here ++ // Treat it as a LoadP and the insertion of phis will be done correctly. ++ if (node->is_Load()) { ++ assert(node->as_Load()->barrier_data() == 0, "Sanity"); ++ call_catch_cleanup_one(phase, node->as_Load(), phase->get_ctrl(node)); ++ } else { ++ for (DUIterator_Fast jmax, i = node->fast_outs(jmax); i < jmax; i++) { ++ Node* use = node->fast_out(i); ++ Node* clone = node->clone(); ++ assert(clone->outcnt() == 0, ""); ++ ++ assert(use->find_edge(node) != -1, "check"); ++ phase->igvn().rehash_node_delayed(use); ++ use->replace_edge(node, clone); ++ ++ Node* new_ctrl; ++ if (use->is_block_start()) { ++ new_ctrl = use; ++ } else if (use->is_CFG()) { ++ new_ctrl = use->in(0); ++ assert (new_ctrl != NULL, ""); ++ } else { ++ new_ctrl = phase->get_ctrl(use); ++ } ++ ++ phase->set_ctrl(clone, new_ctrl); ++ ++ if (phase->C->directive()->ZTraceLoadBarriersOption) tty->print_cr(" Clone op %i as %i to control %i", node->_idx, clone->_idx, new_ctrl->_idx); ++ phase->igvn().register_new_node_with_optimizer(clone); ++ --i, --jmax; ++ } ++ assert(node->outcnt() == 0, "must be empty now"); ++ ++ // Node node is dead. ++ phase->igvn().remove_dead_node(node); ++ } ++ return true; // unwind - return if a use was processed ++} ++ ++// Clone a load to a specific catch_proj ++static Node* clone_load_to_catchproj(PhaseIdealLoop* phase, Node* load, Node* catch_proj) { ++ Node* cloned_load = load->clone(); ++ cloned_load->set_req(0, catch_proj); // set explicit control ++ phase->set_ctrl(cloned_load, catch_proj); // update ++ if (phase->C->directive()->ZTraceLoadBarriersOption) tty->print_cr(" Clone LOAD %i as %i to control %i", load->_idx, cloned_load->_idx, catch_proj->_idx); ++ phase->igvn().register_new_node_with_optimizer(cloned_load); ++ return cloned_load; ++} ++ ++static Node* get_dominating_region(PhaseIdealLoop* phase, Node* node, Node* stop) { ++ Node* region = node; ++ while (!region->isa_Region()) { ++ Node *up = phase->idom(region); ++ assert(up != region, "Must not loop"); ++ assert(up != stop, "Must not find original control"); ++ region = up; ++ } ++ return region; ++} ++ ++// Clone this load to each catch block ++static void call_catch_cleanup_one(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl) { ++ bool trace = phase->C->directive()->ZTraceLoadBarriersOption; ++ phase->igvn().set_delay_transform(true); ++ ++ // Verify pre conditions ++ assert(ctrl->isa_Proj() && ctrl->in(0)->isa_Call(), "Must be a call proj"); ++ assert(ctrl->raw_out(0)->isa_Catch(), "Must be a catch"); ++ ++ if (ctrl->raw_out(0)->isa_Catch()->outcnt() == 1) { ++ if (trace) tty->print_cr("Cleaning up catch: Skipping load %i, call with single catch", load->_idx); ++ return; ++ } ++ ++ // Process the loads successor nodes - if any is between ++ // the call and the catch blocks, they need to be cloned to. ++ // This is done recursively ++ int outcnt = load->outcnt(); ++ uint index = 0; ++ for (int i = 0; i < outcnt; i++) { ++ if (index < load->outcnt()) { ++ Node *n = load->raw_out(index); ++ assert(!n->is_LoadBarrier(), "Sanity"); ++ if (!fixup_uses_in_catch(phase, ctrl, n)) { ++ // if no successor was cloned, progress to next out. ++ index++; ++ } ++ } ++ } ++ ++ // Now all the loads uses has been cloned down ++ // Only thing left is to clone the loads, but they must end up ++ // first in the catch blocks. ++ ++ // We clone the loads oo the catch blocks only when needed. ++ // An array is used to map the catch blocks to each lazily cloned load. ++ // In that way no extra unnecessary loads are cloned. ++ ++ // Any use dominated by original block must have an phi and a region added ++ ++ Node* catch_node = ctrl->raw_out(0); ++ int number_of_catch_projs = catch_node->outcnt(); ++ Node** proj_to_load_mapping = NEW_RESOURCE_ARRAY(Node*, number_of_catch_projs); ++ Copy::zero_to_bytes(proj_to_load_mapping, sizeof(Node*) * number_of_catch_projs); ++ ++ // The phi_map is used to keep track of where phis have already been inserted ++ int phi_map_len = phase->C->unique(); ++ Node** phi_map = NEW_RESOURCE_ARRAY(Node*, phi_map_len); ++ Copy::zero_to_bytes(phi_map, sizeof(Node*) * phi_map_len); ++ ++ for (unsigned int i = 0; i < load->outcnt(); i++) { ++ Node* load_use_control = NULL; ++ Node* load_use = load->raw_out(i); ++ ++ if (phase->has_ctrl(load_use)) { ++ load_use_control = phase->get_ctrl(load_use); ++ } else { ++ load_use_control = load_use->in(0); ++ } ++ assert(load_use_control != NULL, "sanity"); ++ if (trace) tty->print_cr(" Handling use: %i, with control: %i", load_use->_idx, load_use_control->_idx); ++ ++ // Some times the loads use is a phi. For them we need to determine from which catch block ++ // the use is defined. ++ bool load_use_is_phi = false; ++ unsigned int load_use_phi_index = 0; ++ Node* phi_ctrl = NULL; ++ if (load_use->is_Phi()) { ++ // Find phi input that matches load ++ for (unsigned int u = 1; u < load_use->req(); u++) { ++ if (load_use->in(u) == load) { ++ load_use_is_phi = true; ++ load_use_phi_index = u; ++ assert(load_use->in(0)->is_Region(), "Region or broken"); ++ phi_ctrl = load_use->in(0)->in(u); ++ assert(phi_ctrl->is_CFG(), "check"); ++ assert(phi_ctrl != load, "check"); ++ break; ++ } ++ } ++ assert(load_use_is_phi, "must find"); ++ assert(load_use_phi_index > 0, "sanity"); ++ } ++ ++ // For each load use, see which catch projs dominates, create load clone lazily and reconnect ++ bool found_dominating_catchproj = false; ++ for (int c = 0; c < number_of_catch_projs; c++) { ++ Node* catchproj = catch_node->raw_out(c); ++ assert(catchproj != NULL && catchproj->isa_CatchProj(), "Sanity"); ++ ++ if (!phase->is_dominator(catchproj, load_use_control)) { ++ if (load_use_is_phi && phase->is_dominator(catchproj, phi_ctrl)) { ++ // The loads use is local to the catchproj. ++ // fall out and replace load with catch-local load clone. ++ } else { ++ continue; ++ } ++ } ++ assert(!found_dominating_catchproj, "Max one should match"); ++ ++ // Clone loads to catch projs ++ Node* load_clone = proj_to_load_mapping[c]; ++ if (load_clone == NULL) { ++ load_clone = clone_load_to_catchproj(phase, load, catchproj); ++ proj_to_load_mapping[c] = load_clone; ++ } ++ phase->igvn().rehash_node_delayed(load_use); ++ ++ if (load_use_is_phi) { ++ // phis are special - the load is defined from a specific control flow ++ load_use->set_req(load_use_phi_index, load_clone); ++ } else { ++ // Multipe edges can be replaced at once - on calls for example ++ load_use->replace_edge(load, load_clone); ++ } ++ --i; // more than one edge can have been removed, but the next is in later iterations ++ ++ // We could break the for-loop after finding a dominating match. ++ // But keep iterating to catch any bad idom early. ++ found_dominating_catchproj = true; ++ } ++ ++ // We found no single catchproj that dominated the use - The use is at a point after ++ // where control flow from multiple catch projs have merged. We will have to create ++ // phi nodes before the use and tie the output from the cloned loads together. It ++ // can be a single phi or a number of chained phis, depending on control flow ++ if (!found_dominating_catchproj) { ++ ++ // Use phi-control if use is a phi ++ if (load_use_is_phi) { ++ load_use_control = phi_ctrl; ++ } ++ assert(phase->is_dominator(ctrl, load_use_control), "Common use but no dominator"); ++ ++ // Clone a load on all paths ++ for (int c = 0; c < number_of_catch_projs; c++) { ++ Node* catchproj = catch_node->raw_out(c); ++ Node* load_clone = proj_to_load_mapping[c]; ++ if (load_clone == NULL) { ++ load_clone = clone_load_to_catchproj(phase, load, catchproj); ++ proj_to_load_mapping[c] = load_clone; ++ } ++ } ++ ++ // Move up dominator tree from use until dom front is reached ++ Node* next_region = get_dominating_region(phase, load_use_control, ctrl); ++ while (phase->idom(next_region) != catch_node) { ++ next_region = phase->idom(next_region); ++ if (trace) tty->print_cr("Moving up idom to region ctrl %i", next_region->_idx); ++ } ++ assert(phase->is_dominator(catch_node, next_region), "Sanity"); ++ ++ // Create or reuse phi node that collect all cloned loads and feed it to the use. ++ Node* test_phi = phi_map[next_region->_idx]; ++ if ((test_phi != NULL) && test_phi->is_Phi()) { ++ // Reuse an already created phi ++ if (trace) tty->print_cr(" Using cached Phi %i on load_use %i", test_phi->_idx, load_use->_idx); ++ phase->igvn().rehash_node_delayed(load_use); ++ load_use->replace_edge(load, test_phi); ++ // Now this use is done ++ } else { ++ // Otherwise we need to create one or more phis ++ PhiNode* next_phi = new PhiNode(next_region, load->type()); ++ phi_map[next_region->_idx] = next_phi; // cache new phi ++ phase->igvn().rehash_node_delayed(load_use); ++ load_use->replace_edge(load, next_phi); ++ ++ int dominators_of_region = 0; ++ do { ++ // New phi, connect to region and add all loads as in. ++ Node* region = next_region; ++ assert(region->isa_Region() && region->req() > 2, "Catch dead region nodes"); ++ PhiNode* new_phi = next_phi; ++ ++ if (trace) tty->print_cr("Created Phi %i on load %i with control %i", new_phi->_idx, load->_idx, region->_idx); ++ ++ // Need to add all cloned loads to the phi, taking care that the right path is matched ++ dominators_of_region = 0; // reset for new region ++ for (unsigned int reg_i = 1; reg_i < region->req(); reg_i++) { ++ Node* region_pred = region->in(reg_i); ++ assert(region_pred->is_CFG(), "check"); ++ bool pred_has_dominator = false; ++ for (int c = 0; c < number_of_catch_projs; c++) { ++ Node* catchproj = catch_node->raw_out(c); ++ if (phase->is_dominator(catchproj, region_pred)) { ++ new_phi->set_req(reg_i, proj_to_load_mapping[c]); ++ if (trace) tty->print_cr(" - Phi in(%i) set to load %i", reg_i, proj_to_load_mapping[c]->_idx); ++ pred_has_dominator = true; ++ dominators_of_region++; ++ break; ++ } ++ } ++ ++ // Sometimes we need to chain several phis. ++ if (!pred_has_dominator) { ++ assert(dominators_of_region <= 1, "More than one region can't require extra phi"); ++ if (trace) tty->print_cr(" - Region %i pred %i not dominated by catch proj", region->_idx, region_pred->_idx); ++ // Continue search on on this region_pred ++ // - walk up to next region ++ // - create a new phi and connect to first new_phi ++ next_region = get_dominating_region(phase, region_pred, ctrl); ++ ++ // Lookup if there already is a phi, create a new otherwise ++ Node* test_phi = phi_map[next_region->_idx]; ++ if ((test_phi != NULL) && test_phi->is_Phi()) { ++ next_phi = test_phi->isa_Phi(); ++ dominators_of_region++; // record that a match was found and that we are done ++ if (trace) tty->print_cr(" Using cached phi Phi %i on control %i", next_phi->_idx, next_region->_idx); ++ } else { ++ next_phi = new PhiNode(next_region, load->type()); ++ phi_map[next_region->_idx] = next_phi; ++ } ++ new_phi->set_req(reg_i, next_phi); ++ } ++ } ++ ++ new_phi->set_req(0, region); ++ phase->igvn().register_new_node_with_optimizer(new_phi); ++ phase->set_ctrl(new_phi, region); ++ ++ assert(dominators_of_region != 0, "Must have found one this iteration"); ++ } while (dominators_of_region == 1); ++ } ++ --i; ++ } ++ } // end of loop over uses ++ ++ assert(load->outcnt() == 0, "All uses should be handled"); ++ phase->igvn().remove_dead_node(load); ++ phase->C->print_method(PHASE_CALL_CATCH_CLEANUP, 4, load->_idx); ++ ++ // Now we should be home ++ phase->igvn().set_delay_transform(false); ++} ++ ++// Sort out the loads that are between a call ant its catch blocks ++static void process_catch_cleanup_candidate(PhaseIdealLoop* phase, LoadNode* load) { ++ bool trace = phase->C->directive()->ZTraceLoadBarriersOption; ++ ++ Node* ctrl = phase->get_ctrl(load); ++ if (!ctrl->is_Proj() || (ctrl->in(0) == NULL) || !ctrl->in(0)->isa_Call()) { ++ return; ++ } ++ ++ Node* catch_node = ctrl->isa_Proj()->raw_out(0); ++ if (catch_node->is_Catch()) { ++ if (catch_node->outcnt() > 1) { ++ call_catch_cleanup_one(phase, load, ctrl); ++ } else { ++ if (trace) tty->print_cr("Call catch cleanup with only one catch: load %i ", load->_idx); ++ } ++ } ++} ++ ++void ZBarrierSetC2::barrier_insertion_phase(Compile* C, PhaseIterGVN& igvn) const { ++ ResourceMark rm; ++ PhaseIdealLoop v(igvn, false, false, false, true); ++ if (C->failing()) return; ++} ++ ++void ZBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const { ++ // First make sure all loads between call and catch are moved to the catch block ++ clean_catch_blocks(phase); ++ ++ // Then expand barriers on all loads ++ insert_load_barriers(phase); ++ ++ // Handle all Unsafe that need barriers. ++ insert_barriers_on_unsafe(phase); ++ ++ phase->C->clear_major_progress(); ++} ++ ++static bool can_simplify_cas(LoadStoreNode* node) { ++ if (node->isa_LoadStoreConditional()) { ++ Node *expected_in = node->as_LoadStoreConditional()->in(LoadStoreConditionalNode::ExpectedIn); ++ return (expected_in->get_ptr_type() == TypePtr::NULL_PTR); ++ } else { ++ return false; ++ } ++} ++ ++static void insert_barrier_before_unsafe(PhaseIdealLoop* phase, LoadStoreNode* old_node) { ++ ++ Compile *C = phase->C; ++ PhaseIterGVN &igvn = phase->igvn(); ++ LoadStoreNode* zclone = NULL; ++ bool is_weak = false; ++ ++ Node *in_ctrl = old_node->in(MemNode::Control); ++ Node *in_mem = old_node->in(MemNode::Memory); ++ Node *in_adr = old_node->in(MemNode::Address); ++ Node *in_val = old_node->in(MemNode::ValueIn); ++ const TypePtr *adr_type = old_node->adr_type(); ++ const TypePtr* load_type = TypeOopPtr::BOTTOM; // The type for the load we are adding ++ ++ switch (old_node->Opcode()) { ++ case Op_CompareAndExchangeP: { ++ zclone = new ZCompareAndExchangePNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn), ++ adr_type, old_node->get_ptr_type(), ((CompareAndExchangeNode*)old_node)->order()); ++ load_type = old_node->bottom_type()->is_ptr(); ++ break; ++ } ++ case Op_WeakCompareAndSwapP: { ++ if (can_simplify_cas(old_node)) { ++ break; ++ } ++ is_weak = true; ++ zclone = new ZWeakCompareAndSwapPNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn), ++ ((CompareAndSwapNode*)old_node)->order()); ++ adr_type = TypePtr::BOTTOM; ++ break; ++ } ++ case Op_CompareAndSwapP: { ++ if (can_simplify_cas(old_node)) { ++ break; ++ } ++ zclone = new ZCompareAndSwapPNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn), ++ ((CompareAndSwapNode*)old_node)->order()); ++ adr_type = TypePtr::BOTTOM; ++ break; ++ } ++ case Op_GetAndSetP: { ++ zclone = new ZGetAndSetPNode(in_ctrl, in_mem, in_adr, in_val, old_node->adr_type(), old_node->get_ptr_type()); ++ load_type = old_node->bottom_type()->is_ptr(); ++ break; ++ } ++ } ++ if (zclone != NULL) { ++ igvn.register_new_node_with_optimizer(zclone, old_node); ++ ++ // Make load ++ LoadPNode *load = new LoadPNode(NULL, in_mem, in_adr, adr_type, load_type, MemNode::unordered, ++ LoadNode::DependsOnlyOnTest); ++ load_set_expanded_barrier(load); ++ igvn.register_new_node_with_optimizer(load); ++ igvn.replace_node(old_node, zclone); ++ ++ Node *barrier = new LoadBarrierNode(C, NULL, in_mem, load, in_adr, is_weak); ++ Node *barrier_val = new ProjNode(barrier, LoadBarrierNode::Oop); ++ Node *barrier_ctrl = new ProjNode(barrier, LoadBarrierNode::Control); ++ ++ igvn.register_new_node_with_optimizer(barrier); ++ igvn.register_new_node_with_optimizer(barrier_val); ++ igvn.register_new_node_with_optimizer(barrier_ctrl); ++ ++ // loop over all of in_ctrl usages and move to barrier_ctrl ++ for (DUIterator_Last imin, i = in_ctrl->last_outs(imin); i >= imin; --i) { ++ Node *use = in_ctrl->last_out(i); ++ uint l; ++ for (l = 0; use->in(l) != in_ctrl; l++) {} ++ igvn.replace_input_of(use, l, barrier_ctrl); ++ } ++ ++ load->set_req(MemNode::Control, in_ctrl); ++ barrier->set_req(LoadBarrierNode::Control, in_ctrl); ++ zclone->add_req(barrier_val); // add req as keep alive. ++ ++ C->print_method(PHASE_ADD_UNSAFE_BARRIER, 4, zclone->_idx); ++ } ++} ++ ++void ZBarrierSetC2::insert_barriers_on_unsafe(PhaseIdealLoop* phase) const { ++ Compile *C = phase->C; ++ PhaseIterGVN &igvn = phase->igvn(); ++ uint new_ids = C->unique(); ++ VectorSet visited(Thread::current()->resource_area()); ++ GrowableArray nodeStack(Thread::current()->resource_area(), 0, 0, NULL); ++ nodeStack.push(C->root()); ++ visited.test_set(C->root()->_idx); ++ ++ // Traverse all nodes, visit all unsafe ops that require a barrier ++ while (nodeStack.length() > 0) { ++ Node *n = nodeStack.pop(); ++ ++ bool is_old_node = (n->_idx < new_ids); // don't process nodes that were created during cleanup ++ if (is_old_node) { ++ if (n->is_LoadStore()) { ++ LoadStoreNode* lsn = n->as_LoadStore(); ++ if (lsn->has_barrier()) { ++ BasicType bt = lsn->in(MemNode::Address)->bottom_type()->basic_type(); ++ assert ((bt == T_OBJECT || bt == T_ARRAY), "Sanity test"); ++ insert_barrier_before_unsafe(phase, lsn); ++ } ++ } ++ } ++ for (uint i = 0; i < n->len(); i++) { ++ if (n->in(i)) { ++ if (!visited.test_set(n->in(i)->_idx)) { ++ nodeStack.push(n->in(i)); ++ } ++ } ++ } ++ } ++ ++ igvn.optimize(); ++ C->print_method(PHASE_ADD_UNSAFE_BARRIER, 2); ++} ++ ++ ++// The purpose of ZBarrierSetC2::clean_catch_blocks is to prepare the IR for ++// splicing in load barrier nodes. ++// ++// The problem is that we might have instructions between a call and its catch nodes. ++// (This is usually handled in PhaseCFG:call_catch_cleanup, which clones mach nodes in ++// already scheduled blocks.) We can't have loads that require barriers there, ++// because we need to splice in new control flow, and that would violate the IR. ++// ++// clean_catch_blocks find all Loads that require a barrier and clone them and any ++// dependent instructions to each use. The loads must be in the beginning of the catch block ++// before any store. ++// ++// Sometimes the loads use will be at a place dominated by all catch blocks, then we need ++// a load in each catch block, and a Phi at the dominated use. ++ ++void ZBarrierSetC2::clean_catch_blocks(PhaseIdealLoop* phase) const { ++ ++ Compile *C = phase->C; ++ uint new_ids = C->unique(); ++ PhaseIterGVN &igvn = phase->igvn(); ++ VectorSet visited(Thread::current()->resource_area()); ++ GrowableArray nodeStack(Thread::current()->resource_area(), 0, 0, NULL); ++ nodeStack.push(C->root()); ++ visited.test_set(C->root()->_idx); ++ ++ // Traverse all nodes, visit all loads that require a barrier ++ while(nodeStack.length() > 0) { ++ Node *n = nodeStack.pop(); ++ ++ for (uint i = 0; i < n->len(); i++) { ++ if (n->in(i)) { ++ if (!visited.test_set(n->in(i)->_idx)) { ++ nodeStack.push(n->in(i)); ++ } ++ } ++ } ++ ++ bool is_old_node = (n->_idx < new_ids); // don't process nodes that were created during cleanup ++ if (n->is_Load() && is_old_node) { ++ LoadNode* load = n->isa_Load(); ++ // only care about loads that will have a barrier ++ if (load_require_barrier(load)) { ++ process_catch_cleanup_candidate(phase, load); ++ } ++ } ++ } ++ ++ C->print_method(PHASE_CALL_CATCH_CLEANUP, 2); ++} ++ ++class DomDepthCompareClosure : public CompareClosure { ++ PhaseIdealLoop* _phase; ++ ++public: ++ DomDepthCompareClosure(PhaseIdealLoop* phase) : _phase(phase) { } ++ ++ int do_compare(LoadNode* const &n1, LoadNode* const &n2) { ++ int d1 = _phase->dom_depth(_phase->get_ctrl(n1)); ++ int d2 = _phase->dom_depth(_phase->get_ctrl(n2)); ++ if (d1 == d2) { ++ // Compare index if the depth is the same, ensures all entries are unique. ++ return n1->_idx - n2->_idx; ++ } else { ++ return d2 - d1; ++ } ++ } ++}; ++ ++// Traverse graph and add all loadPs to list, sorted by dom depth ++void gather_loadnodes_sorted(PhaseIdealLoop* phase, GrowableArray* loadList) { ++ ++ VectorSet visited(Thread::current()->resource_area()); ++ GrowableArray nodeStack(Thread::current()->resource_area(), 0, 0, NULL); ++ DomDepthCompareClosure ddcc(phase); ++ ++ nodeStack.push(phase->C->root()); ++ while(nodeStack.length() > 0) { ++ Node *n = nodeStack.pop(); ++ if (visited.test(n->_idx)) { ++ continue; ++ } ++ ++ if (n->isa_Load()) { ++ LoadNode *load = n->as_Load(); ++ if (load_require_barrier(load)) { ++ assert(phase->get_ctrl(load) != NULL, "sanity"); ++ assert(phase->dom_depth(phase->get_ctrl(load)) != 0, "sanity"); ++ loadList->insert_sorted(&ddcc, load); ++ } ++ } ++ ++ visited.set(n->_idx); ++ for (uint i = 0; i < n->req(); i++) { ++ if (n->in(i)) { ++ if (!visited.test(n->in(i)->_idx)) { ++ nodeStack.push(n->in(i)); ++ } ++ } ++ } ++ } ++} ++ ++// Add LoadBarriers to all LoadPs ++void ZBarrierSetC2::insert_load_barriers(PhaseIdealLoop* phase) const { ++ bool trace = phase->C->directive()->ZTraceLoadBarriersOption; ++ GrowableArray loadList(Thread::current()->resource_area(), 0, 0, NULL); ++ gather_loadnodes_sorted(phase, &loadList); ++ PhaseIterGVN &igvn = phase->igvn(); ++ int count = 0; ++ ++ for (GrowableArrayIterator loadIter = loadList.begin(); loadIter != loadList.end(); ++loadIter) { ++ LoadNode *load = *loadIter; ++ ++ if (load_has_expanded_barrier(load)) { ++ continue; ++ } ++ ++ do { ++ // Insert a barrier on a loadP ++ // if another load is found that needs to be expanded first, retry on that one ++ LoadNode* result = insert_one_loadbarrier(phase, load, phase->get_ctrl(load)); ++ while (result != NULL) { ++ result = insert_one_loadbarrier(phase, result, phase->get_ctrl(result)); ++ } ++ } while (!load_has_expanded_barrier(load)); ++ } ++ phase->C->print_method(PHASE_INSERT_BARRIER, 2); ++} ++ ++void push_antidependent_stores(PhaseIdealLoop* phase, Node_Stack& nodestack, LoadNode* start_load) { ++ // push all stores on the same mem, that can_alias ++ // Any load found must be handled first ++ PhaseIterGVN &igvn = phase->igvn(); ++ int load_alias_idx = igvn.C->get_alias_index(start_load->adr_type()); ++ ++ Node *mem = start_load->in(1); ++ for (DUIterator_Fast imax, u = mem->fast_outs(imax); u < imax; u++) { ++ Node *mem_use = mem->fast_out(u); ++ ++ if (mem_use == start_load) continue; ++ if (!mem_use->is_Store()) continue; ++ if (!phase->has_ctrl(mem_use)) continue; ++ if (phase->get_ctrl(mem_use) != phase->get_ctrl(start_load)) continue; ++ ++ // add any aliasing store in this block ++ StoreNode *store = mem_use->isa_Store(); ++ const TypePtr *adr_type = store->adr_type(); ++ if (igvn.C->can_alias(adr_type, load_alias_idx)) { ++ nodestack.push(store, 0); ++ } ++ } ++} ++ ++LoadNode* ZBarrierSetC2::insert_one_loadbarrier(PhaseIdealLoop* phase, LoadNode* start_load, Node* ctrl) const { ++ bool trace = phase->C->directive()->ZTraceLoadBarriersOption; ++ PhaseIterGVN &igvn = phase->igvn(); ++ ++ // Check for other loadPs at the same loop depth that is reachable by a DFS ++ // - if found - return it. It needs to be inserted first ++ // - otherwise proceed and insert barrier ++ ++ VectorSet visited(Thread::current()->resource_area()); ++ Node_Stack nodestack(100); ++ ++ nodestack.push(start_load, 0); ++ push_antidependent_stores(phase, nodestack, start_load); ++ ++ while(!nodestack.is_empty()) { ++ Node* n = nodestack.node(); // peek ++ nodestack.pop(); ++ if (visited.test(n->_idx)) { ++ continue; ++ } ++ ++ if (n->is_Load() && n != start_load && load_require_barrier(n->as_Load()) && !load_has_expanded_barrier(n->as_Load())) { ++ // Found another load that needs a barrier in the same block. Must expand later loads first. ++ if (trace) tty->print_cr(" * Found LoadP %i on DFS", n->_idx); ++ return n->as_Load(); // return node that should be expanded first ++ } ++ ++ if (!phase->has_ctrl(n)) continue; ++ if (phase->get_ctrl(n) != phase->get_ctrl(start_load)) continue; ++ if (n->is_Phi()) continue; ++ ++ visited.set(n->_idx); ++ // push all children ++ for (DUIterator_Fast imax, ii = n->fast_outs(imax); ii < imax; ii++) { ++ Node* c = n->fast_out(ii); ++ if (c != NULL) { ++ nodestack.push(c, 0); ++ } ++ } ++ } ++ insert_one_loadbarrier_inner(phase, start_load, ctrl, visited); ++ return NULL; ++} ++ ++void ZBarrierSetC2::insert_one_loadbarrier_inner(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl, VectorSet visited2) const { ++ PhaseIterGVN &igvn = phase->igvn(); ++ Compile* C = igvn.C; ++ bool trace = C->directive()->ZTraceLoadBarriersOption; ++ ++ // create barrier ++ Node* barrier = new LoadBarrierNode(C, NULL, load->in(LoadNode::Memory), NULL, load->in(LoadNode::Address), load_has_weak_barrier(load)); ++ Node* barrier_val = new ProjNode(barrier, LoadBarrierNode::Oop); ++ Node* barrier_ctrl = new ProjNode(barrier, LoadBarrierNode::Control); ++ ++ if (trace) tty->print_cr("Insert load %i with barrier: %i and ctrl : %i", load->_idx, barrier->_idx, ctrl->_idx); ++ ++ // Splice control ++ // - insert barrier control diamond between loads ctrl and ctrl successor on path to block end. ++ // - If control successor is a catch, step over to next. ++ Node* ctrl_succ = NULL; ++ for (DUIterator_Fast imax, j = ctrl->fast_outs(imax); j < imax; j++) { ++ Node* tmp = ctrl->fast_out(j); ++ ++ // - CFG nodes is the ones we are going to splice (1 only!) ++ // - Phi nodes will continue to hang from the region node! ++ // - self loops should be skipped ++ if (tmp->is_Phi() || tmp == ctrl) { ++ continue; ++ } ++ ++ if (tmp->is_CFG()) { ++ assert(ctrl_succ == NULL, "There can be only one"); ++ ctrl_succ = tmp; ++ continue; ++ } ++ } ++ ++ // Now splice control ++ assert(ctrl_succ != load, "sanity"); ++ assert(ctrl_succ != NULL, "Broken IR"); ++ bool found = false; ++ for(uint k = 0; k < ctrl_succ->req(); k++) { ++ if (ctrl_succ->in(k) == ctrl) { ++ assert(!found, "sanity"); ++ if (trace) tty->print_cr(" Move CFG ctrl_succ %i to barrier_ctrl", ctrl_succ->_idx); ++ igvn.replace_input_of(ctrl_succ, k, barrier_ctrl); ++ found = true; ++ k--; ++ } ++ } ++ ++ // For all successors of ctrl - move all visited to become successors of barrier_ctrl instead ++ for (DUIterator_Fast imax, r = ctrl->fast_outs(imax); r < imax; r++) { ++ Node* tmp = ctrl->fast_out(r); ++ if (visited2.test(tmp->_idx) && (tmp != load)) { ++ if (trace) tty->print_cr(" Move ctrl_succ %i to barrier_ctrl", tmp->_idx); ++ igvn.replace_input_of(tmp, 0, barrier_ctrl); ++ --r; --imax; ++ } ++ } ++ ++ // Move the loads user to the barrier ++ for (DUIterator_Fast imax, i = load->fast_outs(imax); i < imax; i++) { ++ Node* u = load->fast_out(i); ++ if (u->isa_LoadBarrier()) { ++ continue; ++ } ++ ++ // find correct input - replace with iterator? ++ for(uint j = 0; j < u->req(); j++) { ++ if (u->in(j) == load) { ++ igvn.replace_input_of(u, j, barrier_val); ++ --i; --imax; // Adjust the iterator of the *outer* loop ++ break; // some nodes (calls) might have several uses from the same node ++ } ++ } ++ } ++ ++ // Connect barrier to load and control ++ barrier->set_req(LoadBarrierNode::Oop, load); ++ barrier->set_req(LoadBarrierNode::Control, ctrl); ++ ++ igvn.rehash_node_delayed(load); ++ igvn.register_new_node_with_optimizer(barrier); ++ igvn.register_new_node_with_optimizer(barrier_val); ++ igvn.register_new_node_with_optimizer(barrier_ctrl); ++ load_set_expanded_barrier(load); ++ ++ C->print_method(PHASE_INSERT_BARRIER, 3, load->_idx); ++} ++ ++// The bad_mask in the ThreadLocalData shouldn't have an anti-dep-check. ++// The bad_mask address if of type TypeRawPtr, but that will alias ++// InitializeNodes until the type system is expanded. ++bool ZBarrierSetC2::needs_anti_dependence_check(const Node* node) const { ++ MachNode* mnode = node->as_Mach(); ++ if (mnode != NULL) { ++ intptr_t offset = 0; ++ const TypePtr *adr_type2 = NULL; ++ const Node* base = mnode->get_base_and_disp(offset, adr_type2); ++ if ((base != NULL) && ++ (base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_ThreadLocal) && ++ (offset == in_bytes(ZThreadLocalData::address_bad_mask_offset()))) { ++ return false; ++ } ++ } ++ return true; ++} +diff --git a/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp b/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp +index fc9dadd67..bef6a7048 100644 +--- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp ++++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp +@@ -29,14 +29,33 @@ + #include "opto/node.hpp" + #include "utilities/growableArray.hpp" + ++class ZCompareAndSwapPNode : public CompareAndSwapPNode { ++public: ++ ZCompareAndSwapPNode(Node* c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapPNode(c, mem, adr, val, ex, mem_ord) { } ++ virtual int Opcode() const; ++}; ++ ++class ZWeakCompareAndSwapPNode : public WeakCompareAndSwapPNode { ++public: ++ ZWeakCompareAndSwapPNode(Node* c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : WeakCompareAndSwapPNode(c, mem, adr, val, ex, mem_ord) { } ++ virtual int Opcode() const; ++}; ++ ++class ZCompareAndExchangePNode : public CompareAndExchangePNode { ++public: ++ ZCompareAndExchangePNode(Node* c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, const Type* t, MemNode::MemOrd mem_ord) : CompareAndExchangePNode(c, mem, adr, val, ex, at, t, mem_ord) { } ++ virtual int Opcode() const; ++}; ++ ++class ZGetAndSetPNode : public GetAndSetPNode { ++public: ++ ZGetAndSetPNode(Node* c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* t) : GetAndSetPNode(c, mem, adr, val, at, t) { } ++ virtual int Opcode() const; ++}; ++ + class LoadBarrierNode : public MultiNode { + private: + bool _weak; +- bool _writeback; // Controls if the barrier writes the healed oop back to memory +- // A swap on a memory location must never write back the healed oop +- bool _oop_reload_allowed; // Controls if the barrier are allowed to reload the oop from memory +- // before healing, otherwise both the oop and the address must be +- // passed to the barrier from the oop + + static bool is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n); + void push_dominated_barriers(PhaseIterGVN* igvn) const; +@@ -57,9 +76,7 @@ public: + Node* mem, + Node* val, + Node* adr, +- bool weak, +- bool writeback, +- bool oop_reload_allowed); ++ bool weak); + + virtual int Opcode() const; + virtual uint size_of() const; +@@ -86,17 +103,11 @@ public: + bool is_weak() const { + return _weak; + } +- +- bool is_writeback() const { +- return _writeback; +- } +- +- bool oop_reload_allowed() const { +- return _oop_reload_allowed; +- } + }; + + class LoadBarrierSlowRegNode : public LoadPNode { ++private: ++ bool _is_weak; + public: + LoadBarrierSlowRegNode(Node *c, + Node *mem, +@@ -104,8 +115,9 @@ public: + const TypePtr *at, + const TypePtr* t, + MemOrd mo, ++ bool weak = false, + ControlDependency control_dependency = DependsOnlyOnTest) +- : LoadPNode(c, mem, adr, at, t, mo, control_dependency) { ++ : LoadPNode(c, mem, adr, at, t, mo, control_dependency), _is_weak(weak) { + init_class_id(Class_LoadBarrierSlowReg); + } + +@@ -118,30 +130,7 @@ public: + } + + virtual int Opcode() const; +-}; +- +-class LoadBarrierWeakSlowRegNode : public LoadPNode { +-public: +- LoadBarrierWeakSlowRegNode(Node *c, +- Node *mem, +- Node *adr, +- const TypePtr *at, +- const TypePtr* t, +- MemOrd mo, +- ControlDependency control_dependency = DependsOnlyOnTest) +- : LoadPNode(c, mem, adr, at, t, mo, control_dependency) { +- init_class_id(Class_LoadBarrierWeakSlowReg); +- } +- +- virtual const char * name() { +- return "LoadBarrierWeakSlowRegNode"; +- } +- +- virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) { +- return NULL; +- } +- +- virtual int Opcode() const; ++ bool is_weak() { return _is_weak; } + }; + + class ZBarrierSetC2State : public ResourceObj { +@@ -157,15 +146,17 @@ public: + LoadBarrierNode* load_barrier_node(int idx) const; + }; + ++enum BarrierInfo { ++ NoBarrier = 0, ++ RequireBarrier = 1, ++ WeakBarrier = 3, // Inclusive with RequireBarrier ++ ExpandedBarrier = 4 ++}; ++ + class ZBarrierSetC2 : public BarrierSetC2 { + private: + ZBarrierSetC2State* state() const; +- Node* make_cas_loadbarrier(C2AtomicAccess& access) const; +- Node* make_cmpx_loadbarrier(C2AtomicAccess& access) const; +- void expand_loadbarrier_basic(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const; + void expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const; +- void expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const; +- const TypeFunc* load_barrier_Type() const; + + protected: + virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const; +@@ -182,34 +173,39 @@ protected: + const Type* val_type) const; + + public: +- Node* load_barrier(GraphKit* kit, +- Node* val, +- Node* adr, +- bool weak = false, +- bool writeback = true, +- bool oop_reload_allowed = true) const; +- + virtual void* create_barrier_state(Arena* comp_arena) const; + virtual bool has_load_barriers() const { return true; } + virtual bool is_gc_barrier_node(Node* node) const; +- virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const { } +- virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful) const; +- virtual void add_users_to_worklist(Unique_Node_List* worklist) const; +- virtual void enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const; ++ + virtual void register_potential_barrier_node(Node* node) const; + virtual void unregister_potential_barrier_node(Node* node) const; ++ virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const { } ++ virtual void enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const; ++ virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful) const; ++ + virtual bool array_copy_requires_gc_barriers(BasicType type) const { return true; } + virtual Node* step_over_gc_barrier(Node* c) const; + // If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be + // expanded later, then now is the time to do so. + virtual bool expand_macro_nodes(PhaseMacroExpand* macro) const; +- +- static void find_dominating_barriers(PhaseIterGVN& igvn); +- static void loop_optimize_gc_barrier(PhaseIdealLoop* phase, Node* node, bool last_round); ++ virtual bool needs_anti_dependence_check(const Node* node) const; + + #ifdef ASSERT + virtual void verify_gc_barriers(bool post_parse) const; + #endif ++ ++ // Load barrier insertion and expansion external ++ virtual void barrier_insertion_phase(Compile* C, PhaseIterGVN &igvn) const; ++ virtual void optimize_loops(PhaseIdealLoop* phase, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const; ++ ++private: ++ // Load barrier insertion and expansion internal ++ void insert_barriers_on_unsafe(PhaseIdealLoop* phase) const; ++ void clean_catch_blocks(PhaseIdealLoop* phase) const; ++ void insert_load_barriers(PhaseIdealLoop* phase) const; ++ LoadNode* insert_one_loadbarrier(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl) const; ++ void insert_one_loadbarrier_inner(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl, VectorSet visited) const; ++ + }; + + #endif // SHARE_GC_Z_C2_ZBARRIERSETC2_HPP +diff --git a/src/hotspot/share/gc/z/zHeap.cpp b/src/hotspot/share/gc/z/zHeap.cpp +index eb72707e0..62f97d2a2 100644 +--- a/src/hotspot/share/gc/z/zHeap.cpp ++++ b/src/hotspot/share/gc/z/zHeap.cpp +@@ -304,35 +304,9 @@ void ZHeap::mark_flush_and_free(Thread* thread) { + _mark.flush_and_free(thread); + } + +-class ZFixupPartialLoadsTask : public ZTask { +-private: +- ZThreadRootsIterator _thread_roots; +- +-public: +- ZFixupPartialLoadsTask() : +- ZTask("ZFixupPartialLoadsTask"), +- _thread_roots() {} +- +- virtual void work() { +- ZMarkRootOopClosure cl; +- _thread_roots.oops_do(&cl); +- } +-}; +- +-void ZHeap::fixup_partial_loads() { +- ZFixupPartialLoadsTask task; +- _workers.run_parallel(&task); +-} +- + bool ZHeap::mark_end() { + assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); + +- // C2 can generate code where a safepoint poll is inserted +- // between a load and the associated load barrier. To handle +- // this case we need to rescan the thread stack here to make +- // sure such oops are marked. +- fixup_partial_loads(); +- + // Try end marking + if (!_mark.end()) { + // Marking not completed, continue concurrent mark +diff --git a/src/hotspot/share/gc/z/z_globals.hpp b/src/hotspot/share/gc/z/z_globals.hpp +index a75329372..9e0f8985b 100644 +--- a/src/hotspot/share/gc/z/z_globals.hpp ++++ b/src/hotspot/share/gc/z/z_globals.hpp +@@ -94,9 +94,6 @@ + diagnostic(bool, ZConcurrentJNIWeakGlobalHandles, true, \ + "Clean JNI WeakGlobalRefs concurrently") \ + \ +- diagnostic(bool, ZOptimizeLoadBarriers, true, \ +- "Apply load barrier optimizations") \ +- \ + develop(bool, ZVerifyLoadBarriers, false, \ + "Verify that reference loads are followed by barriers") + +diff --git a/src/hotspot/share/opto/classes.hpp b/src/hotspot/share/opto/classes.hpp +index 4314edd96..f6b588685 100644 +--- a/src/hotspot/share/opto/classes.hpp ++++ b/src/hotspot/share/opto/classes.hpp +@@ -196,7 +196,10 @@ macro(LoadS) + #endif + zgcmacro(LoadBarrier) + zgcmacro(LoadBarrierSlowReg) +-zgcmacro(LoadBarrierWeakSlowReg) ++zgcmacro(ZCompareAndSwapP) ++zgcmacro(ZWeakCompareAndSwapP) ++zgcmacro(ZCompareAndExchangeP) ++zgcmacro(ZGetAndSetP) + macro(Lock) + macro(Loop) + macro(LoopLimit) +diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp +index 40ced1bb7..ba61967bc 100644 +--- a/src/hotspot/share/opto/compile.cpp ++++ b/src/hotspot/share/opto/compile.cpp +@@ -2167,8 +2167,8 @@ void Compile::Optimize() { + + #endif + +-#ifdef ASSERT + BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); ++#ifdef ASSERT + bs->verify_gc_barriers(true); + #endif + +@@ -2344,12 +2344,6 @@ void Compile::Optimize() { + } + } + +-#if INCLUDE_ZGC +- if (UseZGC) { +- ZBarrierSetC2::find_dominating_barriers(igvn); +- } +-#endif +- + if (failing()) return; + + // Ensure that major progress is now clear +@@ -2369,7 +2363,13 @@ void Compile::Optimize() { + } + + #ifdef ASSERT +- BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); ++ bs->verify_gc_barriers(false); ++#endif ++ ++ bs->barrier_insertion_phase(C, igvn); ++ if (failing()) return; ++ ++#ifdef ASSERT + bs->verify_gc_barriers(false); + #endif + +@@ -2917,10 +2917,6 @@ void Compile::final_graph_reshaping_impl( Node *n, Final_Reshape_Counts &frc) { + case Op_LoadL_unaligned: + case Op_LoadPLocked: + case Op_LoadP: +-#if INCLUDE_ZGC +- case Op_LoadBarrierSlowReg: +- case Op_LoadBarrierWeakSlowReg: +-#endif + case Op_LoadN: + case Op_LoadRange: + case Op_LoadS: { +@@ -2937,6 +2933,29 @@ void Compile::final_graph_reshaping_impl( Node *n, Final_Reshape_Counts &frc) { + break; + } + ++#if INCLUDE_ZGC ++ case Op_LoadBarrier: { ++ assert(0, "There should be no load barriers left"); ++ } ++ case Op_ZGetAndSetP: ++ case Op_ZCompareAndExchangeP: ++ case Op_ZCompareAndSwapP: ++ case Op_ZWeakCompareAndSwapP: ++ case Op_LoadBarrierSlowReg: { ++#ifdef ASSERT ++ if (VerifyOptoOopOffsets) { ++ MemNode *mem = n->as_Mem(); ++ // Check to see if address types have grounded out somehow. ++ const TypeInstPtr *tp = mem->in(MemNode::Address)->bottom_type()->isa_instptr(); ++ ciInstanceKlass *k = tp->klass()->as_instance_klass(); ++ bool oop_offset_is_sane = k->contains_field_offset(tp->offset()); ++ assert(!tp || oop_offset_is_sane, ""); ++ } ++#endif ++ break; ++ } ++#endif ++ + case Op_AddP: { // Assert sane base pointers + Node *addp = n->in(AddPNode::Address); + assert( !addp->is_AddP() || +diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp +index 446324ec4..d58758db7 100644 +--- a/src/hotspot/share/opto/compile.hpp ++++ b/src/hotspot/share/opto/compile.hpp +@@ -52,6 +52,7 @@ class C2Compiler; + class CallGenerator; + class CloneMap; + class ConnectionGraph; ++class IdealGraphPrinter; + class InlineTree; + class Int_Array; + class LoadBarrierNode; +@@ -644,6 +645,7 @@ class Compile : public Phase { + void set_inlining_incrementally(bool z) { _inlining_incrementally = z; } + int inlining_incrementally() const { return _inlining_incrementally; } + void set_major_progress() { _major_progress++; } ++ void restore_major_progress(int progress) { _major_progress += progress; } + void clear_major_progress() { _major_progress = 0; } + int num_loop_opts() const { return _num_loop_opts; } + void set_num_loop_opts(int n) { _num_loop_opts = n; } +@@ -733,7 +735,15 @@ class Compile : public Phase { + C->_latest_stage_start_counter.stamp(); + } + +- void print_method(CompilerPhaseType cpt, int level = 1) { ++ bool should_print(int level = 1) { ++#ifndef PRODUCT ++ return (_printer && _printer->should_print(level)); ++#else ++ return false; ++#endif ++ } ++ ++ void print_method(CompilerPhaseType cpt, int level = 1, int idx = 0) { + EventCompilerPhase event; + if (event.should_commit()) { + event.set_starttime(C->_latest_stage_start_counter); +@@ -743,10 +753,15 @@ class Compile : public Phase { + event.commit(); + } + +- + #ifndef PRODUCT +- if (_printer && _printer->should_print(level)) { +- _printer->print_method(CompilerPhaseTypeHelper::to_string(cpt), level); ++ if (should_print(level)) { ++ char output[1024]; ++ if (idx != 0) { ++ sprintf(output, "%s:%d", CompilerPhaseTypeHelper::to_string(cpt), idx); ++ } else { ++ sprintf(output, "%s", CompilerPhaseTypeHelper::to_string(cpt)); ++ } ++ _printer->print_method(output, level); + } + #endif + C->_latest_stage_start_counter.stamp(); +diff --git a/src/hotspot/share/opto/escape.cpp b/src/hotspot/share/opto/escape.cpp +index 602f1261d..47c83766e 100644 +--- a/src/hotspot/share/opto/escape.cpp ++++ b/src/hotspot/share/opto/escape.cpp +@@ -453,10 +453,6 @@ void ConnectionGraph::add_node_to_connection_graph(Node *n, Unique_Node_List *de + break; + } + case Op_LoadP: +-#if INCLUDE_ZGC +- case Op_LoadBarrierSlowReg: +- case Op_LoadBarrierWeakSlowReg: +-#endif + case Op_LoadN: + case Op_LoadPLocked: { + add_objload_to_connection_graph(n, delayed_worklist); +@@ -666,10 +662,6 @@ void ConnectionGraph::add_final_edges(Node *n) { + break; + } + case Op_LoadP: +-#if INCLUDE_ZGC +- case Op_LoadBarrierSlowReg: +- case Op_LoadBarrierWeakSlowReg: +-#endif + case Op_LoadN: + case Op_LoadPLocked: { + // Using isa_ptr() instead of isa_oopptr() for LoadP and Phi because +diff --git a/src/hotspot/share/opto/idealGraphPrinter.cpp b/src/hotspot/share/opto/idealGraphPrinter.cpp +index d90f1d691..a07a51f3e 100644 +--- a/src/hotspot/share/opto/idealGraphPrinter.cpp ++++ b/src/hotspot/share/opto/idealGraphPrinter.cpp +@@ -350,14 +350,6 @@ void IdealGraphPrinter::end_method() { + _xml->flush(); + } + +-// Print indent +-void IdealGraphPrinter::print_indent() { +- tty->print_cr("printing indent %d", _depth); +- for (int i = 0; i < _depth; i++) { +- _xml->print("%s", INDENT); +- } +-} +- + bool IdealGraphPrinter::traverse_outs() { + return _traverse_outs; + } +@@ -663,14 +655,16 @@ void IdealGraphPrinter::walk_nodes(Node *start, bool edges, VectorSet* temp_set) + } + } + +-void IdealGraphPrinter::print_method(const char *name, int level, bool clear_nodes) { +- print(name, (Node *)C->root(), level, clear_nodes); ++void IdealGraphPrinter::print_method(const char *name, int level) { ++ if (should_print(level)) { ++ print(name, (Node *) C->root()); ++ } + } + + // Print current ideal graph +-void IdealGraphPrinter::print(const char *name, Node *node, int level, bool clear_nodes) { ++void IdealGraphPrinter::print(const char *name, Node *node) { + +- if (!_current_method || !_should_send_method || !should_print(level)) return; ++ if (!_current_method || !_should_send_method) return; + + // Warning, unsafe cast? + _chaitin = (PhaseChaitin *)C->regalloc(); +diff --git a/src/hotspot/share/opto/idealGraphPrinter.hpp b/src/hotspot/share/opto/idealGraphPrinter.hpp +index 2f4b9faf0..10dd02f9f 100644 +--- a/src/hotspot/share/opto/idealGraphPrinter.hpp ++++ b/src/hotspot/share/opto/idealGraphPrinter.hpp +@@ -81,10 +81,6 @@ class IdealGraphPrinter : public CHeapObj { + static const char *METHOD_SHORT_NAME_PROPERTY; + static const char *ASSEMBLY_ELEMENT; + +- elapsedTimer _walk_time; +- elapsedTimer _output_time; +- elapsedTimer _build_blocks_time; +- + static int _file_count; + networkStream *_stream; + xmlStream *_xml; +@@ -97,10 +93,6 @@ class IdealGraphPrinter : public CHeapObj { + bool _traverse_outs; + Compile *C; + +- static void pre_node(Node* node, void *env); +- static void post_node(Node* node, void *env); +- +- void print_indent(); + void print_method(ciMethod *method, int bci, InlineTree *tree); + void print_inline_tree(InlineTree *tree); + void visit_node(Node *n, bool edges, VectorSet* temp_set); +@@ -116,7 +108,6 @@ class IdealGraphPrinter : public CHeapObj { + void tail(const char *name); + void head(const char *name); + void text(const char *s); +- intptr_t get_node_id(Node *n); + IdealGraphPrinter(); + ~IdealGraphPrinter(); + +@@ -130,9 +121,8 @@ class IdealGraphPrinter : public CHeapObj { + void print_inlining(); + void begin_method(); + void end_method(); +- void print_method(const char *name, int level=1, bool clear_nodes = false); +- void print(const char *name, Node *root, int level=1, bool clear_nodes = false); +- void print_xml(const char *name); ++ void print_method(const char *name, int level = 0); ++ void print(const char *name, Node *root); + bool should_print(int level); + void set_compile(Compile* compile) {C = compile; } + }; +diff --git a/src/hotspot/share/opto/lcm.cpp b/src/hotspot/share/opto/lcm.cpp +index 50c0c2dd3..05ec9fa9f 100644 +--- a/src/hotspot/share/opto/lcm.cpp ++++ b/src/hotspot/share/opto/lcm.cpp +@@ -170,7 +170,6 @@ void PhaseCFG::implicit_null_check(Block* block, Node *proj, Node *val, int allo + case Op_LoadL: + case Op_LoadP: + case Op_LoadBarrierSlowReg: +- case Op_LoadBarrierWeakSlowReg: + case Op_LoadN: + case Op_LoadS: + case Op_LoadKlass: +diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp +index dd8ae8557..d99e2f81b 100644 +--- a/src/hotspot/share/opto/loopnode.cpp ++++ b/src/hotspot/share/opto/loopnode.cpp +@@ -982,7 +982,7 @@ void LoopNode::verify_strip_mined(int expect_skeleton) const { + wq.push(u); + bool found_sfpt = false; + for (uint next = 0; next < wq.size() && !found_sfpt; next++) { +- Node *n = wq.at(next); ++ Node* n = wq.at(next); + for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax && !found_sfpt; i++) { + Node* u = n->fast_out(i); + if (u == sfpt) { +@@ -996,6 +996,19 @@ void LoopNode::verify_strip_mined(int expect_skeleton) const { + assert(found_sfpt, "no node in loop that's not input to safepoint"); + } + } ++ ++ if (UseZGC && !inner_out->in(0)->is_CountedLoopEnd()) { ++ // In some very special cases there can be a load that has no other uses than the ++ // counted loop safepoint. Then its loadbarrier will be placed between the inner ++ // loop exit and the safepoint. This is very rare ++ ++ Node* ifnode = inner_out->in(1)->in(0); ++ // Region->IfTrue->If == Region->Iffalse->If ++ if (ifnode == inner_out->in(2)->in(0)) { ++ inner_out = ifnode->in(0); ++ } ++ } ++ + CountedLoopEndNode* cle = inner_out->in(0)->as_CountedLoopEnd(); + assert(cle == inner->loopexit_or_null(), "mismatch"); + bool has_skeleton = outer_le->in(1)->bottom_type()->singleton() && outer_le->in(1)->bottom_type()->is_int()->get_con() == 0; +@@ -2713,7 +2726,7 @@ bool PhaseIdealLoop::process_expensive_nodes() { + //----------------------------build_and_optimize------------------------------- + // Create a PhaseLoop. Build the ideal Loop tree. Map each Ideal Node to + // its corresponding LoopNode. If 'optimize' is true, do some loop cleanups. +-void PhaseIdealLoop::build_and_optimize(bool do_split_ifs, bool skip_loop_opts, bool last_round) { ++void PhaseIdealLoop::build_and_optimize(bool do_split_ifs, bool skip_loop_opts, bool last_round, bool z_barrier_insertion) { + ResourceMark rm; + + int old_progress = C->major_progress(); +@@ -2777,7 +2790,8 @@ void PhaseIdealLoop::build_and_optimize(bool do_split_ifs, bool skip_loop_opts, + } + + // Nothing to do, so get out +- bool stop_early = !C->has_loops() && !skip_loop_opts && !do_split_ifs && !_verify_me && !_verify_only; ++ bool stop_early = !C->has_loops() && !skip_loop_opts && !do_split_ifs && !_verify_me && !_verify_only && ++ !z_barrier_insertion; + bool do_expensive_nodes = C->should_optimize_expensive_nodes(_igvn); + if (stop_early && !do_expensive_nodes) { + _igvn.optimize(); // Cleanup NeverBranches +@@ -2869,9 +2883,7 @@ void PhaseIdealLoop::build_and_optimize(bool do_split_ifs, bool skip_loop_opts, + build_loop_late( visited, worklist, nstack ); + + if (_verify_only) { +- // restore major progress flag +- for (int i = 0; i < old_progress; i++) +- C->set_major_progress(); ++ C->restore_major_progress(old_progress); + assert(C->unique() == unique, "verification mode made Nodes? ? ?"); + assert(_igvn._worklist.size() == orig_worklist_size, "shouldn't push anything"); + return; +@@ -2915,10 +2927,7 @@ void PhaseIdealLoop::build_and_optimize(bool do_split_ifs, bool skip_loop_opts, + + if (skip_loop_opts) { + // restore major progress flag +- for (int i = 0; i < old_progress; i++) { +- C->set_major_progress(); +- } +- ++ C->restore_major_progress(old_progress); + // Cleanup any modified bits + _igvn.optimize(); + +@@ -2928,6 +2937,16 @@ void PhaseIdealLoop::build_and_optimize(bool do_split_ifs, bool skip_loop_opts, + return; + } + ++ if(z_barrier_insertion) { ++ BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); ++ bs->optimize_loops(this, visited, nstack, worklist); ++ _igvn.optimize(); ++ if (C->log() != NULL) { ++ log_loop_tree(_ltree_root, _ltree_root, C->log()); ++ } ++ return; ++ } ++ + if (ReassociateInvariants) { + // Reassociate invariants and prep for split_thru_phi + for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) { +@@ -3094,8 +3113,7 @@ void PhaseIdealLoop::verify() const { + _ltree_root->verify_tree(loop_verify._ltree_root, NULL); + // Reset major-progress. It was cleared by creating a verify version of + // PhaseIdealLoop. +- for( int i=0; iset_major_progress(); ++ C->restore_major_progress(old_progress); + } + + //------------------------------verify_compare--------------------------------- +@@ -4213,7 +4231,6 @@ void PhaseIdealLoop::build_loop_late_post( Node *n ) { + case Op_LoadS: + case Op_LoadP: + case Op_LoadBarrierSlowReg: +- case Op_LoadBarrierWeakSlowReg: + case Op_LoadN: + case Op_LoadRange: + case Op_LoadD_unaligned: +diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp +index 5dad7a9bd..cc679d158 100644 +--- a/src/hotspot/share/opto/loopnode.hpp ++++ b/src/hotspot/share/opto/loopnode.hpp +@@ -929,7 +929,7 @@ public: + } + + // build the loop tree and perform any requested optimizations +- void build_and_optimize(bool do_split_if, bool skip_loop_opts, bool last_round = false); ++ void build_and_optimize(bool do_split_if, bool skip_loop_opts, bool last_round = false, bool z_barrier_insertion = false); + + // Dominators for the sea of nodes + void Dominators(); +@@ -939,13 +939,13 @@ public: + Node *dom_lca_internal( Node *n1, Node *n2 ) const; + + // Compute the Ideal Node to Loop mapping +- PhaseIdealLoop( PhaseIterGVN &igvn, bool do_split_ifs, bool skip_loop_opts = false, bool last_round = false) : ++ PhaseIdealLoop( PhaseIterGVN &igvn, bool do_split_ifs, bool skip_loop_opts = false, bool last_round = false, bool z_barrier_insertion = false) : + PhaseTransform(Ideal_Loop), + _igvn(igvn), + _dom_lca_tags(arena()), // Thread::resource_area + _verify_me(NULL), + _verify_only(false) { +- build_and_optimize(do_split_ifs, skip_loop_opts, last_round); ++ build_and_optimize(do_split_ifs, skip_loop_opts, last_round, z_barrier_insertion); + } + + // Verify that verify_me made the same decisions as a fresh run. +diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp +index 42018e8d9..411cf0841 100644 +--- a/src/hotspot/share/opto/loopopts.cpp ++++ b/src/hotspot/share/opto/loopopts.cpp +@@ -1419,12 +1419,6 @@ void PhaseIdealLoop::split_if_with_blocks_post(Node *n, bool last_round) { + get_loop(get_ctrl(n)) == get_loop(get_ctrl(n->in(1))) ) { + _igvn.replace_node( n, n->in(1) ); + } +- +-#if INCLUDE_ZGC +- if (UseZGC) { +- ZBarrierSetC2::loop_optimize_gc_barrier(this, n, last_round); +- } +-#endif + } + + //------------------------------split_if_with_blocks--------------------------- +diff --git a/src/hotspot/share/opto/matcher.cpp b/src/hotspot/share/opto/matcher.cpp +index 1824c89cd..bdcf9c424 100644 +--- a/src/hotspot/share/opto/matcher.cpp ++++ b/src/hotspot/share/opto/matcher.cpp +@@ -2279,6 +2279,33 @@ void Matcher::find_shared( Node *n ) { + n->del_req(LoadStoreConditionalNode::ExpectedIn); + break; + } ++#if INCLUDE_ZGC ++ case Op_ZCompareAndExchangeP: ++ case Op_ZCompareAndSwapP: ++ case Op_ZWeakCompareAndSwapP: { ++ Node *mem = n->in(MemNode::Address); ++ Node *keepalive = n->in(5); ++ Node *pair1 = new BinaryNode(mem, keepalive); ++ ++ Node *newval = n->in(MemNode::ValueIn); ++ Node *oldval = n->in(LoadStoreConditionalNode::ExpectedIn); ++ Node *pair2 = new BinaryNode(oldval, newval); ++ ++ n->set_req(MemNode::Address, pair1); ++ n->set_req(MemNode::ValueIn, pair2); ++ n->del_req(5); ++ n->del_req(LoadStoreConditionalNode::ExpectedIn); ++ break; ++ } ++ case Op_ZGetAndSetP: { ++ Node *keepalive = n->in(4); ++ Node *newval = n->in(MemNode::ValueIn); ++ Node *pair = new BinaryNode(newval, keepalive); ++ n->set_req(MemNode::ValueIn, pair); ++ n->del_req(4); ++ break; ++ } ++#endif // INCLUDE_ZGC + case Op_CMoveD: // Convert trinary to binary-tree + case Op_CMoveF: + case Op_CMoveI: +diff --git a/src/hotspot/share/opto/memnode.cpp b/src/hotspot/share/opto/memnode.cpp +index 1bbc6fcd0..6fa851af2 100644 +--- a/src/hotspot/share/opto/memnode.cpp ++++ b/src/hotspot/share/opto/memnode.cpp +@@ -927,14 +927,6 @@ static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, + // a load node that reads from the source array so we may be able to + // optimize out the ArrayCopy node later. + Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const { +-#if INCLUDE_ZGC +- if (UseZGC) { +- if (bottom_type()->make_oopptr() != NULL) { +- return NULL; +- } +- } +-#endif +- + Node* ld_adr = in(MemNode::Address); + intptr_t ld_off = 0; + AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off); +@@ -2819,7 +2811,8 @@ const Type* SCMemProjNode::Value(PhaseGVN* phase) const + LoadStoreNode::LoadStoreNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* rt, uint required ) + : Node(required), + _type(rt), +- _adr_type(at) ++ _adr_type(at), ++ _has_barrier(false) + { + init_req(MemNode::Control, c ); + init_req(MemNode::Memory , mem); +@@ -3113,16 +3106,6 @@ Node *MemBarNode::Ideal(PhaseGVN *phase, bool can_reshape) { + return NULL; + } + +-#if INCLUDE_ZGC +- if (UseZGC) { +- if (req() == (Precedent+1) && in(MemBarNode::Precedent)->in(0) != NULL && in(MemBarNode::Precedent)->in(0)->is_LoadBarrier()) { +- Node* load_node = in(MemBarNode::Precedent)->in(0)->in(LoadBarrierNode::Oop); +- set_req(MemBarNode::Precedent, load_node); +- return this; +- } +- } +-#endif +- + bool progress = false; + // Eliminate volatile MemBars for scalar replaced objects. + if (can_reshape && req() == (Precedent+1)) { +diff --git a/src/hotspot/share/opto/memnode.hpp b/src/hotspot/share/opto/memnode.hpp +index e0ffedc06..084a91c49 100644 +--- a/src/hotspot/share/opto/memnode.hpp ++++ b/src/hotspot/share/opto/memnode.hpp +@@ -182,6 +182,8 @@ private: + // this field. + const MemOrd _mo; + ++ uint _barrier; // Bit field with barrier information ++ + protected: + virtual uint cmp(const Node &n) const; + virtual uint size_of() const; // Size is bigger +@@ -193,7 +195,7 @@ protected: + public: + + LoadNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const Type *rt, MemOrd mo, ControlDependency control_dependency) +- : MemNode(c,mem,adr,at), _type(rt), _mo(mo), _control_dependency(control_dependency) { ++ : MemNode(c,mem,adr,at), _type(rt), _mo(mo), _control_dependency(control_dependency), _barrier(0) { + init_class_id(Class_Load); + } + inline bool is_unordered() const { return !is_acquire(); } +@@ -262,6 +264,10 @@ public: + Node* convert_to_unsigned_load(PhaseGVN& gvn); + Node* convert_to_signed_load(PhaseGVN& gvn); + ++ void copy_barrier_info(const Node* src) { _barrier = src->as_Load()->_barrier; } ++ uint barrier_data() { return _barrier; } ++ void set_barrier_data(uint barrier_data) { _barrier |= barrier_data; } ++ + #ifndef PRODUCT + virtual void dump_spec(outputStream *st) const; + #endif +@@ -810,6 +816,7 @@ class LoadStoreNode : public Node { + private: + const Type* const _type; // What kind of value is loaded? + const TypePtr* _adr_type; // What kind of memory is being addressed? ++ bool _has_barrier; + virtual uint size_of() const; // Size is bigger + public: + LoadStoreNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* rt, uint required ); +@@ -822,6 +829,8 @@ public: + + bool result_not_used() const; + MemBarNode* trailing_membar() const; ++ void set_has_barrier() { _has_barrier = true; }; ++ bool has_barrier() const { return _has_barrier; }; + }; + + class LoadStoreConditionalNode : public LoadStoreNode { +diff --git a/src/hotspot/share/opto/node.cpp b/src/hotspot/share/opto/node.cpp +index 434363631..2ff9bb2cd 100644 +--- a/src/hotspot/share/opto/node.cpp ++++ b/src/hotspot/share/opto/node.cpp +@@ -546,6 +546,9 @@ Node *Node::clone() const { + if (n->is_SafePoint()) { + n->as_SafePoint()->clone_replaced_nodes(); + } ++ if (n->is_Load()) { ++ n->as_Load()->copy_barrier_info(this); ++ } + return n; // Return the clone + } + +@@ -1462,10 +1465,14 @@ bool Node::rematerialize() const { + //------------------------------needs_anti_dependence_check--------------------- + // Nodes which use memory without consuming it, hence need antidependences. + bool Node::needs_anti_dependence_check() const { +- if( req() < 2 || (_flags & Flag_needs_anti_dependence_check) == 0 ) ++ if( req() < 2 || (_flags & Flag_needs_anti_dependence_check) == 0 ) { + return false; +- else +- return in(1)->bottom_type()->has_memory(); ++ } ++ BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); ++ if (!bs->needs_anti_dependence_check(this)) { ++ return false; ++ } ++ return in(1)->bottom_type()->has_memory(); + } + + +diff --git a/src/hotspot/share/opto/node.hpp b/src/hotspot/share/opto/node.hpp +index 608aa0c8f..49f415a31 100644 +--- a/src/hotspot/share/opto/node.hpp ++++ b/src/hotspot/share/opto/node.hpp +@@ -84,8 +84,8 @@ class JumpProjNode; + class LoadNode; + class LoadBarrierNode; + class LoadBarrierSlowRegNode; +-class LoadBarrierWeakSlowRegNode; + class LoadStoreNode; ++class LoadStoreConditionalNode; + class LockNode; + class LoopNode; + class MachBranchNode; +@@ -691,8 +691,7 @@ public: + DEFINE_CLASS_ID(Mem, Node, 4) + DEFINE_CLASS_ID(Load, Mem, 0) + DEFINE_CLASS_ID(LoadVector, Load, 0) +- DEFINE_CLASS_ID(LoadBarrierSlowReg, Load, 1) +- DEFINE_CLASS_ID(LoadBarrierWeakSlowReg, Load, 2) ++ DEFINE_CLASS_ID(LoadBarrierSlowReg, Load, 1) + DEFINE_CLASS_ID(Store, Mem, 1) + DEFINE_CLASS_ID(StoreVector, Store, 0) + DEFINE_CLASS_ID(LoadStore, Mem, 2) +@@ -834,9 +833,9 @@ public: + DEFINE_CLASS_QUERY(JumpProj) + DEFINE_CLASS_QUERY(Load) + DEFINE_CLASS_QUERY(LoadStore) ++ DEFINE_CLASS_QUERY(LoadStoreConditional) + DEFINE_CLASS_QUERY(LoadBarrier) + DEFINE_CLASS_QUERY(LoadBarrierSlowReg) +- DEFINE_CLASS_QUERY(LoadBarrierWeakSlowReg) + DEFINE_CLASS_QUERY(Lock) + DEFINE_CLASS_QUERY(Loop) + DEFINE_CLASS_QUERY(Mach) +diff --git a/src/hotspot/share/opto/phaseX.cpp b/src/hotspot/share/opto/phaseX.cpp +index a94233cdc..880efe991 100644 +--- a/src/hotspot/share/opto/phaseX.cpp ++++ b/src/hotspot/share/opto/phaseX.cpp +@@ -958,9 +958,6 @@ PhaseIterGVN::PhaseIterGVN( PhaseGVN *gvn ) : PhaseGVN(gvn), + n->is_Mem() ) + add_users_to_worklist(n); + } +- +- BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); +- bs->add_users_to_worklist(&_worklist); + } + + /** +diff --git a/src/hotspot/share/opto/phasetype.hpp b/src/hotspot/share/opto/phasetype.hpp +index f0eac0b0b..b01b7630f 100644 +--- a/src/hotspot/share/opto/phasetype.hpp ++++ b/src/hotspot/share/opto/phasetype.hpp +@@ -53,6 +53,9 @@ enum CompilerPhaseType { + PHASE_INCREMENTAL_INLINE, + PHASE_INCREMENTAL_BOXING_INLINE, + PHASE_BEFORE_MACRO_EXPANSION, ++ PHASE_CALL_CATCH_CLEANUP, ++ PHASE_INSERT_BARRIER, ++ PHASE_ADD_UNSAFE_BARRIER, + PHASE_END, + PHASE_FAILURE, + +@@ -90,6 +93,9 @@ class CompilerPhaseTypeHelper { + case PHASE_INCREMENTAL_INLINE: return "Incremental Inline"; + case PHASE_INCREMENTAL_BOXING_INLINE: return "Incremental Boxing Inline"; + case PHASE_BEFORE_MACRO_EXPANSION: return "Before macro expansion"; ++ case PHASE_CALL_CATCH_CLEANUP: return "Call catch cleanup"; ++ case PHASE_INSERT_BARRIER: return "Insert barrier"; ++ case PHASE_ADD_UNSAFE_BARRIER: return "Add barrier to unsafe op"; + case PHASE_END: return "End"; + case PHASE_FAILURE: return "Failure"; + default: +diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp +index 593baf972..b40587ba5 100644 +--- a/src/hotspot/share/opto/vectornode.cpp ++++ b/src/hotspot/share/opto/vectornode.cpp +@@ -253,7 +253,6 @@ void VectorNode::vector_operands(Node* n, uint* start, uint* end) { + case Op_LoadF: case Op_LoadD: + case Op_LoadP: case Op_LoadN: + case Op_LoadBarrierSlowReg: +- case Op_LoadBarrierWeakSlowReg: + *start = 0; + *end = 0; // no vector operands + break; +diff --git a/src/hotspot/share/runtime/stackValue.cpp b/src/hotspot/share/runtime/stackValue.cpp +index e7a122ce1..3573555c8 100644 +--- a/src/hotspot/share/runtime/stackValue.cpp ++++ b/src/hotspot/share/runtime/stackValue.cpp +@@ -122,13 +122,7 @@ StackValue* StackValue::create_stack_value(const frame* fr, const RegisterMap* r + val = (oop)NULL; + } + #endif +-#if INCLUDE_ZGC +- // Deoptimization must make sure all oop have passed load barrier +- if (UseZGC) { +- val = ZBarrier::load_barrier_on_oop_field_preloaded((oop*)value_addr, val); +- } +-#endif +- ++ assert(oopDesc::is_oop_or_null(val, false), "bad oop found"); + Handle h(Thread::current(), val); // Wrap a handle around the oop + return new StackValue(h); + } +diff --git a/src/hotspot/share/utilities/growableArray.hpp b/src/hotspot/share/utilities/growableArray.hpp +index eb59003a7..ba8304ab7 100644 +--- a/src/hotspot/share/utilities/growableArray.hpp ++++ b/src/hotspot/share/utilities/growableArray.hpp +@@ -152,6 +152,12 @@ class GenericGrowableArray : public ResourceObj { + template class GrowableArrayIterator; + template class GrowableArrayFilterIterator; + ++template ++class CompareClosure : public Closure { ++public: ++ virtual int do_compare(const E&, const E&) = 0; ++}; ++ + template class GrowableArray : public GenericGrowableArray { + friend class VMStructs; + +@@ -443,6 +449,37 @@ template class GrowableArray : public GenericGrowableArray { + } + return min; + } ++ ++ E insert_sorted(CompareClosure* cc, const E& key) { ++ bool found; ++ int location = find_sorted(cc, key, found); ++ if (!found) { ++ insert_before(location, key); ++ } ++ return at(location); ++ } ++ ++ template ++ int find_sorted(CompareClosure* cc, const K& key, bool& found) { ++ found = false; ++ int min = 0; ++ int max = length() - 1; ++ ++ while (max >= min) { ++ int mid = (int)(((uint)max + min) / 2); ++ E value = at(mid); ++ int diff = cc->do_compare(key, value); ++ if (diff > 0) { ++ min = mid + 1; ++ } else if (diff < 0) { ++ max = mid - 1; ++ } else { ++ found = true; ++ return mid; ++ } ++ } ++ return min; ++ } + }; + + // Global GrowableArray methods (one instance in the library per each 'E' type). +-- +2.12.3 + diff --git a/8225648-TESTBUG-java-lang-annotation-loaderLeak-Main.patch b/8225648-TESTBUG-java-lang-annotation-loaderLeak-Main.patch new file mode 100644 index 0000000..8f9fbac --- /dev/null +++ b/8225648-TESTBUG-java-lang-annotation-loaderLeak-Main.patch @@ -0,0 +1,27 @@ +From 96ac09c507a663b853235c29a607aca2c1edfc10 Mon Sep 17 00:00:00 2001 +Date: Mon, 2 Dec 2019 17:42:17 +0000 +Subject: [PATCH] 8225648: [TESTBUG]java/lang/annotation/loaderLeak/Main.java + fails with -Xcomp + +Summary: : annotations cause memory leak +LLT: NA +Bug url: https://bugs.openjdk.java.net/browse/JDK-8225648 +--- + test/jdk/java/lang/annotation/loaderLeak/Main.java | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/test/jdk/java/lang/annotation/loaderLeak/Main.java b/test/jdk/java/lang/annotation/loaderLeak/Main.java +index 7e249ebc1..4245008f8 100644 +--- a/test/jdk/java/lang/annotation/loaderLeak/Main.java ++++ b/test/jdk/java/lang/annotation/loaderLeak/Main.java +@@ -54,6 +54,7 @@ public class Main { + System.gc(); + System.gc(); + if (c.get() == null) throw new AssertionError(); ++ Reference.reachabilityFence(loader); + System.gc(); + System.gc(); + loader = null; +-- +2.12.3 + diff --git a/8226536-Catch-OOM-from-deopt-that-fails-rematerializ.patch b/8226536-Catch-OOM-from-deopt-that-fails-rematerializ.patch new file mode 100644 index 0000000..8af83f7 --- /dev/null +++ b/8226536-Catch-OOM-from-deopt-that-fails-rematerializ.patch @@ -0,0 +1,76 @@ +From 7a656d6c14da1ec666ee5ebf55995c6b6bf4ab7f Mon Sep 17 00:00:00 2001 +Date: Tue, 17 Mar 2020 11:06:41 +0800 +Subject: [PATCH] 8226536: Catch OOM from deopt that fails rematerializing + objects + +Summary: : fix testcase, catch OOM from deopt that fails rematerializing objects +LLT: jtreg +Bug url: https://bugs.openjdk.java.net/browse/JDK-8226536 +--- + .../vmTestbase/nsk/share/gc/gp/GarbageUtils.java | 36 ++++++++++++++++++++-- + 1 file changed, 34 insertions(+), 2 deletions(-) + +diff --git a/test/hotspot/jtreg/vmTestbase/nsk/share/gc/gp/GarbageUtils.java b/test/hotspot/jtreg/vmTestbase/nsk/share/gc/gp/GarbageUtils.java +index 333bf070b..4d18a6871 100644 +--- a/test/hotspot/jtreg/vmTestbase/nsk/share/gc/gp/GarbageUtils.java ++++ b/test/hotspot/jtreg/vmTestbase/nsk/share/gc/gp/GarbageUtils.java +@@ -26,6 +26,7 @@ package nsk.share.gc.gp; + import java.io.IOException; + import java.io.PrintWriter; + import java.io.StringWriter; ++import java.lang.invoke.*; + import java.util.*; + import nsk.share.gc.gp.array.*; + import nsk.share.gc.gp.string.*; +@@ -193,6 +194,36 @@ public final class GarbageUtils { + return eatMemory(stresser, gp, initialFactor, minMemoryChunk, factor, OOM_TYPE.ANY); + } + ++ static int numberOfOOMEs = 0; ++ ++ /** ++ * Minimal wrapper of the main implementation. Catches any OOM ++ * that might be thrown when rematerializing Objects when deoptimizing. ++ * ++ * It is Important that the impl is not inlined. ++ */ ++ ++ public static int eatMemory(ExecutionController stresser, GarbageProducer gp, long initialFactor, long minMemoryChunk, long factor, OOM_TYPE type) { ++ try { ++ // Using a methodhandle invoke of eatMemoryImpl to prevent inlining of it ++ MethodHandles.Lookup lookup = MethodHandles.lookup(); ++ MethodType mt = MethodType.methodType( ++ int.class, ++ ExecutionController.class, ++ GarbageProducer.class, ++ long.class, ++ long.class, ++ long.class, ++ OOM_TYPE.class); ++ MethodHandle eat = lookup.findStatic(GarbageUtils.class, "eatMemoryImpl", mt); ++ return (int) eat.invoke(stresser, gp, initialFactor, minMemoryChunk, factor, type); ++ } catch (OutOfMemoryError e) { ++ return numberOfOOMEs++; ++ } catch (Throwable t) { ++ throw new RuntimeException(t); ++ } ++ } ++ + /** + * Eat memory using given garbage producer. + * +@@ -210,8 +241,9 @@ public final class GarbageUtils { + * @param type of OutOfMemory Exception: Java heap space or Metadata space + * @return number of OOME occured + */ +- public static int eatMemory(ExecutionController stresser, GarbageProducer gp, long initialFactor, long minMemoryChunk, long factor, OOM_TYPE type) { +- int numberOfOOMEs = 0; ++ ++ public static int eatMemoryImpl(ExecutionController stresser, GarbageProducer gp, long initialFactor, long minMemoryChunk, long factor, OOM_TYPE type) { ++ numberOfOOMEs = 0; + try { + StringWriter sw = new StringWriter(10000); + PrintWriter pw = new PrintWriter(sw); +-- +2.12.3 + diff --git a/8231584-Deadlock-with-ClassLoader.findLibrary-and-Sy.patch b/8231584-Deadlock-with-ClassLoader.findLibrary-and-Sy.patch new file mode 100644 index 0000000..0cdbe05 --- /dev/null +++ b/8231584-Deadlock-with-ClassLoader.findLibrary-and-Sy.patch @@ -0,0 +1,365 @@ +From 54e889ba40a633fdac60674e54a1ca948fa3d3bf Mon Sep 17 00:00:00 2001 +Date: Wed, 27 Nov 2019 15:11:50 +0000 +Subject: [PATCH] 8231584: Deadlock with ClassLoader.findLibrary and + System.loadLibrary call + +Summary: : Deadlock with ClassLoader.findLibrary and System.loadLibrary call +LLT: LoadLibraryTest.java +Bug url: https://bugs.openjdk.java.net/browse/JDK-8231584 +--- + .../share/classes/java/lang/ClassLoader.java | 23 ++- + src/java.base/share/classes/java/lang/Runtime.java | 8 +- + src/java.base/share/classes/java/lang/System.java | 2 + + .../lang/Runtime/loadLibrary/LoadLibraryTest.java | 158 +++++++++++++++++++++ + .../java/lang/Runtime/loadLibrary/src/Target.java | 34 +++++ + .../java/lang/Runtime/loadLibrary/src/Target2.java | 29 ++++ + 6 files changed, 243 insertions(+), 11 deletions(-) + create mode 100644 test/jdk/java/lang/Runtime/loadLibrary/LoadLibraryTest.java + create mode 100644 test/jdk/java/lang/Runtime/loadLibrary/src/Target.java + create mode 100644 test/jdk/java/lang/Runtime/loadLibrary/src/Target2.java + +diff --git a/src/java.base/share/classes/java/lang/ClassLoader.java b/src/java.base/share/classes/java/lang/ClassLoader.java +index aa211b079..7fb707508 100644 +--- a/src/java.base/share/classes/java/lang/ClassLoader.java ++++ b/src/java.base/share/classes/java/lang/ClassLoader.java +@@ -1,5 +1,5 @@ + /* +- * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. ++ * Copyright (c) 2019, Azul Systems, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it +@@ -1994,6 +1994,17 @@ public abstract class ClassLoader { + return scl; + } + ++ /* ++ * Initialize default paths for native libraries search. ++ * Must be done early as JDK may load libraries during bootstrap. ++ * ++ * @see java.lang.System#initPhase1 ++ */ ++ static void initLibraryPaths() { ++ usr_paths = initializePath("java.library.path"); ++ sys_paths = initializePath("sun.boot.library.path"); ++ } ++ + // Returns true if the specified class loader can be found in this class + // loader's delegation chain. + boolean isAncestor(ClassLoader cl) { +@@ -2463,8 +2474,7 @@ public abstract class ClassLoader { + * + * We use a static stack to hold the list of libraries we are + * loading because this can happen only when called by the +- * same thread because Runtime.load and Runtime.loadLibrary +- * are synchronous. ++ * same thread because this block is synchronous + * + * If there is a pending load operation for the library, we + * immediately return success; otherwise, we raise +@@ -2609,10 +2619,9 @@ public abstract class ClassLoader { + boolean isAbsolute) { + ClassLoader loader = + (fromClass == null) ? null : fromClass.getClassLoader(); +- if (sys_paths == null) { +- usr_paths = initializePath("java.library.path"); +- sys_paths = initializePath("sun.boot.library.path"); +- } ++ assert sys_paths != null : "should be initialized at this point"; ++ assert usr_paths != null : "should be initialized at this point"; ++ + if (isAbsolute) { + if (loadLibrary0(fromClass, new File(name))) { + return; +diff --git a/src/java.base/share/classes/java/lang/Runtime.java b/src/java.base/share/classes/java/lang/Runtime.java +index 128e6dbc8..3a9a7d838 100644 +--- a/src/java.base/share/classes/java/lang/Runtime.java ++++ b/src/java.base/share/classes/java/lang/Runtime.java +@@ -1,5 +1,5 @@ + /* +- * Copyright (c) 1995, 2018, Oracle and/or its affiliates. All rights reserved. ++ * Copyright (c) 2019, Azul Systems, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it +@@ -755,7 +755,7 @@ public class Runtime { + load0(Reflection.getCallerClass(), filename); + } + +- synchronized void load0(Class fromClass, String filename) { ++ void load0(Class fromClass, String filename) { + SecurityManager security = System.getSecurityManager(); + if (security != null) { + security.checkLink(filename); +@@ -817,14 +817,14 @@ public class Runtime { + loadLibrary0(Reflection.getCallerClass(), libname); + } + +- synchronized void loadLibrary0(Class fromClass, String libname) { ++ void loadLibrary0(Class fromClass, String libname) { + SecurityManager security = System.getSecurityManager(); + if (security != null) { + security.checkLink(libname); + } + if (libname.indexOf((int)File.separatorChar) != -1) { + throw new UnsatisfiedLinkError( +- "Directory separator should not appear in library name: " + libname); ++ "Directory separator should not appear in library name: " + libname); + } + ClassLoader.loadLibrary(fromClass, libname, false); + } +diff --git a/src/java.base/share/classes/java/lang/System.java b/src/java.base/share/classes/java/lang/System.java +index 2bb5390c0..216f33586 100644 +--- a/src/java.base/share/classes/java/lang/System.java ++++ b/src/java.base/share/classes/java/lang/System.java +@@ -1988,6 +1988,8 @@ public final class System { + // register shared secrets + setJavaLangAccess(); + ++ ClassLoader.initLibraryPaths(); ++ + // Subsystems that are invoked during initialization can invoke + // VM.isBooted() in order to avoid doing things that should + // wait until the VM is fully initialized. The initialization level +diff --git a/test/jdk/java/lang/Runtime/loadLibrary/LoadLibraryTest.java b/test/jdk/java/lang/Runtime/loadLibrary/LoadLibraryTest.java +new file mode 100644 +index 000000000..f08c60dfa +--- /dev/null ++++ b/test/jdk/java/lang/Runtime/loadLibrary/LoadLibraryTest.java +@@ -0,0 +1,158 @@ ++/* ++ * Copyright (c) 2018, Amazon and/or its affiliates. All rights reserved. ++ * Copyright (c) 2019, Azul Systems, Inc. All rights reserved. ++ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++ * ++ * This code is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License version 2 only, as ++ * published by the Free Software Foundation. ++ * ++ * This code is distributed in the hope that it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++ * version 2 for more details (a copy is included in the LICENSE file that ++ * accompanied this code). ++ * ++ * You should have received a copy of the GNU General Public License version ++ * 2 along with this work; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++ * or visit www.oracle.com if you need additional information or have any ++ * questions. ++ */ ++ ++/** ++ * @test ++ * @bug 8231584 ++ * @library /test/lib ++ * @run main/othervm LoadLibraryTest ++ */ ++ ++import java.nio.file.FileSystems; ++import java.nio.file.Files; ++import java.nio.file.Paths; ++import java.nio.file.Path; ++import java.net.MalformedURLException; ++import java.net.URLClassLoader; ++import java.net.URL; ++ ++import jdk.test.lib.compiler.CompilerUtils; ++ ++public class LoadLibraryTest { ++ static Thread thread1 = null; ++ static Thread thread2 = null; ++ ++ static volatile boolean thread1Ready = false; ++ ++ private static final String TEST_SRC = System.getProperty("test.src"); ++ private static final Path SRC_DIR = Paths.get(TEST_SRC, "src"); ++ private static final Path CLS_DIR = Paths.get("classes"); ++ ++ static TestClassLoader loader; ++ static void someLibLoad() { ++ try { ++/* ++ FileSystems.getDefault(); ++ ++ // jdk/jdk: loads directly from Bootstrap Classloader (doesn't take lock on Runtime) ++ java.net.NetworkInterface.getNetworkInterfaces(); ++ ++ System.out.println(jdk.net.ExtendedSocketOptions.SO_FLOW_SLA); ++*/ ++ Class c = Class.forName("Target2", true, loader); ++ } catch (Exception e) { ++ throw new RuntimeException(e); ++ } ++ } ++ ++ static class TestClassLoader extends URLClassLoader { ++ boolean passed = false; ++ ++ public boolean passed() { ++ return passed; ++ } ++ ++ TestClassLoader() throws MalformedURLException { ++ super(new URL[] { new URL("file://" + CLS_DIR.toAbsolutePath().toString() + '/') }); ++ } ++ ++ public String findLibrary(String name) { ++ System.out.println("findLibrary " + name); ++ ++ if ("someLibrary".equals(name)) { ++ try { ++ synchronized(thread1) { ++ while(!thread1Ready) { ++ thread1.wait(); ++ } ++ thread1.notifyAll(); ++ } ++ ++ Thread.sleep(10000); ++ ++ System.out.println("Thread2 load"); ++ someLibLoad(); ++ ++ // no deadlock happened ++ passed = true; ++ } catch (Exception e) { ++ throw new RuntimeException(e); ++ } ++ return null; ++ } ++ ++ return super.findLibrary(name); ++ } ++ } ++ ++ ++ public static void main(String[] args) throws Exception { ++ loader = new TestClassLoader(); ++ ++ if (!CompilerUtils.compile(SRC_DIR, CLS_DIR)) { ++ throw new Exception("Can't compile"); ++ } ++ ++ thread1 = new Thread() { ++ public void run() { ++ try { ++ synchronized(this) { ++ thread1Ready = true; ++ thread1.notifyAll(); ++ thread1.wait(); ++ } ++ } catch(InterruptedException e) { ++ throw new RuntimeException(e); ++ } ++ ++ System.out.println("Thread1 load"); ++ someLibLoad(); ++ }; ++ }; ++ ++ thread2 = new Thread() { ++ public void run() { ++ try { ++ Class c = Class.forName("Target", true, loader); ++ System.out.println(c); ++ } catch (Exception e) { ++ throw new RuntimeException(e); ++ } ++ }; ++ }; ++ ++ thread1.setDaemon(true); ++ thread2.setDaemon(true); ++ ++ thread1.start(); ++ thread2.start(); ++ ++ thread1.join(); ++ thread2.join(); ++ ++ if (!loader.passed()) { ++ throw new RuntimeException("FAIL"); ++ } ++ } ++} +diff --git a/test/jdk/java/lang/Runtime/loadLibrary/src/Target.java b/test/jdk/java/lang/Runtime/loadLibrary/src/Target.java +new file mode 100644 +index 000000000..fc5148105 +--- /dev/null ++++ b/test/jdk/java/lang/Runtime/loadLibrary/src/Target.java +@@ -0,0 +1,34 @@ ++/* ++ * Copyright (c) 2019, Azul Systems, Inc. All rights reserved. ++ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++ * ++ * This code is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License version 2 only, as ++ * published by the Free Software Foundation. ++ * ++ * This code is distributed in the hope that it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++ * version 2 for more details (a copy is included in the LICENSE file that ++ * accompanied this code). ++ * ++ * You should have received a copy of the GNU General Public License version ++ * 2 along with this work; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++ * or visit www.oracle.com if you need additional information or have any ++ * questions. ++ */ ++ ++class Target { ++ static { ++ try { ++ System.loadLibrary("someLibrary"); ++ throw new RuntimeException("someLibrary was loaded"); ++ } catch (UnsatisfiedLinkError e) { ++ // expected: we do not have a someLibrary ++ } ++ } ++} ++ +diff --git a/test/jdk/java/lang/Runtime/loadLibrary/src/Target2.java b/test/jdk/java/lang/Runtime/loadLibrary/src/Target2.java +new file mode 100644 +index 000000000..bc8dfc5e6 +--- /dev/null ++++ b/test/jdk/java/lang/Runtime/loadLibrary/src/Target2.java +@@ -0,0 +1,29 @@ ++/* ++ * Copyright (c) 2019, Azul Systems, Inc. All rights reserved. ++ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++ * ++ * This code is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License version 2 only, as ++ * published by the Free Software Foundation. ++ * ++ * This code is distributed in the hope that it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++ * version 2 for more details (a copy is included in the LICENSE file that ++ * accompanied this code). ++ * ++ * You should have received a copy of the GNU General Public License version ++ * 2 along with this work; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++ * or visit www.oracle.com if you need additional information or have any ++ * questions. ++ */ ++ ++class Target2 { ++ static { ++ System.loadLibrary("awt"); ++ } ++} ++ +-- +2.12.3 + diff --git a/8233061-ZGC-Enforce-memory-ordering-in-segmented-bit.patch b/8233061-ZGC-Enforce-memory-ordering-in-segmented-bit.patch new file mode 100644 index 0000000..a851739 --- /dev/null +++ b/8233061-ZGC-Enforce-memory-ordering-in-segmented-bit.patch @@ -0,0 +1,101 @@ +From d2137837d518a8bdb8e075109e502e78bd2f9fa9 Mon Sep 17 00:00:00 2001 +Date: Wed, 19 Feb 2020 17:36:32 +0800 +Subject: [PATCH] 8233061: ZGC: Enforce memory ordering in segmented bit maps + +Summary: : +LLT: renaissance +Bug url: https://bugs.openjdk.java.net/browse/JDK-8233061 +--- + src/hotspot/share/gc/z/zLiveMap.cpp | 20 +++++++++----------- + src/hotspot/share/gc/z/zLiveMap.inline.hpp | 9 +++++---- + 2 files changed, 14 insertions(+), 15 deletions(-) + +diff --git a/src/hotspot/share/gc/z/zLiveMap.cpp b/src/hotspot/share/gc/z/zLiveMap.cpp +index 7187b6166..c1d79b794 100644 +--- a/src/hotspot/share/gc/z/zLiveMap.cpp ++++ b/src/hotspot/share/gc/z/zLiveMap.cpp +@@ -50,7 +50,9 @@ void ZLiveMap::reset(size_t index) { + + // Multiple threads can enter here, make sure only one of them + // resets the marking information while the others busy wait. +- for (uint32_t seqnum = _seqnum; seqnum != ZGlobalSeqNum; seqnum = _seqnum) { ++ for (uint32_t seqnum = OrderAccess::load_acquire(&_seqnum); ++ seqnum != ZGlobalSeqNum; ++ seqnum = OrderAccess::load_acquire(&_seqnum)) { + if ((seqnum != seqnum_initializing) && + (Atomic::cmpxchg(seqnum_initializing, &_seqnum, seqnum) == seqnum)) { + // Reset marking information +@@ -61,13 +63,13 @@ void ZLiveMap::reset(size_t index) { + segment_live_bits().clear(); + segment_claim_bits().clear(); + +- // Make sure the newly reset marking information is +- // globally visible before updating the page seqnum. +- OrderAccess::storestore(); +- +- // Update seqnum + assert(_seqnum == seqnum_initializing, "Invalid"); +- _seqnum = ZGlobalSeqNum; ++ ++ // Make sure the newly reset marking information is ordered ++ // before the update of the page seqnum, such that when the ++ // up-to-date seqnum is load acquired, the bit maps will not ++ // contain stale information. ++ OrderAccess::release_store(&_seqnum, ZGlobalSeqNum); + break; + } + +@@ -89,10 +91,6 @@ void ZLiveMap::reset_segment(BitMap::idx_t segment) { + if (!claim_segment(segment)) { + // Already claimed, wait for live bit to be set + while (!is_segment_live(segment)) { +- // Busy wait. The loadload barrier is needed to make +- // sure we re-read the live bit every time we loop. +- OrderAccess::loadload(); +- + // Mark reset contention + if (!contention) { + // Count contention once +diff --git a/src/hotspot/share/gc/z/zLiveMap.inline.hpp b/src/hotspot/share/gc/z/zLiveMap.inline.hpp +index 1e4d56f41..fb45a892c 100644 +--- a/src/hotspot/share/gc/z/zLiveMap.inline.hpp ++++ b/src/hotspot/share/gc/z/zLiveMap.inline.hpp +@@ -30,6 +30,7 @@ + #include "gc/z/zOop.inline.hpp" + #include "gc/z/zUtils.inline.hpp" + #include "runtime/atomic.hpp" ++#include "runtime/orderAccess.hpp" + #include "utilities/bitMap.inline.hpp" + #include "utilities/debug.hpp" + +@@ -38,7 +39,7 @@ inline void ZLiveMap::reset() { + } + + inline bool ZLiveMap::is_marked() const { +- return _seqnum == ZGlobalSeqNum; ++ return OrderAccess::load_acquire(&_seqnum) == ZGlobalSeqNum; + } + + inline uint32_t ZLiveMap::live_objects() const { +@@ -68,15 +69,15 @@ inline BitMapView ZLiveMap::segment_claim_bits() { + } + + inline bool ZLiveMap::is_segment_live(BitMap::idx_t segment) const { +- return segment_live_bits().at(segment); ++ return segment_live_bits().par_at(segment); + } + + inline bool ZLiveMap::set_segment_live_atomic(BitMap::idx_t segment) { +- return segment_live_bits().par_set_bit(segment); ++ return segment_live_bits().par_set_bit(segment, memory_order_release); + } + + inline bool ZLiveMap::claim_segment(BitMap::idx_t segment) { +- return segment_claim_bits().par_set_bit(segment); ++ return segment_claim_bits().par_set_bit(segment, memory_order_acq_rel); + } + + inline BitMap::idx_t ZLiveMap::first_live_segment() const { +-- +2.12.3 + diff --git a/8233073-Make-BitMap-accessors-more-memory-ordering-f.patch b/8233073-Make-BitMap-accessors-more-memory-ordering-f.patch new file mode 100644 index 0000000..d460525 --- /dev/null +++ b/8233073-Make-BitMap-accessors-more-memory-ordering-f.patch @@ -0,0 +1,180 @@ +From d1c89ec547d6d59c56874b458fc49f56e25011b2 Mon Sep 17 00:00:00 2001 +Date: Wed, 19 Feb 2020 17:17:43 +0800 +Subject: [PATCH] 8233073: Make BitMap accessors more memory ordering friendly + +Summary: : +LLT: renaissance +Bug url: https://bugs.openjdk.java.net/browse/JDK-8233073 +--- + src/hotspot/share/c1/c1_Instruction.cpp | 1 + + src/hotspot/share/opto/graphKit.cpp | 1 + + src/hotspot/share/opto/parse1.cpp | 1 + + src/hotspot/share/utilities/bitMap.hpp | 17 +++++++++++--- + src/hotspot/share/utilities/bitMap.inline.hpp | 34 ++++++++++++++++++++++----- + 5 files changed, 45 insertions(+), 9 deletions(-) + +diff --git a/src/hotspot/share/c1/c1_Instruction.cpp b/src/hotspot/share/c1/c1_Instruction.cpp +index ee3be899a..62d8b48bc 100644 +--- a/src/hotspot/share/c1/c1_Instruction.cpp ++++ b/src/hotspot/share/c1/c1_Instruction.cpp +@@ -29,6 +29,7 @@ + #include "c1/c1_ValueStack.hpp" + #include "ci/ciObjArrayKlass.hpp" + #include "ci/ciTypeArrayKlass.hpp" ++#include "utilities/bitMap.inline.hpp" + + + // Implementation of Instruction +diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp +index 2ceb3a6ae..981d6b860 100644 +--- a/src/hotspot/share/opto/graphKit.cpp ++++ b/src/hotspot/share/opto/graphKit.cpp +@@ -43,6 +43,7 @@ + #include "opto/runtime.hpp" + #include "runtime/deoptimization.hpp" + #include "runtime/sharedRuntime.hpp" ++#include "utilities/bitMap.inline.hpp" + + //----------------------------GraphKit----------------------------------------- + // Main utility constructor. +diff --git a/src/hotspot/share/opto/parse1.cpp b/src/hotspot/share/opto/parse1.cpp +index a6408b493..0b30303a3 100644 +--- a/src/hotspot/share/opto/parse1.cpp ++++ b/src/hotspot/share/opto/parse1.cpp +@@ -41,6 +41,7 @@ + #include "runtime/handles.inline.hpp" + #include "runtime/safepointMechanism.hpp" + #include "runtime/sharedRuntime.hpp" ++#include "utilities/bitMap.inline.hpp" + #include "utilities/copy.hpp" + + // Static array so we can figure out which bytecodes stop us from compiling +diff --git a/src/hotspot/share/utilities/bitMap.hpp b/src/hotspot/share/utilities/bitMap.hpp +index 656e11fb7..4d6749f5b 100644 +--- a/src/hotspot/share/utilities/bitMap.hpp ++++ b/src/hotspot/share/utilities/bitMap.hpp +@@ -26,6 +26,7 @@ + #define SHARE_VM_UTILITIES_BITMAP_HPP + + #include "memory/allocation.hpp" ++#include "runtime/atomic.hpp" + #include "utilities/align.hpp" + + // Forward decl; +@@ -94,6 +95,8 @@ class BitMap { + void set_word (idx_t word) { set_word(word, ~(bm_word_t)0); } + void clear_word(idx_t word) { _map[word] = 0; } + ++ static inline const bm_word_t load_word_ordered(const volatile bm_word_t* const addr, atomic_memory_order memory_order); ++ + // Utilities for ranges of bits. Ranges are half-open [beg, end). + + // Ranges within a single word. +@@ -193,6 +196,9 @@ class BitMap { + return (*word_addr(index) & bit_mask(index)) != 0; + } + ++ // memory_order must be memory_order_relaxed or memory_order_acquire. ++ bool par_at(idx_t index, atomic_memory_order memory_order = memory_order_acquire) const; ++ + // Align bit index up or down to the next bitmap word boundary, or check + // alignment. + static idx_t word_align_up(idx_t bit) { +@@ -209,9 +215,14 @@ class BitMap { + inline void set_bit(idx_t bit); + inline void clear_bit(idx_t bit); + +- // Atomically set or clear the specified bit. +- inline bool par_set_bit(idx_t bit); +- inline bool par_clear_bit(idx_t bit); ++ // Attempts to change a bit to a desired value. The operation returns true if ++ // this thread changed the value of the bit. It was changed with a RMW operation ++ // using the specified memory_order. The operation returns false if the change ++ // could not be set due to the bit already being observed in the desired state. ++ // The atomic access that observed the bit in the desired state has acquire ++ // semantics, unless memory_order is memory_order_relaxed or memory_order_release. ++ inline bool par_set_bit(idx_t bit, atomic_memory_order memory_order = memory_order_conservative); ++ inline bool par_clear_bit(idx_t bit, atomic_memory_order memory_order = memory_order_conservative); + + // Put the given value at the given offset. The parallel version + // will CAS the value into the bitmap and is quite a bit slower. +diff --git a/src/hotspot/share/utilities/bitMap.inline.hpp b/src/hotspot/share/utilities/bitMap.inline.hpp +index b10726d18..7a7e2ad43 100644 +--- a/src/hotspot/share/utilities/bitMap.inline.hpp ++++ b/src/hotspot/share/utilities/bitMap.inline.hpp +@@ -26,6 +26,7 @@ + #define SHARE_VM_UTILITIES_BITMAP_INLINE_HPP + + #include "runtime/atomic.hpp" ++#include "runtime/orderAccess.hpp" + #include "utilities/bitMap.hpp" + + inline void BitMap::set_bit(idx_t bit) { +@@ -38,18 +39,39 @@ inline void BitMap::clear_bit(idx_t bit) { + *word_addr(bit) &= ~bit_mask(bit); + } + +-inline bool BitMap::par_set_bit(idx_t bit) { ++inline const BitMap::bm_word_t BitMap::load_word_ordered(const volatile bm_word_t* const addr, atomic_memory_order memory_order) { ++ if (memory_order == memory_order_relaxed || memory_order == memory_order_release) { ++ return Atomic::load(addr); ++ } else { ++ assert(memory_order == memory_order_acq_rel || ++ memory_order == memory_order_acquire || ++ memory_order == memory_order_conservative, ++ "unexpected memory ordering"); ++ return OrderAccess::load_acquire(addr); ++ } ++} ++ ++inline bool BitMap::par_at(idx_t index, atomic_memory_order memory_order) const { ++ verify_index(index); ++ assert(memory_order == memory_order_acquire || ++ memory_order == memory_order_relaxed, ++ "unexpected memory ordering"); ++ const volatile bm_word_t* const addr = word_addr(index); ++ return (load_word_ordered(addr, memory_order) & bit_mask(index)) != 0; ++} ++ ++inline bool BitMap::par_set_bit(idx_t bit, atomic_memory_order memory_order) { + verify_index(bit); + volatile bm_word_t* const addr = word_addr(bit); + const bm_word_t mask = bit_mask(bit); +- bm_word_t old_val = *addr; ++ bm_word_t old_val = load_word_ordered(addr, memory_order); + + do { + const bm_word_t new_val = old_val | mask; + if (new_val == old_val) { + return false; // Someone else beat us to it. + } +- const bm_word_t cur_val = Atomic::cmpxchg(new_val, addr, old_val); ++ const bm_word_t cur_val = Atomic::cmpxchg(new_val, addr, old_val, memory_order); + if (cur_val == old_val) { + return true; // Success. + } +@@ -57,18 +79,18 @@ inline bool BitMap::par_set_bit(idx_t bit) { + } while (true); + } + +-inline bool BitMap::par_clear_bit(idx_t bit) { ++inline bool BitMap::par_clear_bit(idx_t bit, atomic_memory_order memory_order) { + verify_index(bit); + volatile bm_word_t* const addr = word_addr(bit); + const bm_word_t mask = ~bit_mask(bit); +- bm_word_t old_val = *addr; ++ bm_word_t old_val = load_word_ordered(addr, memory_order); + + do { + const bm_word_t new_val = old_val & mask; + if (new_val == old_val) { + return false; // Someone else beat us to it. + } +- const bm_word_t cur_val = Atomic::cmpxchg(new_val, addr, old_val); ++ const bm_word_t cur_val = Atomic::cmpxchg(new_val, addr, old_val, memory_order); + if (cur_val == old_val) { + return true; // Success. + } +-- +2.12.3 + diff --git a/8233506-ZGC-the-load-for-Reference.get-can-be-conver.patch b/8233506-ZGC-the-load-for-Reference.get-can-be-conver.patch new file mode 100644 index 0000000..9c07cea --- /dev/null +++ b/8233506-ZGC-the-load-for-Reference.get-can-be-conver.patch @@ -0,0 +1,472 @@ +From aa824cddc917b1fcac41a0efe5e8c794f2d5cff9 Mon Sep 17 00:00:00 2001 +Date: Thu, 26 Mar 2020 16:17:45 +0000 +Subject: [PATCH] 8233506:ZGC: the load for Reference.get() can be converted to + a load for strong refs Summary: : LLT: JDK8233506 + Bug url: https://bugs.openjdk.java.net/browse/JDK-8233506 + +--- + src/hotspot/share/gc/shared/c2/barrierSetC2.cpp | 73 +++++++++++++++---------- + src/hotspot/share/gc/shared/c2/barrierSetC2.hpp | 7 ++- + src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp | 42 +++++--------- + src/hotspot/share/opto/graphKit.cpp | 9 +-- + src/hotspot/share/opto/graphKit.hpp | 10 ++-- + src/hotspot/share/opto/memnode.cpp | 9 ++- + src/hotspot/share/opto/memnode.hpp | 7 ++- + 7 files changed, 85 insertions(+), 72 deletions(-) + +diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp +index 545275644..48fe04b08 100644 +--- a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp ++++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp +@@ -115,10 +115,13 @@ Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) con + + Node* load; + if (in_native) { +- load = kit->make_load(control, adr, val_type, access.type(), mo); ++ load = kit->make_load(control, adr, val_type, access.type(), mo, dep, ++ requires_atomic_access, unaligned, ++ mismatched, unsafe, access.barrier_data()); + } else { + load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo, +- dep, requires_atomic_access, unaligned, mismatched, unsafe); ++ dep, requires_atomic_access, unaligned, mismatched, unsafe, ++ access.barrier_data()); + } + + access.set_raw_access(load); +@@ -348,28 +351,28 @@ Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* + if (adr->bottom_type()->is_ptr_to_narrowoop()) { + Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); + Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); +- load_store = kit->gvn().transform(new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo)); ++ load_store = new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo); + } else + #endif + { +- load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo)); ++ load_store = new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo); + } + } else { + switch (access.type()) { + case T_BYTE: { +- load_store = kit->gvn().transform(new CompareAndExchangeBNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); ++ load_store = new CompareAndExchangeBNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); + break; + } + case T_SHORT: { +- load_store = kit->gvn().transform(new CompareAndExchangeSNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); ++ load_store = new CompareAndExchangeSNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); + break; + } + case T_INT: { +- load_store = kit->gvn().transform(new CompareAndExchangeINode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); ++ load_store = new CompareAndExchangeINode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); + break; + } + case T_LONG: { +- load_store = kit->gvn().transform(new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); ++ load_store = new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo); + break; + } + default: +@@ -377,6 +380,9 @@ Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* + } + } + ++ load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); ++ load_store = kit->gvn().transform(load_store); ++ + access.set_raw_access(load_store); + pin_atomic_op(access); + +@@ -405,50 +411,50 @@ Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node + Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); + Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); + if (is_weak_cas) { +- load_store = kit->gvn().transform(new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); ++ load_store = new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo); + } else { +- load_store = kit->gvn().transform(new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); ++ load_store = new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo); + } + } else + #endif + { + if (is_weak_cas) { +- load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); ++ load_store = new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo); + } else { +- load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); ++ load_store = new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo); + } + } + } else { + switch(access.type()) { + case T_BYTE: { + if (is_weak_cas) { +- load_store = kit->gvn().transform(new WeakCompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo)); ++ load_store = new WeakCompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo); + } else { +- load_store = kit->gvn().transform(new CompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo)); ++ load_store = new CompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo); + } + break; + } + case T_SHORT: { + if (is_weak_cas) { +- load_store = kit->gvn().transform(new WeakCompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo)); ++ load_store = new WeakCompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo); + } else { +- load_store = kit->gvn().transform(new CompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo)); ++ load_store = new CompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo); + } + break; + } + case T_INT: { + if (is_weak_cas) { +- load_store = kit->gvn().transform(new WeakCompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo)); ++ load_store = new WeakCompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo); + } else { +- load_store = kit->gvn().transform(new CompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo)); ++ load_store = new CompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo); + } + break; + } + case T_LONG: { + if (is_weak_cas) { +- load_store = kit->gvn().transform(new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo)); ++ load_store = new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo); + } else { +- load_store = kit->gvn().transform(new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo)); ++ load_store = new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo); + } + break; + } +@@ -457,6 +463,9 @@ Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node + } + } + ++ load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); ++ load_store = kit->gvn().transform(load_store); ++ + access.set_raw_access(load_store); + pin_atomic_op(access); + +@@ -478,27 +487,30 @@ Node* BarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_va + } else + #endif + { +- load_store = kit->gvn().transform(new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr())); ++ load_store = new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr()); + } + } else { + switch (access.type()) { + case T_BYTE: +- load_store = kit->gvn().transform(new GetAndSetBNode(kit->control(), mem, adr, new_val, adr_type)); ++ load_store = new GetAndSetBNode(kit->control(), mem, adr, new_val, adr_type); + break; + case T_SHORT: +- load_store = kit->gvn().transform(new GetAndSetSNode(kit->control(), mem, adr, new_val, adr_type)); ++ load_store = new GetAndSetSNode(kit->control(), mem, adr, new_val, adr_type); + break; + case T_INT: +- load_store = kit->gvn().transform(new GetAndSetINode(kit->control(), mem, adr, new_val, adr_type)); ++ load_store = new GetAndSetINode(kit->control(), mem, adr, new_val, adr_type); + break; + case T_LONG: +- load_store = kit->gvn().transform(new GetAndSetLNode(kit->control(), mem, adr, new_val, adr_type)); ++ load_store = new GetAndSetLNode(kit->control(), mem, adr, new_val, adr_type); + break; + default: + ShouldNotReachHere(); + } + } + ++ load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); ++ load_store = kit->gvn().transform(load_store); ++ + access.set_raw_access(load_store); + pin_atomic_op(access); + +@@ -520,21 +532,24 @@ Node* BarrierSetC2::atomic_add_at_resolved(C2AtomicAccess& access, Node* new_val + + switch(access.type()) { + case T_BYTE: +- load_store = kit->gvn().transform(new GetAndAddBNode(kit->control(), mem, adr, new_val, adr_type)); ++ load_store = new GetAndAddBNode(kit->control(), mem, adr, new_val, adr_type); + break; + case T_SHORT: +- load_store = kit->gvn().transform(new GetAndAddSNode(kit->control(), mem, adr, new_val, adr_type)); ++ load_store = new GetAndAddSNode(kit->control(), mem, adr, new_val, adr_type); + break; + case T_INT: +- load_store = kit->gvn().transform(new GetAndAddINode(kit->control(), mem, adr, new_val, adr_type)); ++ load_store = new GetAndAddINode(kit->control(), mem, adr, new_val, adr_type); + break; + case T_LONG: +- load_store = kit->gvn().transform(new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type)); ++ load_store = new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type); + break; + default: + ShouldNotReachHere(); + } + ++ load_store->as_LoadStore()->set_barrier_data(access.barrier_data()); ++ load_store = kit->gvn().transform(load_store); ++ + access.set_raw_access(load_store); + pin_atomic_op(access); + +diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp +index 487988bd8..8b4be7d11 100644 +--- a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp ++++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp +@@ -96,6 +96,7 @@ protected: + Node* _base; + C2AccessValuePtr& _addr; + Node* _raw_access; ++ uint8_t _barrier_data; + + void fixup_decorators(); + void* barrier_set_state() const; +@@ -108,7 +109,8 @@ public: + _type(type), + _base(base), + _addr(addr), +- _raw_access(NULL) ++ _raw_access(NULL), ++ _barrier_data(0) + { + fixup_decorators(); + } +@@ -122,6 +124,9 @@ public: + bool is_raw() const { return (_decorators & AS_RAW) != 0; } + Node* raw_access() const { return _raw_access; } + ++ uint8_t barrier_data() const { return _barrier_data; } ++ void set_barrier_data(uint8_t data) { _barrier_data = data; } ++ + void set_raw_access(Node* raw_access) { _raw_access = raw_access; } + virtual void set_memory() {} // no-op for normal accesses, but not for atomic accesses. + +diff --git a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp +index a12973464..e178761a0 100644 +--- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp ++++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp +@@ -174,48 +174,36 @@ int ZBarrierSetC2::estimate_stub_size() const { + return size; + } + +-static bool barrier_needed(C2Access access) { +- return ZBarrierSet::barrier_needed(access.decorators(), access.type()); +-} +- +-Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { +- Node* result = BarrierSetC2::load_at_resolved(access, val_type); +- if (barrier_needed(access) && access.raw_access()->is_Mem()) { +- if ((access.decorators() & ON_WEAK_OOP_REF) != 0) { +- access.raw_access()->as_Load()->set_barrier_data(ZLoadBarrierWeak); ++static void set_barrier_data(C2Access& access) { ++ if (ZBarrierSet::barrier_needed(access.decorators(), access.type())) { ++ if (access.decorators() & ON_WEAK_OOP_REF) { ++ access.set_barrier_data(ZLoadBarrierWeak); + } else { +- access.raw_access()->as_Load()->set_barrier_data(ZLoadBarrierStrong); ++ access.set_barrier_data(ZLoadBarrierStrong); + } + } ++} + +- return result; ++Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { ++ set_barrier_data(access); ++ return BarrierSetC2::load_at_resolved(access, val_type); + } + + Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val, + Node* new_val, const Type* val_type) const { +- Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type); +- if (barrier_needed(access)) { +- access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong); +- } +- return result; ++ set_barrier_data(access); ++ return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type); + } + + Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val, + Node* new_val, const Type* value_type) const { +- Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); +- if (barrier_needed(access)) { +- access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong); +- } +- return result; +- ++ set_barrier_data(access); ++ return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); + } + + Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const { +- Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type); +- if (barrier_needed(access)) { +- access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong); +- } +- return result; ++ set_barrier_data(access); ++ return BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type); + } + + bool ZBarrierSetC2::array_copy_requires_gc_barriers(BasicType type) const { +diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp +index 7bf2f6cfb..a1547b42f 100644 +--- a/src/hotspot/share/opto/graphKit.cpp ++++ b/src/hotspot/share/opto/graphKit.cpp +@@ -1493,18 +1493,19 @@ Node* GraphKit::make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, + bool require_atomic_access, + bool unaligned, + bool mismatched, +- bool unsafe) { ++ bool unsafe, ++ uint8_t barrier_data) { + assert(adr_idx != Compile::AliasIdxTop, "use other make_load factory" ); + const TypePtr* adr_type = NULL; // debug-mode-only argument + debug_only(adr_type = C->get_adr_type(adr_idx)); + Node* mem = memory(adr_idx); + Node* ld; + if (require_atomic_access && bt == T_LONG) { +- ld = LoadLNode::make_atomic(ctl, mem, adr, adr_type, t, mo, control_dependency, unaligned, mismatched, unsafe); ++ ld = LoadLNode::make_atomic(ctl, mem, adr, adr_type, t, mo, control_dependency, unaligned, mismatched, unsafe, barrier_data); + } else if (require_atomic_access && bt == T_DOUBLE) { +- ld = LoadDNode::make_atomic(ctl, mem, adr, adr_type, t, mo, control_dependency, unaligned, mismatched, unsafe); ++ ld = LoadDNode::make_atomic(ctl, mem, adr, adr_type, t, mo, control_dependency, unaligned, mismatched, unsafe, barrier_data); + } else { +- ld = LoadNode::make(_gvn, ctl, mem, adr, adr_type, t, bt, mo, control_dependency, unaligned, mismatched, unsafe); ++ ld = LoadNode::make(_gvn, ctl, mem, adr, adr_type, t, bt, mo, control_dependency, unaligned, mismatched, unsafe, barrier_data); + } + ld = _gvn.transform(ld); + if (((bt == T_OBJECT) && C->do_escape_analysis()) || C->eliminate_boxing()) { +diff --git a/src/hotspot/share/opto/graphKit.hpp b/src/hotspot/share/opto/graphKit.hpp +index 07c20bbd5..df5d18ccc 100644 +--- a/src/hotspot/share/opto/graphKit.hpp ++++ b/src/hotspot/share/opto/graphKit.hpp +@@ -518,27 +518,27 @@ class GraphKit : public Phase { + Node* make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, + MemNode::MemOrd mo, LoadNode::ControlDependency control_dependency = LoadNode::DependsOnlyOnTest, + bool require_atomic_access = false, bool unaligned = false, +- bool mismatched = false, bool unsafe = false) { ++ bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0) { + // This version computes alias_index from bottom_type + return make_load(ctl, adr, t, bt, adr->bottom_type()->is_ptr(), + mo, control_dependency, require_atomic_access, +- unaligned, mismatched, unsafe); ++ unaligned, mismatched, unsafe, barrier_data); + } + Node* make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, const TypePtr* adr_type, + MemNode::MemOrd mo, LoadNode::ControlDependency control_dependency = LoadNode::DependsOnlyOnTest, + bool require_atomic_access = false, bool unaligned = false, +- bool mismatched = false, bool unsafe = false) { ++ bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0) { + // This version computes alias_index from an address type + assert(adr_type != NULL, "use other make_load factory"); + return make_load(ctl, adr, t, bt, C->get_alias_index(adr_type), + mo, control_dependency, require_atomic_access, +- unaligned, mismatched, unsafe); ++ unaligned, mismatched, unsafe, barrier_data); + } + // This is the base version which is given an alias index. + Node* make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, int adr_idx, + MemNode::MemOrd mo, LoadNode::ControlDependency control_dependency = LoadNode::DependsOnlyOnTest, + bool require_atomic_access = false, bool unaligned = false, +- bool mismatched = false, bool unsafe = false); ++ bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0); + + // Create & transform a StoreNode and store the effect into the + // parser's memory state. +diff --git a/src/hotspot/share/opto/memnode.cpp b/src/hotspot/share/opto/memnode.cpp +index ee0f09e11..ff0a5726c 100644 +--- a/src/hotspot/share/opto/memnode.cpp ++++ b/src/hotspot/share/opto/memnode.cpp +@@ -808,7 +808,7 @@ bool LoadNode::is_immutable_value(Node* adr) { + //----------------------------LoadNode::make----------------------------------- + // Polymorphic factory method: + Node *LoadNode::make(PhaseGVN& gvn, Node *ctl, Node *mem, Node *adr, const TypePtr* adr_type, const Type *rt, BasicType bt, MemOrd mo, +- ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe) { ++ ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) { + Compile* C = gvn.C; + + // sanity check the alias category against the created node type +@@ -859,6 +859,7 @@ Node *LoadNode::make(PhaseGVN& gvn, Node *ctl, Node *mem, Node *adr, const TypeP + if (unsafe) { + load->set_unsafe_access(); + } ++ load->set_barrier_data(barrier_data); + if (load->Opcode() == Op_LoadN) { + Node* ld = gvn.transform(load); + return new DecodeNNode(ld, ld->bottom_type()->make_ptr()); +@@ -868,7 +869,7 @@ Node *LoadNode::make(PhaseGVN& gvn, Node *ctl, Node *mem, Node *adr, const TypeP + } + + LoadLNode* LoadLNode::make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, MemOrd mo, +- ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe) { ++ ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) { + bool require_atomic = true; + LoadLNode* load = new LoadLNode(ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic); + if (unaligned) { +@@ -880,11 +881,12 @@ LoadLNode* LoadLNode::make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr + if (unsafe) { + load->set_unsafe_access(); + } ++ load->set_barrier_data(barrier_data); + return load; + } + + LoadDNode* LoadDNode::make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, MemOrd mo, +- ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe) { ++ ControlDependency control_dependency, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) { + bool require_atomic = true; + LoadDNode* load = new LoadDNode(ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic); + if (unaligned) { +@@ -896,6 +898,7 @@ LoadDNode* LoadDNode::make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr + if (unsafe) { + load->set_unsafe_access(); + } ++ load->set_barrier_data(barrier_data); + return load; + } + +diff --git a/src/hotspot/share/opto/memnode.hpp b/src/hotspot/share/opto/memnode.hpp +index 7468abdbc..14a4a67c6 100644 +--- a/src/hotspot/share/opto/memnode.hpp ++++ b/src/hotspot/share/opto/memnode.hpp +@@ -227,7 +227,8 @@ public: + static Node* make(PhaseGVN& gvn, Node *c, Node *mem, Node *adr, + const TypePtr* at, const Type *rt, BasicType bt, + MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest, +- bool unaligned = false, bool mismatched = false, bool unsafe = false); ++ bool unaligned = false, bool mismatched = false, bool unsafe = false, ++ uint8_t barrier_data = 0); + + virtual uint hash() const; // Check the type + +@@ -408,7 +409,7 @@ public: + bool require_atomic_access() const { return _require_atomic_access; } + static LoadLNode* make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, + const Type* rt, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest, +- bool unaligned = false, bool mismatched = false, bool unsafe = false); ++ bool unaligned = false, bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0); + #ifndef PRODUCT + virtual void dump_spec(outputStream *st) const { + LoadNode::dump_spec(st); +@@ -460,7 +461,7 @@ public: + bool require_atomic_access() const { return _require_atomic_access; } + static LoadDNode* make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, + const Type* rt, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest, +- bool unaligned = false, bool mismatched = false, bool unsafe = false); ++ bool unaligned = false, bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0); + #ifndef PRODUCT + virtual void dump_spec(outputStream *st) const { + LoadNode::dump_spec(st); +-- +2.12.3 + diff --git a/Add-ability-to-configure-third-port-for-remote-JMX.patch b/Add-ability-to-configure-third-port-for-remote-JMX.patch new file mode 100644 index 0000000..47c99d4 --- /dev/null +++ b/Add-ability-to-configure-third-port-for-remote-JMX.patch @@ -0,0 +1,67 @@ +From b2ff55a515ffbead944a6cc64c77af0fb5bb9116 Mon Sep 17 00:00:00 2001 +Date: Mon, 6 Jan 2020 16:25:39 +0000 +Subject: [PATCH] Add ability to configure third port for remote JMX + +Summary: : +LLT: NA +Bug url: NA +--- + .../jdk/internal/agent/AgentConfigurationError.java | 2 ++ + .../sun/management/jmxremote/ConnectorBootstrap.java | 19 ++++++++++++++++++- + 2 files changed, 20 insertions(+), 1 deletion(-) + +diff --git a/src/jdk.management.agent/share/classes/jdk/internal/agent/AgentConfigurationError.java b/src/jdk.management.agent/share/classes/jdk/internal/agent/AgentConfigurationError.java +index fb00b9f79..5f5e1672f 100644 +--- a/src/jdk.management.agent/share/classes/jdk/internal/agent/AgentConfigurationError.java ++++ b/src/jdk.management.agent/share/classes/jdk/internal/agent/AgentConfigurationError.java +@@ -55,6 +55,8 @@ public class AgentConfigurationError extends Error { + "agent.err.invalid.jmxremote.port"; + public static final String INVALID_JMXREMOTE_RMI_PORT = + "agent.err.invalid.jmxremote.rmi.port"; ++ public static final String INVALID_JMXLOCAL_PORT = ++ "agent.err.invalid.jmxlocal.port"; + public static final String PASSWORD_FILE_NOT_SET = + "agent.err.password.file.notset"; + public static final String PASSWORD_FILE_NOT_READABLE = +diff --git a/src/jdk.management.agent/share/classes/sun/management/jmxremote/ConnectorBootstrap.java b/src/jdk.management.agent/share/classes/sun/management/jmxremote/ConnectorBootstrap.java +index 8d2031f88..12848cbeb 100644 +--- a/src/jdk.management.agent/share/classes/sun/management/jmxremote/ConnectorBootstrap.java ++++ b/src/jdk.management.agent/share/classes/sun/management/jmxremote/ConnectorBootstrap.java +@@ -116,6 +116,8 @@ public final class ConnectorBootstrap { + "com.sun.management.jmxremote.host"; + public static final String RMI_PORT = + "com.sun.management.jmxremote.rmi.port"; ++ public static final String LOCAL_PORT = ++ "com.sun.management.jmxlocal.port"; + public static final String CONFIG_FILE_NAME = + "com.sun.management.config.file"; + public static final String USE_LOCAL_ONLY = +@@ -539,9 +541,24 @@ public final class ConnectorBootstrap { + localhost = "127.0.0.1"; + } + ++ // User can specify a port to be used to start Local Connector Server, ++ // if port is not specified random one will be allocated. ++ int localPort = 0; ++ String localPortStr = System.getProperty(PropertyNames.LOCAL_PORT); ++ try { ++ if (localPortStr != null) { ++ localPort = Integer.parseInt(localPortStr); ++ } ++ } catch (NumberFormatException x) { ++ throw new AgentConfigurationError(INVALID_JMXLOCAL_PORT, x, localPortStr); ++ } ++ if (localPort < 0) { ++ throw new AgentConfigurationError(INVALID_JMXLOCAL_PORT, localPortStr); ++ } ++ + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + try { +- JMXServiceURL url = new JMXServiceURL("rmi", localhost, 0); ++ JMXServiceURL url = new JMXServiceURL("rmi", localhost, localPort); + // Do we accept connections from local interfaces only? + Properties props = Agent.getManagementProperties(); + if (props == null) { +-- +2.12.3 + diff --git a/Add-loadload-membar-to-avoid-loading-a-incorrect-offset.patch b/Add-loadload-membar-to-avoid-loading-a-incorrect-offset.patch new file mode 100644 index 0000000..52bce47 --- /dev/null +++ b/Add-loadload-membar-to-avoid-loading-a-incorrect-offset.patch @@ -0,0 +1,37 @@ +From 954a4ad3a2a8bb54fb64a524b53f9c87c54cb7a1 Mon Sep 17 00:00:00 2001 +Date: Wed, 26 Feb 2020 09:47:33 +0800 +Subject: [PATCH] aarch64: add_loadload_membar_to_avoid_loading_a_incorrect_offset + +Summary: : +LLT: jtreg +Bug url: NA +--- + src/hotspot/cpu/aarch64/templateTable_aarch64.cpp | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp +index b8a9a46a8..019a4aadd 100644 +--- a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp ++++ b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp +@@ -2963,6 +2963,8 @@ void TemplateTable::fast_storefield(TosState state) + // access constant pool cache + __ get_cache_and_index_at_bcp(r2, r1, 1); + ++ __ membar(MacroAssembler::LoadLoad); ++ + // test for volatile with r3 + __ ldrw(r3, Address(r2, in_bytes(base + + ConstantPoolCacheEntry::flags_offset()))); +@@ -3055,6 +3057,9 @@ void TemplateTable::fast_accessfield(TosState state) + + // access constant pool cache + __ get_cache_and_index_at_bcp(r2, r1, 1); ++ ++ __ membar(MacroAssembler::LoadLoad); ++ + __ ldr(r1, Address(r2, in_bytes(ConstantPoolCache::base_offset() + + ConstantPoolCacheEntry::f2_offset()))); + __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() + +-- +2.12.3 + diff --git a/ZGC-Redesign-C2-load-barrier-to-expand-on-th.patch b/ZGC-Redesign-C2-load-barrier-to-expand-on-th.patch new file mode 100644 index 0000000..8f3ab21 --- /dev/null +++ b/ZGC-Redesign-C2-load-barrier-to-expand-on-th.patch @@ -0,0 +1,5882 @@ +From b61b8df19a63354db26efb582e84bc276c076e3a Mon Sep 17 00:00:00 2001 +Date: Wed, 5 Feb 2020 17:26:06 +0800 +Subject: [PATCH] 8230565: ZGC: Redesign C2 load barrier to expand on the + MachNode level + +Summary: : +LLT: jtreg +Bug url: https://bugs.openjdk.java.net/browse/JDK-8230565 +--- + src/hotspot/cpu/aarch64/aarch64.ad | 362 +++-- + .../aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp | 246 +-- + .../aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp | 26 +- + src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp | 59 + + src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp | 6 + + src/hotspot/cpu/aarch64/register_aarch64.hpp | 5 + + .../templateInterpreterGenerator_aarch64.cpp | 4 +- + .../cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp | 403 +++-- + .../cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp | 30 +- + src/hotspot/cpu/x86/x86.ad | 136 +- + src/hotspot/cpu/x86/x86_64.ad | 492 ++---- + src/hotspot/share/adlc/formssel.cpp | 8 - + src/hotspot/share/compiler/compilerDirectives.hpp | 3 +- + src/hotspot/share/gc/shared/c2/barrierSetC2.hpp | 8 +- + src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp | 1646 ++++---------------- + src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp | 181 +-- + src/hotspot/share/gc/z/zBarrierSetAssembler.hpp | 5 +- + src/hotspot/share/opto/classes.cpp | 3 - + src/hotspot/share/opto/classes.hpp | 11 - + src/hotspot/share/opto/compile.cpp | 52 +- + src/hotspot/share/opto/compile.hpp | 25 +- + src/hotspot/share/opto/escape.cpp | 15 - + src/hotspot/share/opto/lcm.cpp | 1 - + src/hotspot/share/opto/loopnode.cpp | 13 - + src/hotspot/share/opto/loopopts.cpp | 3 - + src/hotspot/share/opto/machnode.hpp | 9 +- + src/hotspot/share/opto/matcher.cpp | 45 +- + src/hotspot/share/opto/memnode.cpp | 5 +- + src/hotspot/share/opto/memnode.hpp | 46 +- + src/hotspot/share/opto/node.cpp | 7 - + src/hotspot/share/opto/node.hpp | 6 - + src/hotspot/share/opto/output.cpp | 424 ++--- + src/hotspot/share/opto/output.hpp | 5 +- + src/hotspot/share/opto/phaseX.cpp | 8 +- + src/hotspot/share/opto/vectornode.cpp | 1 - + 35 files changed, 1529 insertions(+), 2770 deletions(-) + +diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad +index 3b898f548..93dd78e51 100644 +--- a/src/hotspot/cpu/aarch64/aarch64.ad ++++ b/src/hotspot/cpu/aarch64/aarch64.ad +@@ -1131,6 +1131,7 @@ definitions %{ + source_hpp %{ + + #include "gc/z/c2/zBarrierSetC2.hpp" ++#include "gc/z/zThreadLocalData.hpp" + + %} + +@@ -2497,17 +2498,7 @@ void Compile::reshape_address(AddPNode* addp) { + __ INSN(REG, as_Register(BASE)); \ + } + +-typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr); +-typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr); +-typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt, +- MacroAssembler::SIMD_RegVariant T, const Address &adr); +- +- // Used for all non-volatile memory accesses. The use of +- // $mem->opcode() to discover whether this pattern uses sign-extended +- // offsets is something of a kludge. +- static void loadStore(MacroAssembler masm, mem_insn insn, +- Register reg, int opcode, +- Register base, int index, int size, int disp) ++static Address mem2address(int opcode, Register base, int index, int size, int disp) + { + Address::extend scale; + +@@ -2526,16 +2517,33 @@ typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt, + } + + if (index == -1) { +- (masm.*insn)(reg, Address(base, disp)); ++ return Address(base, disp); + } else { + assert(disp == 0, "unsupported address mode: disp = %d", disp); +- (masm.*insn)(reg, Address(base, as_Register(index), scale)); ++ return Address(base, as_Register(index), scale); + } + } + ++typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr); ++typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr); ++typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr); ++typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt, ++ MacroAssembler::SIMD_RegVariant T, const Address &adr); ++ ++ // Used for all non-volatile memory accesses. The use of ++ // $mem->opcode() to discover whether this pattern uses sign-extended ++ // offsets is something of a kludge. ++ static void loadStore(MacroAssembler masm, mem_insn insn, ++ Register reg, int opcode, ++ Register base, int index, int size, int disp) ++ { ++ Address addr = mem2address(opcode, base, index, size, disp); ++ (masm.*insn)(reg, addr); ++ } ++ + static void loadStore(MacroAssembler masm, mem_float_insn insn, +- FloatRegister reg, int opcode, +- Register base, int index, int size, int disp) ++ FloatRegister reg, int opcode, ++ Register base, int index, int size, int disp) + { + Address::extend scale; + +@@ -2557,8 +2565,8 @@ typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt, + } + + static void loadStore(MacroAssembler masm, mem_vector_insn insn, +- FloatRegister reg, MacroAssembler::SIMD_RegVariant T, +- int opcode, Register base, int index, int size, int disp) ++ FloatRegister reg, MacroAssembler::SIMD_RegVariant T, ++ int opcode, Register base, int index, int size, int disp) + { + if (index == -1) { + (masm.*insn)(reg, T, Address(base, disp)); +@@ -3793,7 +3801,7 @@ frame %{ + static const int hi[Op_RegL + 1] = { // enum name + 0, // Op_Node + 0, // Op_Set +- OptoReg::Bad, // Op_RegN ++ OptoReg::Bad, // Op_RegN + OptoReg::Bad, // Op_RegI + R0_H_num, // Op_RegP + OptoReg::Bad, // Op_RegF +@@ -6923,7 +6931,7 @@ instruct loadRange(iRegINoSp dst, memory mem) + instruct loadP(iRegPNoSp dst, memory mem) + %{ + match(Set dst (LoadP mem)); +- predicate(!needs_acquiring_load(n)); ++ predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0)); + + ins_cost(4 * INSN_COST); + format %{ "ldr $dst, $mem\t# ptr" %} +@@ -7616,6 +7624,7 @@ instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem) + instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem) + %{ + match(Set dst (LoadP mem)); ++ predicate(n->as_Load()->barrier_data() == 0); + + ins_cost(VOLATILE_REF_COST); + format %{ "ldar $dst, $mem\t# ptr" %} +@@ -8511,6 +8520,7 @@ instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoS + instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ + + match(Set res (CompareAndSwapP mem (Binary oldval newval))); ++ predicate(n->as_LoadStore()->barrier_data() == 0); + ins_cost(2 * VOLATILE_REF_COST); + + effect(KILL cr); +@@ -8624,7 +8634,7 @@ instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegL + + instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ + +- predicate(needs_acquiring_load_exclusive(n)); ++ predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0)); + match(Set res (CompareAndSwapP mem (Binary oldval newval))); + ins_cost(VOLATILE_REF_COST); + +@@ -8755,6 +8765,7 @@ instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN ne + %} + + instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ ++ predicate(n->as_LoadStore()->barrier_data() == 0); + match(Set res (CompareAndExchangeP mem (Binary oldval newval))); + ins_cost(2 * VOLATILE_REF_COST); + effect(TEMP_DEF res, KILL cr); +@@ -8854,7 +8865,7 @@ instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN + %} + + instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ +- predicate(needs_acquiring_load_exclusive(n)); ++ predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0)); + match(Set res (CompareAndExchangeP mem (Binary oldval newval))); + ins_cost(VOLATILE_REF_COST); + effect(TEMP_DEF res, KILL cr); +@@ -8955,6 +8966,7 @@ instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN ne + %} + + instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ ++ predicate(n->as_LoadStore()->barrier_data() == 0); + match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); + ins_cost(2 * VOLATILE_REF_COST); + effect(KILL cr); +@@ -9062,8 +9074,8 @@ instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN + %} + + instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ +- predicate(needs_acquiring_load_exclusive(n)); + match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); ++ predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0)); + ins_cost(VOLATILE_REF_COST); + effect(KILL cr); + format %{ +@@ -9113,6 +9125,7 @@ instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{ + %} + + instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{ ++ predicate(n->as_LoadStore()->barrier_data() == 0); + match(Set prev (GetAndSetP mem newv)); + ins_cost(2 * VOLATILE_REF_COST); + format %{ "atomic_xchg $prev, $newv, [$mem]" %} +@@ -9156,7 +9169,7 @@ instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{ + %} + + instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{ +- predicate(needs_acquiring_load_exclusive(n)); ++ predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0)); + match(Set prev (GetAndSetP mem newv)); + ins_cost(VOLATILE_REF_COST); + format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %} +@@ -17476,145 +17489,238 @@ instruct vsrl2L_imm(vecX dst, vecX src, immI shift) %{ + + source %{ + +-#include "gc/z/zBarrierSetAssembler.hpp" ++static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) { ++ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, weak); ++ __ ldr(tmp, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); ++ __ andr(tmp, tmp, ref); ++ __ cbnz(tmp, *stub->entry()); ++ __ bind(*stub->continuation()); ++} + +-static void z_load_barrier_slow_reg(MacroAssembler& _masm, Register dst, +- Register base, int index, int scale, +- int disp, bool weak) { +- const address stub = weak ? ZBarrierSet::assembler()->load_barrier_weak_slow_stub(dst) +- : ZBarrierSet::assembler()->load_barrier_slow_stub(dst); ++static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) { ++ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, false /* weak */); ++ __ b(*stub->entry()); ++ __ bind(*stub->continuation()); ++} + +- if (index == -1) { +- if (disp != 0) { +- __ lea(dst, Address(base, disp)); +- } else { +- __ mov(dst, base); +- } +- } else { +- Register index_reg = as_Register(index); +- if (disp == 0) { +- __ lea(dst, Address(base, index_reg, Address::lsl(scale))); +- } else { +- __ lea(dst, Address(base, disp)); +- __ lea(dst, Address(dst, index_reg, Address::lsl(scale))); ++%} ++ ++// Load Pointer ++instruct zLoadP(iRegPNoSp dst, memory mem, rFlagsReg cr) ++%{ ++ match(Set dst (LoadP mem)); ++ predicate(UseZGC && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() == ZLoadBarrierStrong)); ++ effect(TEMP dst, KILL cr); ++ ++ ins_cost(4 * INSN_COST); ++ ++ format %{ "ldr $dst, $mem" %} ++ ++ ins_encode %{ ++ const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); ++ __ ldr($dst$$Register, ref_addr); ++ if (barrier_data() != ZLoadBarrierElided) { ++ z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, false /* weak */); + } +- } ++ %} + +- __ far_call(RuntimeAddress(stub)); +-} ++ ins_pipe(iload_reg_mem); ++%} ++ ++// Load Weak Pointer ++instruct zLoadWeakP(iRegPNoSp dst, memory mem, rFlagsReg cr) ++%{ ++ match(Set dst (LoadP mem)); ++ predicate(UseZGC && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() == ZLoadBarrierWeak)); ++ effect(TEMP dst, KILL cr); ++ ++ ins_cost(4 * INSN_COST); + ++ format %{ "ldr $dst, $mem" %} ++ ++ ins_encode %{ ++ const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); ++ __ ldr($dst$$Register, ref_addr); ++ z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, true /* weak */); ++ %} ++ ++ ins_pipe(iload_reg_mem); + %} + +-// +-// Execute ZGC load barrier (strong) slow path +-// +-instruct loadBarrierSlowReg(iRegP dst, memory mem, rFlagsReg cr, +- vRegD_V0 v0, vRegD_V1 v1, vRegD_V2 v2, vRegD_V3 v3, vRegD_V4 v4, +- vRegD_V5 v5, vRegD_V6 v6, vRegD_V7 v7, vRegD_V8 v8, vRegD_V9 v9, +- vRegD_V10 v10, vRegD_V11 v11, vRegD_V12 v12, vRegD_V13 v13, vRegD_V14 v14, +- vRegD_V15 v15, vRegD_V16 v16, vRegD_V17 v17, vRegD_V18 v18, vRegD_V19 v19, +- vRegD_V20 v20, vRegD_V21 v21, vRegD_V22 v22, vRegD_V23 v23, vRegD_V24 v24, +- vRegD_V25 v25, vRegD_V26 v26, vRegD_V27 v27, vRegD_V28 v28, vRegD_V29 v29, +- vRegD_V30 v30, vRegD_V31 v31) %{ +- match(Set dst (LoadBarrierSlowReg mem)); +- predicate(!n->as_LoadBarrierSlowReg()->is_weak()); +- +- effect(DEF dst, KILL cr, +- KILL v0, KILL v1, KILL v2, KILL v3, KILL v4, KILL v5, KILL v6, KILL v7, +- KILL v8, KILL v9, KILL v10, KILL v11, KILL v12, KILL v13, KILL v14, +- KILL v15, KILL v16, KILL v17, KILL v18, KILL v19, KILL v20, KILL v21, +- KILL v22, KILL v23, KILL v24, KILL v25, KILL v26, KILL v27, KILL v28, +- KILL v29, KILL v30, KILL v31); +- +- format %{"LoadBarrierSlowReg $dst, $mem" %} +- ins_encode %{ +- z_load_barrier_slow_reg(_masm, $dst$$Register, $mem$$base$$Register, +- $mem$$index, $mem$$scale, $mem$$disp, false); ++// Load Pointer Volatile ++instruct zLoadPVolatile(iRegPNoSp dst, indirect mem /* sync_memory */, rFlagsReg cr) ++%{ ++ match(Set dst (LoadP mem)); ++ predicate(UseZGC && needs_acquiring_load(n) && n->as_Load()->barrier_data() == ZLoadBarrierStrong); ++ effect(TEMP dst, KILL cr); ++ ++ ins_cost(VOLATILE_REF_COST); ++ ++ format %{ "ldar $dst, $mem\t" %} ++ ++ ins_encode %{ ++ __ ldar($dst$$Register, $mem$$Register); ++ if (barrier_data() != ZLoadBarrierElided) { ++ z_load_barrier(_masm, this, Address($mem$$Register), $dst$$Register, rscratch2 /* tmp */, false /* weak */); ++ } + %} +- ins_pipe(pipe_slow); ++ ++ ins_pipe(pipe_serial); + %} + +-// +-// Execute ZGC load barrier (weak) slow path +-// +-instruct loadBarrierWeakSlowReg(iRegP dst, memory mem, rFlagsReg cr, +- vRegD_V0 v0, vRegD_V1 v1, vRegD_V2 v2, vRegD_V3 v3, vRegD_V4 v4, +- vRegD_V5 v5, vRegD_V6 v6, vRegD_V7 v7, vRegD_V8 v8, vRegD_V9 v9, +- vRegD_V10 v10, vRegD_V11 v11, vRegD_V12 v12, vRegD_V13 v13, vRegD_V14 v14, +- vRegD_V15 v15, vRegD_V16 v16, vRegD_V17 v17, vRegD_V18 v18, vRegD_V19 v19, +- vRegD_V20 v20, vRegD_V21 v21, vRegD_V22 v22, vRegD_V23 v23, vRegD_V24 v24, +- vRegD_V25 v25, vRegD_V26 v26, vRegD_V27 v27, vRegD_V28 v28, vRegD_V29 v29, +- vRegD_V30 v30, vRegD_V31 v31) %{ +- match(Set dst (LoadBarrierSlowReg mem)); +- predicate(n->as_LoadBarrierSlowReg()->is_weak()); +- +- effect(DEF dst, KILL cr, +- KILL v0, KILL v1, KILL v2, KILL v3, KILL v4, KILL v5, KILL v6, KILL v7, +- KILL v8, KILL v9, KILL v10, KILL v11, KILL v12, KILL v13, KILL v14, +- KILL v15, KILL v16, KILL v17, KILL v18, KILL v19, KILL v20, KILL v21, +- KILL v22, KILL v23, KILL v24, KILL v25, KILL v26, KILL v27, KILL v28, +- KILL v29, KILL v30, KILL v31); +- +- format %{"LoadBarrierWeakSlowReg $dst, $mem" %} +- ins_encode %{ +- z_load_barrier_slow_reg(_masm, $dst$$Register, $mem$$base$$Register, +- $mem$$index, $mem$$scale, $mem$$disp, true); ++instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ ++ match(Set res (CompareAndSwapP mem (Binary oldval newval))); ++ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); ++ predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); ++ effect(KILL cr, TEMP_DEF res); ++ ++ ins_cost(2 * VOLATILE_REF_COST); ++ ++ format %{ "cmpxchg $mem, $oldval, $newval\n\t" ++ "cset $res, EQ" %} ++ ++ ins_encode %{ ++ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ false /* acquire */, true /* release */, false /* weak */, rscratch2); ++ __ cset($res$$Register, Assembler::EQ); ++ if (barrier_data() != ZLoadBarrierElided) { ++ Label good; ++ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); ++ __ andr(rscratch1, rscratch1, rscratch2); ++ __ cbz(rscratch1, good); ++ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ false /* acquire */, true /* release */, false /* weak */, rscratch2); ++ __ cset($res$$Register, Assembler::EQ); ++ __ bind(good); ++ } + %} ++ + ins_pipe(pipe_slow); + %} + +-// Specialized versions of compareAndExchangeP that adds a keepalive that is consumed +-// but doesn't affect output. ++instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ ++ match(Set res (CompareAndSwapP mem (Binary oldval newval))); ++ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); ++ predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)); ++ effect(KILL cr, TEMP_DEF res); + +-instruct z_compareAndExchangeP(iRegPNoSp res, indirect mem, +- iRegP oldval, iRegP newval, iRegP keepalive, +- rFlagsReg cr) %{ +- match(Set res (ZCompareAndExchangeP (Binary mem keepalive) (Binary oldval newval))); + ins_cost(2 * VOLATILE_REF_COST); +- effect(TEMP_DEF res, KILL cr); +- format %{ +- "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval" +- %} ++ ++ format %{ "cmpxchg $mem, $oldval, $newval\n\t" ++ "cset $res, EQ" %} ++ + ins_encode %{ +- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, +- Assembler::xword, /*acquire*/ false, /*release*/ true, +- /*weak*/ false, $res$$Register); ++ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ true /* acquire */, true /* release */, false /* weak */, rscratch2); ++ __ cset($res$$Register, Assembler::EQ); ++ if (barrier_data() != ZLoadBarrierElided) { ++ Label good; ++ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); ++ __ andr(rscratch1, rscratch1, rscratch2); ++ __ cbz(rscratch1, good); ++ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */ ); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ true /* acquire */, true /* release */, false /* weak */, rscratch2); ++ __ cset($res$$Register, Assembler::EQ); ++ __ bind(good); ++ } + %} ++ + ins_pipe(pipe_slow); + %} + +-instruct z_compareAndSwapP(iRegINoSp res, +- indirect mem, +- iRegP oldval, iRegP newval, iRegP keepalive, +- rFlagsReg cr) %{ +- +- match(Set res (ZCompareAndSwapP (Binary mem keepalive) (Binary oldval newval))); +- match(Set res (ZWeakCompareAndSwapP (Binary mem keepalive) (Binary oldval newval))); ++instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ ++ match(Set res (CompareAndExchangeP mem (Binary oldval newval))); ++ predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); ++ effect(TEMP_DEF res, KILL cr); + + ins_cost(2 * VOLATILE_REF_COST); + +- effect(KILL cr); ++ format %{ "cmpxchg $res = $mem, $oldval, $newval" %} + +- format %{ +- "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval" +- "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" +- %} +- +- ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval), +- aarch64_enc_cset_eq(res)); ++ ins_encode %{ ++ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ false /* acquire */, true /* release */, false /* weak */, $res$$Register); ++ if (barrier_data() != ZLoadBarrierElided) { ++ Label good; ++ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); ++ __ andr(rscratch1, rscratch1, $res$$Register); ++ __ cbz(rscratch1, good); ++ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ false /* acquire */, true /* release */, false /* weak */, $res$$Register); ++ __ bind(good); ++ } ++ %} + + ins_pipe(pipe_slow); + %} + ++instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ ++ match(Set res (CompareAndExchangeP mem (Binary oldval newval))); ++ predicate(UseZGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); ++ effect(TEMP_DEF res, KILL cr); ++ ++ ins_cost(2 * VOLATILE_REF_COST); ++ ++ format %{ "cmpxchg $res = $mem, $oldval, $newval" %} + +-instruct z_get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev, +- iRegP keepalive) %{ +- match(Set prev (ZGetAndSetP mem (Binary newv keepalive))); ++ ins_encode %{ ++ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ true /* acquire */, true /* release */, false /* weak */, $res$$Register); ++ if (barrier_data() != ZLoadBarrierElided) { ++ Label good; ++ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); ++ __ andr(rscratch1, rscratch1, $res$$Register); ++ __ cbz(rscratch1, good); ++ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */); ++ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, ++ true /* acquire */, true /* release */, false /* weak */, $res$$Register); ++ __ bind(good); ++ } ++ %} ++ ++ ins_pipe(pipe_slow); ++%} ++ ++instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{ ++ match(Set prev (GetAndSetP mem newv)); ++ predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); ++ effect(TEMP_DEF prev, KILL cr); + + ins_cost(2 * VOLATILE_REF_COST); ++ + format %{ "atomic_xchg $prev, $newv, [$mem]" %} ++ + ins_encode %{ +- __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base)); ++ __ atomic_xchg($prev$$Register, $newv$$Register, $mem$$Register); ++ if (barrier_data() != ZLoadBarrierElided) { ++ z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, false /* weak */); ++ } ++ %} ++ ++ ins_pipe(pipe_serial); ++%} ++ ++instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{ ++ match(Set prev (GetAndSetP mem newv)); ++ predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)); ++ effect(TEMP_DEF prev, KILL cr); ++ ++ ins_cost(VOLATILE_REF_COST); ++ ++ format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %} ++ ++ ins_encode %{ ++ __ atomic_xchgal($prev$$Register, $newv$$Register, $mem$$Register); ++ if (barrier_data() != ZLoadBarrierElided) { ++ z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, false /* weak */); ++ } + %} + ins_pipe(pipe_serial); + %} +diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp +index 8e169ace4..787c0c1af 100644 +--- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp ++++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp +@@ -24,22 +24,23 @@ + #include "precompiled.hpp" + #include "asm/macroAssembler.inline.hpp" + #include "code/codeBlob.hpp" ++#include "code/vmreg.inline.hpp" + #include "gc/z/zBarrier.inline.hpp" + #include "gc/z/zBarrierSet.hpp" + #include "gc/z/zBarrierSetAssembler.hpp" + #include "gc/z/zBarrierSetRuntime.hpp" ++#include "gc/z/zThreadLocalData.hpp" + #include "memory/resourceArea.hpp" ++#include "runtime/sharedRuntime.hpp" ++#include "utilities/macros.hpp" + #ifdef COMPILER1 + #include "c1/c1_LIRAssembler.hpp" + #include "c1/c1_MacroAssembler.hpp" + #include "gc/z/c1/zBarrierSetC1.hpp" + #endif // COMPILER1 +- +-#include "gc/z/zThreadLocalData.hpp" +- +-ZBarrierSetAssembler::ZBarrierSetAssembler() : +- _load_barrier_slow_stub(), +- _load_barrier_weak_slow_stub() {} ++#ifdef COMPILER2 ++#include "gc/z/c2/zBarrierSetC2.hpp" ++#endif // COMPILER2 + + #ifdef PRODUCT + #define BLOCK_COMMENT(str) /* nothing */ +@@ -63,27 +64,25 @@ void ZBarrierSetAssembler::load_at(MacroAssembler* masm, + return; + } + +- // rscratch1 can be passed as src or dst, so don't use it. +- RegSet savedRegs = RegSet::of(rscratch2, rheapbase); ++ assert_different_registers(rscratch1, rscratch2, src.base()); ++ assert_different_registers(rscratch1, rscratch2, dst); + +- Label done; +- assert_different_registers(rheapbase, rscratch2, dst); +- assert_different_registers(rheapbase, rscratch2, src.base()); ++ RegSet savedRegs = RegSet::range(r0, r28) - RegSet::of(dst, rscratch1, rscratch2); + +- __ push(savedRegs, sp); ++ Label done; + + // Load bad mask into scratch register. +- __ ldr(rheapbase, address_bad_mask_from_thread(rthread)); ++ __ ldr(rscratch1, address_bad_mask_from_thread(rthread)); + __ lea(rscratch2, src); + __ ldr(dst, src); + + // Test reference against bad mask. If mask bad, then we need to fix it up. +- __ tst(dst, rheapbase); ++ __ tst(dst, rscratch1); + __ br(Assembler::EQ, done); + + __ enter(); + +- __ push(RegSet::range(r0,r28) - RegSet::of(dst), sp); ++ __ push(savedRegs, sp); + + if (c_rarg0 != dst) { + __ mov(c_rarg0, dst); +@@ -91,13 +90,15 @@ void ZBarrierSetAssembler::load_at(MacroAssembler* masm, + __ mov(c_rarg1, rscratch2); + + int step = 4 * wordSize; +- __ mov(rscratch1, -step); ++ __ mov(rscratch2, -step); + __ sub(sp, sp, step); + + for (int i = 28; i >= 4; i -= 4) { + __ st1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2), +- as_FloatRegister(i+3), __ T1D, Address(__ post(sp, rscratch1))); ++ as_FloatRegister(i+3), __ T1D, Address(__ post(sp, rscratch2))); + } ++ __ st1(as_FloatRegister(0), as_FloatRegister(1), as_FloatRegister(2), ++ as_FloatRegister(3), __ T1D, Address(sp)); + + __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2); + +@@ -111,13 +112,10 @@ void ZBarrierSetAssembler::load_at(MacroAssembler* masm, + __ mov(dst, r0); + } + +- __ pop(RegSet::range(r0,r28) - RegSet::of(dst), sp); ++ __ pop(savedRegs, sp); + __ leave(); + + __ bind(done); +- +- // Restore tmps +- __ pop(savedRegs, sp); + } + + #ifdef ASSERT +@@ -209,7 +207,7 @@ void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, + + // The Address offset is too large to direct load - -784. Our range is +127, -128. + __ mov(tmp, (long int)(in_bytes(ZThreadLocalData::address_bad_mask_offset()) - +- in_bytes(JavaThread::jni_environment_offset()))); ++ in_bytes(JavaThread::jni_environment_offset()))); + // Load address bad mask + __ add(tmp, jni_env, tmp); + __ ldr(tmp, Address(tmp)); +@@ -297,111 +295,173 @@ void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* + __ prologue("zgc_load_barrier stub", false); + + // We don't use push/pop_clobbered_registers() - we need to pull out the result from r0. +- for (int i = 0; i < 32; i +=2) { +- __ stpd(as_FloatRegister(i), as_FloatRegister(i+1), Address(__ pre(sp,-16))); ++ for (int i = 0; i < 32; i += 2) { ++ __ stpd(as_FloatRegister(i), as_FloatRegister(i + 1), Address(__ pre(sp,-16))); + } + +- RegSet saveRegs = RegSet::range(r0,r28) - RegSet::of(r0); +- __ push(saveRegs, sp); +- ++ const RegSet save_regs = RegSet::range(r1, r28); ++ __ push(save_regs, sp); + // Setup arguments + __ load_parameter(0, c_rarg0); + __ load_parameter(1, c_rarg1); + + __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2); + +- __ pop(saveRegs, sp); ++ __ pop(save_regs, sp); + +- for (int i = 30; i >0; i -=2) { +- __ ldpd(as_FloatRegister(i), as_FloatRegister(i+1), Address(__ post(sp, 16))); ++ for (int i = 30; i >0; i -= 2) { ++ __ ldpd(as_FloatRegister(i), as_FloatRegister(i + 1), Address(__ post(sp, 16))); + } + + __ epilogue(); + } + #endif // COMPILER1 + +-#undef __ +-#define __ cgen->assembler()-> +- +-// Generates a register specific stub for calling +-// ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or +-// ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded(). +-// +-// The raddr register serves as both input and output for this stub. When the stub is +-// called the raddr register contains the object field address (oop*) where the bad oop +-// was loaded from, which caused the slow path to be taken. On return from the stub the +-// raddr register contains the good/healed oop returned from +-// ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or +-// ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded(). +-static address generate_load_barrier_stub(StubCodeGenerator* cgen, Register raddr, DecoratorSet decorators) { +- // Don't generate stub for invalid registers +- if (raddr == zr || raddr == r29 || raddr == r30) { +- return NULL; ++#ifdef COMPILER2 ++ ++OptoReg::Name ZBarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) { ++ if (!OptoReg::is_reg(opto_reg)) { ++ return OptoReg::Bad; + } + +- // Create stub name +- char name[64]; +- const bool weak = (decorators & ON_WEAK_OOP_REF) != 0; +- os::snprintf(name, sizeof(name), "zgc_load_barrier%s_stub_%s", weak ? "_weak" : "", raddr->name()); ++ const VMReg vm_reg = OptoReg::as_VMReg(opto_reg); ++ if (vm_reg->is_FloatRegister()) { ++ return opto_reg & ~1; ++ } + +- __ align(CodeEntryAlignment); +- StubCodeMark mark(cgen, "StubRoutines", os::strdup(name, mtCode)); +- address start = __ pc(); ++ return opto_reg; ++} + +- // Save live registers +- RegSet savedRegs = RegSet::range(r0,r18) - RegSet::of(raddr); ++#undef __ ++#define __ _masm-> ++ ++class ZSaveLiveRegisters { ++private: ++ MacroAssembler* const _masm; ++ RegSet _gp_regs; ++ RegSet _fp_regs; ++ ++public: ++ void initialize(ZLoadBarrierStubC2* stub) { ++ // Create mask of live registers ++ RegMask live = stub->live(); ++ ++ // Record registers that needs to be saved/restored ++ while (live.is_NotEmpty()) { ++ const OptoReg::Name opto_reg = live.find_first_elem(); ++ live.Remove(opto_reg); ++ if (OptoReg::is_reg(opto_reg)) { ++ const VMReg vm_reg = OptoReg::as_VMReg(opto_reg); ++ if (vm_reg->is_Register()) { ++ _gp_regs += RegSet::of(vm_reg->as_Register()); ++ } else if (vm_reg->is_FloatRegister()) { ++ _fp_regs += RegSet::of((Register)vm_reg->as_FloatRegister()); ++ } else { ++ fatal("Unknown register type"); ++ } ++ } ++ } + +- __ enter(); +- __ push(savedRegs, sp); ++ // Remove C-ABI SOE registers, scratch regs and _ref register that will be updated ++ _gp_regs -= RegSet::range(r19, r30) + RegSet::of(r8, r9, stub->ref()); ++ } + +- // Setup arguments +- if (raddr != c_rarg1) { +- __ mov(c_rarg1, raddr); +- } ++ ZSaveLiveRegisters(MacroAssembler* masm, ZLoadBarrierStubC2* stub) : ++ _masm(masm), ++ _gp_regs(), ++ _fp_regs() { + +- __ ldr(c_rarg0, Address(raddr)); ++ // Figure out what registers to save/restore ++ initialize(stub); + +- // Call barrier function +- __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), c_rarg0, c_rarg1); ++ // Save registers ++ __ push(_gp_regs, sp); ++ __ push_fp(_fp_regs, sp); ++ } + +- // Move result returned in r0 to raddr, if needed +- if (raddr != r0) { +- __ mov(raddr, r0); ++ ~ZSaveLiveRegisters() { ++ // Restore registers ++ __ pop_fp(_fp_regs, sp); ++ __ pop(_gp_regs, sp); + } ++}; + +- __ pop(savedRegs, sp); +- __ leave(); +- __ ret(lr); ++#undef __ ++#define __ _masm-> ++ ++class ZSetupArguments { ++private: ++ MacroAssembler* const _masm; ++ const Register _ref; ++ const Address _ref_addr; ++ ++public: ++ ZSetupArguments(MacroAssembler* masm, ZLoadBarrierStubC2* stub) : ++ _masm(masm), ++ _ref(stub->ref()), ++ _ref_addr(stub->ref_addr()) { ++ ++ // Setup arguments ++ if (_ref_addr.base() == noreg) { ++ // No self healing ++ if (_ref != c_rarg0) { ++ __ mov(c_rarg0, _ref); ++ } ++ __ mov(c_rarg1, 0); ++ } else { ++ // Self healing ++ if (_ref == c_rarg0) { ++ // _ref is already at correct place ++ __ lea(c_rarg1, _ref_addr); ++ } else if (_ref != c_rarg1) { ++ // _ref is in wrong place, but not in c_rarg1, so fix it first ++ __ lea(c_rarg1, _ref_addr); ++ __ mov(c_rarg0, _ref); ++ } else if (_ref_addr.base() != c_rarg0 && _ref_addr.index() != c_rarg0) { ++ assert(_ref == c_rarg1, "Mov ref first, vacating c_rarg0"); ++ __ mov(c_rarg0, _ref); ++ __ lea(c_rarg1, _ref_addr); ++ } else { ++ assert(_ref == c_rarg1, "Need to vacate c_rarg1 and _ref_addr is using c_rarg0"); ++ if (_ref_addr.base() == c_rarg0 || _ref_addr.index() == c_rarg0) { ++ __ mov(rscratch2, c_rarg1); ++ __ lea(c_rarg1, _ref_addr); ++ __ mov(c_rarg0, rscratch2); ++ } else { ++ ShouldNotReachHere(); ++ } ++ } ++ } ++ } + +- return start; +-} ++ ~ZSetupArguments() { ++ // Transfer result ++ if (_ref != r0) { ++ __ mov(_ref, r0); ++ } ++ } ++}; + + #undef __ ++#define __ masm-> + +-static void barrier_stubs_init_inner(const char* label, const DecoratorSet decorators, address* stub) { +- const int nregs = 28; // Exclude FP, XZR, SP from calculation. +- const int code_size = nregs * 254; // Rough estimate of code size +- +- ResourceMark rm; ++void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const { ++ BLOCK_COMMENT("ZLoadBarrierStubC2"); + +- CodeBuffer buf(BufferBlob::create(label, code_size)); +- StubCodeGenerator cgen(&buf); ++ // Stub entry ++ __ bind(*stub->entry()); + +- for (int i = 0; i < nregs; i++) { +- const Register reg = as_Register(i); +- stub[i] = generate_load_barrier_stub(&cgen, reg, decorators); ++ { ++ ZSaveLiveRegisters save_live_registers(masm, stub); ++ ZSetupArguments setup_arguments(masm, stub); ++ __ mov(rscratch1, stub->slow_path()); ++ __ blr(rscratch1); + } +-} + +-void ZBarrierSetAssembler::barrier_stubs_init() { +- barrier_stubs_init_inner("zgc_load_barrier_stubs", ON_STRONG_OOP_REF, _load_barrier_slow_stub); +- barrier_stubs_init_inner("zgc_load_barrier_weak_stubs", ON_WEAK_OOP_REF, _load_barrier_weak_slow_stub); ++ // Stub exit ++ __ b(*stub->continuation()); + } + +-address ZBarrierSetAssembler::load_barrier_slow_stub(Register reg) { +- return _load_barrier_slow_stub[reg->encoding()]; +-} ++#undef __ + +-address ZBarrierSetAssembler::load_barrier_weak_slow_stub(Register reg) { +- return _load_barrier_weak_slow_stub[reg->encoding()]; +-} ++#endif // COMPILER2 +diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp +index 7e8be01cc..cca873825 100644 +--- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp ++++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp +@@ -24,6 +24,12 @@ + #ifndef CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP + #define CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP + ++#include "code/vmreg.hpp" ++#include "oops/accessDecorators.hpp" ++#ifdef COMPILER2 ++#include "opto/optoreg.hpp" ++#endif // COMPILER2 ++ + #ifdef COMPILER1 + class LIR_Assembler; + class LIR_OprDesc; +@@ -32,14 +38,13 @@ class StubAssembler; + class ZLoadBarrierStubC1; + #endif // COMPILER1 + +-class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase { +-private: +- address _load_barrier_slow_stub[RegisterImpl::number_of_registers]; +- address _load_barrier_weak_slow_stub[RegisterImpl::number_of_registers]; ++#ifdef COMPILER2 ++class Node; ++class ZLoadBarrierStubC2; ++#endif // COMPILER2 + ++class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase { + public: +- ZBarrierSetAssembler(); +- + virtual void load_at(MacroAssembler* masm, + DecoratorSet decorators, + BasicType type, +@@ -83,10 +88,13 @@ public: + DecoratorSet decorators) const; + #endif // COMPILER1 + +- virtual void barrier_stubs_init(); ++#ifdef COMPILER2 ++ OptoReg::Name refine_register(const Node* node, ++ OptoReg::Name opto_reg); + +- address load_barrier_slow_stub(Register reg); +- address load_barrier_weak_slow_stub(Register reg); ++ void generate_c2_load_barrier_stub(MacroAssembler* masm, ++ ZLoadBarrierStubC2* stub) const; ++#endif // COMPILER2 + }; + + #endif // CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP +diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +index 508c13c27..611f13b0e 100644 +--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp ++++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +@@ -2096,6 +2096,65 @@ int MacroAssembler::pop(unsigned int bitset, Register stack) { + + return count; + } ++ ++// Push lots of registers in the bit set supplied. Don't push sp. ++// Return the number of words pushed ++int MacroAssembler::push_fp(unsigned int bitset, Register stack) { ++ int words_pushed = 0; ++ ++ // Scan bitset to accumulate register pairs ++ unsigned char regs[32]; ++ int count = 0; ++ for (int reg = 0; reg <= 31; reg++) { ++ if (1 & bitset) ++ regs[count++] = reg; ++ bitset >>= 1; ++ } ++ regs[count++] = zr->encoding_nocheck(); ++ count &= ~1; // Only push an even number of regs ++ ++ // Always pushing full 128 bit registers. ++ if (count) { ++ stpq(as_FloatRegister(regs[0]), as_FloatRegister(regs[1]), Address(pre(stack, -count * wordSize * 2))); ++ words_pushed += 2; ++ } ++ for (int i = 2; i < count; i += 2) { ++ stpq(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize * 2)); ++ words_pushed += 2; ++ } ++ ++ assert(words_pushed == count, "oops, pushed != count"); ++ return count; ++} ++ ++int MacroAssembler::pop_fp(unsigned int bitset, Register stack) { ++ int words_pushed = 0; ++ ++ // Scan bitset to accumulate register pairs ++ unsigned char regs[32]; ++ int count = 0; ++ for (int reg = 0; reg <= 31; reg++) { ++ if (1 & bitset) ++ regs[count++] = reg; ++ bitset >>= 1; ++ } ++ regs[count++] = zr->encoding_nocheck(); ++ count &= ~1; ++ ++ for (int i = 2; i < count; i += 2) { ++ ldpq(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize * 2)); ++ words_pushed += 2; ++ } ++ if (count) { ++ ldpq(as_FloatRegister(regs[0]), as_FloatRegister(regs[1]), Address(post(stack, count * wordSize * 2))); ++ words_pushed += 2; ++ } ++ ++ assert(words_pushed == count, "oops, pushed != count"); ++ ++ return count; ++} ++ + #ifdef ASSERT + void MacroAssembler::verify_heapbase(const char* msg) { + #if 0 +diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp +index af8056be0..44497ea7c 100644 +--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp ++++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp +@@ -440,12 +440,18 @@ private: + int push(unsigned int bitset, Register stack); + int pop(unsigned int bitset, Register stack); + ++ int push_fp(unsigned int bitset, Register stack); ++ int pop_fp(unsigned int bitset, Register stack); ++ + void mov(Register dst, Address a); + + public: + void push(RegSet regs, Register stack) { if (regs.bits()) push(regs.bits(), stack); } + void pop(RegSet regs, Register stack) { if (regs.bits()) pop(regs.bits(), stack); } + ++ void push_fp(RegSet regs, Register stack) { if (regs.bits()) push_fp(regs.bits(), stack); } ++ void pop_fp(RegSet regs, Register stack) { if (regs.bits()) pop_fp(regs.bits(), stack); } ++ + // Push and pop everything that might be clobbered by a native + // runtime call except rscratch1 and rscratch2. (They are always + // scratch, so we don't have to protect them.) Only save the lower +diff --git a/src/hotspot/cpu/aarch64/register_aarch64.hpp b/src/hotspot/cpu/aarch64/register_aarch64.hpp +index 8cda52a0a..5f7662c89 100644 +--- a/src/hotspot/cpu/aarch64/register_aarch64.hpp ++++ b/src/hotspot/cpu/aarch64/register_aarch64.hpp +@@ -230,6 +230,11 @@ public: + return *this; + } + ++ RegSet &operator-=(const RegSet aSet) { ++ *this = *this - aSet; ++ return *this; ++ } ++ + static RegSet of(Register r1) { + return RegSet(r1); + } +diff --git a/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp +index 45e47f19e..026f441fc 100644 +--- a/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp ++++ b/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp +@@ -873,8 +873,8 @@ void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) { + } + + // Get mirror and store it in the frame as GC root for this Method* +- __ load_mirror(rscratch1, rmethod); +- __ stp(rscratch1, zr, Address(sp, 4 * wordSize)); ++ __ load_mirror(r10, rmethod); ++ __ stp(r10, zr, Address(sp, 4 * wordSize)); + + __ ldr(rcpool, Address(rmethod, Method::const_offset())); + __ ldr(rcpool, Address(rcpool, ConstMethod::constants_offset())); +diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp +index 3aa3f8579..18f455086 100644 +--- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp ++++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp +@@ -27,16 +27,16 @@ + #include "gc/z/zBarrierSet.hpp" + #include "gc/z/zBarrierSetAssembler.hpp" + #include "gc/z/zBarrierSetRuntime.hpp" +-#include "runtime/stubCodeGenerator.hpp" ++#include "runtime/sharedRuntime.hpp" + #include "utilities/macros.hpp" + #ifdef COMPILER1 + #include "c1/c1_LIRAssembler.hpp" + #include "c1/c1_MacroAssembler.hpp" + #include "gc/z/c1/zBarrierSetC1.hpp" + #endif // COMPILER1 +- +-#undef __ +-#define __ masm-> ++#ifdef COMPILER2 ++#include "gc/z/c2/zBarrierSetC2.hpp" ++#endif // COMPILER2 + + #ifdef PRODUCT + #define BLOCK_COMMENT(str) /* nothing */ +@@ -44,6 +44,9 @@ + #define BLOCK_COMMENT(str) __ block_comment(str) + #endif + ++#undef __ ++#define __ masm-> ++ + static void call_vm(MacroAssembler* masm, + address entry_point, + Register arg0, +@@ -333,126 +336,326 @@ void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* + + #endif // COMPILER1 + +-#undef __ +-#define __ cgen->assembler()-> +- +-// Generates a register specific stub for calling +-// ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or +-// ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded(). +-// +-// The raddr register serves as both input and output for this stub. When the stub is +-// called the raddr register contains the object field address (oop*) where the bad oop +-// was loaded from, which caused the slow path to be taken. On return from the stub the +-// raddr register contains the good/healed oop returned from +-// ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or +-// ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded(). +-static address generate_load_barrier_stub(StubCodeGenerator* cgen, Register raddr, DecoratorSet decorators) { +- // Don't generate stub for invalid registers +- if (raddr == rsp || raddr == r12 || raddr == r15) { +- return NULL; ++#ifdef COMPILER2 ++ ++OptoReg::Name ZBarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) { ++ if (!OptoReg::is_reg(opto_reg)) { ++ return OptoReg::Bad; ++ } ++ ++ const VMReg vm_reg = OptoReg::as_VMReg(opto_reg); ++ if (vm_reg->is_XMMRegister()) { ++ opto_reg &= ~15; ++ switch (node->ideal_reg()) { ++ case Op_VecX: ++ opto_reg |= 2; ++ break; ++ case Op_VecY: ++ opto_reg |= 4; ++ break; ++ case Op_VecZ: ++ opto_reg |= 8; ++ break; ++ default: ++ opto_reg |= 1; ++ break; ++ } + } + +- // Create stub name +- char name[64]; +- const bool weak = (decorators & ON_WEAK_OOP_REF) != 0; +- os::snprintf(name, sizeof(name), "load_barrier%s_stub_%s", weak ? "_weak" : "", raddr->name()); ++ return opto_reg; ++} ++ ++// We use the vec_spill_helper from the x86.ad file to avoid reinventing this wheel ++extern int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load, ++ int stack_offset, int reg, uint ireg, outputStream* st); ++#undef __ ++#define __ _masm-> + +- __ align(CodeEntryAlignment); +- StubCodeMark mark(cgen, "StubRoutines", os::strdup(name, mtCode)); +- address start = __ pc(); ++class ZSaveLiveRegisters { ++private: ++ struct XMMRegisterData { ++ XMMRegister _reg; ++ int _size; + +- // Save live registers +- if (raddr != rax) { +- __ push(rax); +- } +- if (raddr != rcx) { +- __ push(rcx); +- } +- if (raddr != rdx) { +- __ push(rdx); ++ // Used by GrowableArray::find() ++ bool operator == (const XMMRegisterData& other) { ++ return _reg == other._reg; ++ } ++ }; ++ ++ MacroAssembler* const _masm; ++ GrowableArray _gp_registers; ++ GrowableArray _xmm_registers; ++ int _spill_size; ++ int _spill_offset; ++ ++ static int xmm_compare_register_size(XMMRegisterData* left, XMMRegisterData* right) { ++ if (left->_size == right->_size) { ++ return 0; ++ } ++ ++ return (left->_size < right->_size) ? -1 : 1; + } +- if (raddr != rsi) { +- __ push(rsi); ++ ++ static int xmm_slot_size(OptoReg::Name opto_reg) { ++ // The low order 4 bytes denote what size of the XMM register is live ++ return (opto_reg & 15) << 3; + } +- if (raddr != rdi) { +- __ push(rdi); ++ ++ static uint xmm_ideal_reg_for_size(int reg_size) { ++ switch (reg_size) { ++ case 8: ++ return Op_VecD; ++ case 16: ++ return Op_VecX; ++ case 32: ++ return Op_VecY; ++ case 64: ++ return Op_VecZ; ++ default: ++ fatal("Invalid register size %d", reg_size); ++ return 0; ++ } + } +- if (raddr != r8) { +- __ push(r8); ++ ++ bool xmm_needs_vzeroupper() const { ++ return _xmm_registers.is_nonempty() && _xmm_registers.at(0)._size > 16; + } +- if (raddr != r9) { +- __ push(r9); ++ ++ void xmm_register_save(const XMMRegisterData& reg_data) { ++ const OptoReg::Name opto_reg = OptoReg::as_OptoReg(reg_data._reg->as_VMReg()); ++ const uint ideal_reg = xmm_ideal_reg_for_size(reg_data._size); ++ _spill_offset -= reg_data._size; ++ vec_spill_helper(__ code(), false /* do_size */, false /* is_load */, _spill_offset, opto_reg, ideal_reg, tty); + } +- if (raddr != r10) { +- __ push(r10); ++ ++ void xmm_register_restore(const XMMRegisterData& reg_data) { ++ const OptoReg::Name opto_reg = OptoReg::as_OptoReg(reg_data._reg->as_VMReg()); ++ const uint ideal_reg = xmm_ideal_reg_for_size(reg_data._size); ++ vec_spill_helper(__ code(), false /* do_size */, true /* is_load */, _spill_offset, opto_reg, ideal_reg, tty); ++ _spill_offset += reg_data._size; + } +- if (raddr != r11) { +- __ push(r11); ++ ++ void gp_register_save(Register reg) { ++ _spill_offset -= 8; ++ __ movq(Address(rsp, _spill_offset), reg); + } + +- // Setup arguments +- if (c_rarg1 != raddr) { +- __ movq(c_rarg1, raddr); ++ void gp_register_restore(Register reg) { ++ __ movq(reg, Address(rsp, _spill_offset)); ++ _spill_offset += 8; + } +- __ movq(c_rarg0, Address(raddr, 0)); + +- // Call barrier function +- __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), c_rarg0, c_rarg1); ++ void initialize(ZLoadBarrierStubC2* stub) { ++ // Create mask of caller saved registers that need to ++ // be saved/restored if live ++ RegMask caller_saved; ++ caller_saved.Insert(OptoReg::as_OptoReg(rax->as_VMReg())); ++ caller_saved.Insert(OptoReg::as_OptoReg(rcx->as_VMReg())); ++ caller_saved.Insert(OptoReg::as_OptoReg(rdx->as_VMReg())); ++ caller_saved.Insert(OptoReg::as_OptoReg(rsi->as_VMReg())); ++ caller_saved.Insert(OptoReg::as_OptoReg(rdi->as_VMReg())); ++ caller_saved.Insert(OptoReg::as_OptoReg(r8->as_VMReg())); ++ caller_saved.Insert(OptoReg::as_OptoReg(r9->as_VMReg())); ++ caller_saved.Insert(OptoReg::as_OptoReg(r10->as_VMReg())); ++ caller_saved.Insert(OptoReg::as_OptoReg(r11->as_VMReg())); ++ caller_saved.Remove(OptoReg::as_OptoReg(stub->ref()->as_VMReg())); + +- // Move result returned in rax to raddr, if needed +- if (raddr != rax) { +- __ movq(raddr, rax); +- } ++ // Create mask of live registers ++ RegMask live = stub->live(); ++ if (stub->tmp() != noreg) { ++ live.Insert(OptoReg::as_OptoReg(stub->tmp()->as_VMReg())); ++ } + +- // Restore saved registers +- if (raddr != r11) { +- __ pop(r11); +- } +- if (raddr != r10) { +- __ pop(r10); +- } +- if (raddr != r9) { +- __ pop(r9); +- } +- if (raddr != r8) { +- __ pop(r8); +- } +- if (raddr != rdi) { +- __ pop(rdi); +- } +- if (raddr != rsi) { +- __ pop(rsi); +- } +- if (raddr != rdx) { +- __ pop(rdx); ++ int gp_spill_size = 0; ++ int xmm_spill_size = 0; ++ ++ // Record registers that needs to be saved/restored ++ while (live.is_NotEmpty()) { ++ const OptoReg::Name opto_reg = live.find_first_elem(); ++ const VMReg vm_reg = OptoReg::as_VMReg(opto_reg); ++ ++ live.Remove(opto_reg); ++ ++ if (vm_reg->is_Register()) { ++ if (caller_saved.Member(opto_reg)) { ++ _gp_registers.append(vm_reg->as_Register()); ++ gp_spill_size += 8; ++ } ++ } else if (vm_reg->is_XMMRegister()) { ++ // We encode in the low order 4 bits of the opto_reg, how large part of the register is live ++ const VMReg vm_reg_base = OptoReg::as_VMReg(opto_reg & ~15); ++ const int reg_size = xmm_slot_size(opto_reg); ++ const XMMRegisterData reg_data = { vm_reg_base->as_XMMRegister(), reg_size }; ++ const int reg_index = _xmm_registers.find(reg_data); ++ if (reg_index == -1) { ++ // Not previously appended ++ _xmm_registers.append(reg_data); ++ xmm_spill_size += reg_size; ++ } else { ++ // Previously appended, update size ++ const int reg_size_prev = _xmm_registers.at(reg_index)._size; ++ if (reg_size > reg_size_prev) { ++ _xmm_registers.at_put(reg_index, reg_data); ++ xmm_spill_size += reg_size - reg_size_prev; ++ } ++ } ++ } else { ++ fatal("Unexpected register type"); ++ } ++ } ++ ++ // Sort by size, largest first ++ _xmm_registers.sort(xmm_compare_register_size); ++ ++ // Stack pointer must be 16 bytes aligned for the call ++ _spill_offset = _spill_size = align_up(xmm_spill_size + gp_spill_size, 16); ++ } ++ ++public: ++ ZSaveLiveRegisters(MacroAssembler* masm, ZLoadBarrierStubC2* stub) : ++ _masm(masm), ++ _gp_registers(), ++ _xmm_registers(), ++ _spill_size(0), ++ _spill_offset(0) { ++ ++ // ++ // Stack layout after registers have been spilled: ++ // ++ // | ... | original rsp, 16 bytes aligned ++ // ------------------ ++ // | zmm0 high | ++ // | ... | ++ // | zmm0 low | 16 bytes aligned ++ // | ... | ++ // | ymm1 high | ++ // | ... | ++ // | ymm1 low | 16 bytes aligned ++ // | ... | ++ // | xmmN high | ++ // | ... | ++ // | xmmN low | 8 bytes aligned ++ // | reg0 | 8 bytes aligned ++ // | reg1 | ++ // | ... | ++ // | regN | new rsp, if 16 bytes aligned ++ // | | else new rsp, 16 bytes aligned ++ // ------------------ ++ // ++ ++ // Figure out what registers to save/restore ++ initialize(stub); ++ ++ // Allocate stack space ++ if (_spill_size > 0) { ++ __ subptr(rsp, _spill_size); ++ } ++ ++ // Save XMM/YMM/ZMM registers ++ for (int i = 0; i < _xmm_registers.length(); i++) { ++ xmm_register_save(_xmm_registers.at(i)); ++ } ++ ++ if (xmm_needs_vzeroupper()) { ++ __ vzeroupper(); ++ } ++ ++ // Save general purpose registers ++ for (int i = 0; i < _gp_registers.length(); i++) { ++ gp_register_save(_gp_registers.at(i)); ++ } + } +- if (raddr != rcx) { +- __ pop(rcx); ++ ++ ~ZSaveLiveRegisters() { ++ // Restore general purpose registers ++ for (int i = _gp_registers.length() - 1; i >= 0; i--) { ++ gp_register_restore(_gp_registers.at(i)); ++ } ++ ++ __ vzeroupper(); ++ ++ // Restore XMM/YMM/ZMM registers ++ for (int i = _xmm_registers.length() - 1; i >= 0; i--) { ++ xmm_register_restore(_xmm_registers.at(i)); ++ } ++ ++ // Free stack space ++ if (_spill_size > 0) { ++ __ addptr(rsp, _spill_size); ++ } + } +- if (raddr != rax) { +- __ pop(rax); ++}; ++ ++class ZSetupArguments { ++private: ++ MacroAssembler* const _masm; ++ const Register _ref; ++ const Address _ref_addr; ++ ++public: ++ ZSetupArguments(MacroAssembler* masm, ZLoadBarrierStubC2* stub) : ++ _masm(masm), ++ _ref(stub->ref()), ++ _ref_addr(stub->ref_addr()) { ++ ++ // Setup arguments ++ if (_ref_addr.base() == noreg) { ++ // No self healing ++ if (_ref != c_rarg0) { ++ __ movq(c_rarg0, _ref); ++ } ++ __ xorq(c_rarg1, c_rarg1); ++ } else { ++ // Self healing ++ if (_ref == c_rarg0) { ++ __ lea(c_rarg1, _ref_addr); ++ } else if (_ref != c_rarg1) { ++ __ lea(c_rarg1, _ref_addr); ++ __ movq(c_rarg0, _ref); ++ } else if (_ref_addr.base() != c_rarg0 && _ref_addr.index() != c_rarg0) { ++ __ movq(c_rarg0, _ref); ++ __ lea(c_rarg1, _ref_addr); ++ } else { ++ __ xchgq(c_rarg0, c_rarg1); ++ if (_ref_addr.base() == c_rarg0) { ++ __ lea(c_rarg1, Address(c_rarg1, _ref_addr.index(), _ref_addr.scale(), _ref_addr.disp())); ++ } else if (_ref_addr.index() == c_rarg0) { ++ __ lea(c_rarg1, Address(_ref_addr.base(), c_rarg1, _ref_addr.scale(), _ref_addr.disp())); ++ } else { ++ ShouldNotReachHere(); ++ } ++ } ++ } + } + +- __ ret(0); +- +- return start; +-} ++ ~ZSetupArguments() { ++ // Transfer result ++ if (_ref != rax) { ++ __ movq(_ref, rax); ++ } ++ } ++}; + + #undef __ ++#define __ masm-> + +-void ZBarrierSetAssembler::barrier_stubs_init() { +- // Load barrier stubs +- int stub_code_size = 256 * 16; // Rough estimate of code size ++void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const { ++ BLOCK_COMMENT("ZLoadBarrierStubC2"); + +- ResourceMark rm; +- BufferBlob* bb = BufferBlob::create("zgc_load_barrier_stubs", stub_code_size); +- CodeBuffer buf(bb); +- StubCodeGenerator cgen(&buf); ++ // Stub entry ++ __ bind(*stub->entry()); + +- Register rr = as_Register(0); +- for (int i = 0; i < RegisterImpl::number_of_registers; i++) { +- _load_barrier_slow_stub[i] = generate_load_barrier_stub(&cgen, rr, ON_STRONG_OOP_REF); +- _load_barrier_weak_slow_stub[i] = generate_load_barrier_stub(&cgen, rr, ON_WEAK_OOP_REF); +- rr = rr->successor(); ++ { ++ ZSaveLiveRegisters save_live_registers(masm, stub); ++ ZSetupArguments setup_arguments(masm, stub); ++ __ call(RuntimeAddress(stub->slow_path())); + } ++ ++ // Stub exit ++ __ jmp(*stub->continuation()); + } ++ ++#undef __ ++ ++#endif // COMPILER2 +diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp +index 3687754e7..e433882a4 100644 +--- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp ++++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp +@@ -24,6 +24,14 @@ + #ifndef CPU_X86_GC_Z_ZBARRIERSETASSEMBLER_X86_HPP + #define CPU_X86_GC_Z_ZBARRIERSETASSEMBLER_X86_HPP + ++#include "code/vmreg.hpp" ++#include "oops/accessDecorators.hpp" ++#ifdef COMPILER2 ++#include "opto/optoreg.hpp" ++#endif // COMPILER2 ++ ++class MacroAssembler; ++ + #ifdef COMPILER1 + class LIR_Assembler; + class LIR_OprDesc; +@@ -32,18 +40,13 @@ class StubAssembler; + class ZLoadBarrierStubC1; + #endif // COMPILER1 + +-class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase { +- address _load_barrier_slow_stub[RegisterImpl::number_of_registers]; +- address _load_barrier_weak_slow_stub[RegisterImpl::number_of_registers]; ++#ifdef COMPILER2 ++class Node; ++class ZLoadBarrierStubC2; ++#endif // COMPILER2 + ++class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase { + public: +- ZBarrierSetAssembler() : +- _load_barrier_slow_stub(), +- _load_barrier_weak_slow_stub() {} +- +- address load_barrier_slow_stub(Register reg) { return _load_barrier_slow_stub[reg->encoding()]; } +- address load_barrier_weak_slow_stub(Register reg) { return _load_barrier_weak_slow_stub[reg->encoding()]; } +- + virtual void load_at(MacroAssembler* masm, + DecoratorSet decorators, + BasicType type, +@@ -86,7 +89,12 @@ public: + DecoratorSet decorators) const; + #endif // COMPILER1 + +- virtual void barrier_stubs_init(); ++#ifdef COMPILER2 ++ OptoReg::Name refine_register(const Node* node, ++ OptoReg::Name opto_reg); ++ void generate_c2_load_barrier_stub(MacroAssembler* masm, ++ ZLoadBarrierStubC2* stub) const; ++#endif // COMPILER2 + }; + + #endif // CPU_X86_GC_Z_ZBARRIERSETASSEMBLER_X86_HPP +diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad +index 7052ccba6..8f0e2225a 100644 +--- a/src/hotspot/cpu/x86/x86.ad ++++ b/src/hotspot/cpu/x86/x86.ad +@@ -1097,138 +1097,6 @@ reg_class vectorz_reg_legacy(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0 + reg_class_dynamic vectorz_reg(vectorz_reg_evex, vectorz_reg_legacy, %{ VM_Version::supports_evex() %} ); + reg_class_dynamic vectorz_reg_vl(vectorz_reg_evex, vectorz_reg_legacy, %{ VM_Version::supports_evex() && VM_Version::supports_avx512vl() %} ); + +-reg_class xmm0_reg(XMM0, XMM0b, XMM0c, XMM0d); +-reg_class ymm0_reg(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h); +-reg_class zmm0_reg(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h, XMM0i, XMM0j, XMM0k, XMM0l, XMM0m, XMM0n, XMM0o, XMM0p); +- +-reg_class xmm1_reg(XMM1, XMM1b, XMM1c, XMM1d); +-reg_class ymm1_reg(XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h); +-reg_class zmm1_reg(XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h, XMM1i, XMM1j, XMM1k, XMM1l, XMM1m, XMM1n, XMM1o, XMM1p); +- +-reg_class xmm2_reg(XMM2, XMM2b, XMM2c, XMM2d); +-reg_class ymm2_reg(XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h); +-reg_class zmm2_reg(XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h, XMM2i, XMM2j, XMM2k, XMM2l, XMM2m, XMM2n, XMM2o, XMM2p); +- +-reg_class xmm3_reg(XMM3, XMM3b, XMM3c, XMM3d); +-reg_class ymm3_reg(XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h); +-reg_class zmm3_reg(XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h, XMM3i, XMM3j, XMM3k, XMM3l, XMM3m, XMM3n, XMM3o, XMM3p); +- +-reg_class xmm4_reg(XMM4, XMM4b, XMM4c, XMM4d); +-reg_class ymm4_reg(XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h); +-reg_class zmm4_reg(XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h, XMM4i, XMM4j, XMM4k, XMM4l, XMM4m, XMM4n, XMM4o, XMM4p); +- +-reg_class xmm5_reg(XMM5, XMM5b, XMM5c, XMM5d); +-reg_class ymm5_reg(XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h); +-reg_class zmm5_reg(XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h, XMM5i, XMM5j, XMM5k, XMM5l, XMM5m, XMM5n, XMM5o, XMM5p); +- +-reg_class xmm6_reg(XMM6, XMM6b, XMM6c, XMM6d); +-reg_class ymm6_reg(XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h); +-reg_class zmm6_reg(XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h, XMM6i, XMM6j, XMM6k, XMM6l, XMM6m, XMM6n, XMM6o, XMM6p); +- +-reg_class xmm7_reg(XMM7, XMM7b, XMM7c, XMM7d); +-reg_class ymm7_reg(XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h); +-reg_class zmm7_reg(XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h, XMM7i, XMM7j, XMM7k, XMM7l, XMM7m, XMM7n, XMM7o, XMM7p); +- +-#ifdef _LP64 +- +-reg_class xmm8_reg(XMM8, XMM8b, XMM8c, XMM8d); +-reg_class ymm8_reg(XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h); +-reg_class zmm8_reg(XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h, XMM8i, XMM8j, XMM8k, XMM8l, XMM8m, XMM8n, XMM8o, XMM8p); +- +-reg_class xmm9_reg(XMM9, XMM9b, XMM9c, XMM9d); +-reg_class ymm9_reg(XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h); +-reg_class zmm9_reg(XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h, XMM9i, XMM9j, XMM9k, XMM9l, XMM9m, XMM9n, XMM9o, XMM9p); +- +-reg_class xmm10_reg(XMM10, XMM10b, XMM10c, XMM10d); +-reg_class ymm10_reg(XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h); +-reg_class zmm10_reg(XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p); +- +-reg_class xmm11_reg(XMM11, XMM11b, XMM11c, XMM11d); +-reg_class ymm11_reg(XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h); +-reg_class zmm11_reg(XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p); +- +-reg_class xmm12_reg(XMM12, XMM12b, XMM12c, XMM12d); +-reg_class ymm12_reg(XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h); +-reg_class zmm12_reg(XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p); +- +-reg_class xmm13_reg(XMM13, XMM13b, XMM13c, XMM13d); +-reg_class ymm13_reg(XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h); +-reg_class zmm13_reg(XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p); +- +-reg_class xmm14_reg(XMM14, XMM14b, XMM14c, XMM14d); +-reg_class ymm14_reg(XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h); +-reg_class zmm14_reg(XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p); +- +-reg_class xmm15_reg(XMM15, XMM15b, XMM15c, XMM15d); +-reg_class ymm15_reg(XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h); +-reg_class zmm15_reg(XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p); +- +-reg_class xmm16_reg(XMM16, XMM16b, XMM16c, XMM16d); +-reg_class ymm16_reg(XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h); +-reg_class zmm16_reg(XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h, XMM16i, XMM16j, XMM16k, XMM16l, XMM16m, XMM16n, XMM16o, XMM16p); +- +-reg_class xmm17_reg(XMM17, XMM17b, XMM17c, XMM17d); +-reg_class ymm17_reg(XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h); +-reg_class zmm17_reg(XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h, XMM17i, XMM17j, XMM17k, XMM17l, XMM17m, XMM17n, XMM17o, XMM17p); +- +-reg_class xmm18_reg(XMM18, XMM18b, XMM18c, XMM18d); +-reg_class ymm18_reg(XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h); +-reg_class zmm18_reg(XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h, XMM18i, XMM18j, XMM18k, XMM18l, XMM18m, XMM18n, XMM18o, XMM18p); +- +-reg_class xmm19_reg(XMM19, XMM19b, XMM19c, XMM19d); +-reg_class ymm19_reg(XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h); +-reg_class zmm19_reg(XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h, XMM19i, XMM19j, XMM19k, XMM19l, XMM19m, XMM19n, XMM19o, XMM19p); +- +-reg_class xmm20_reg(XMM20, XMM20b, XMM20c, XMM20d); +-reg_class ymm20_reg(XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h); +-reg_class zmm20_reg(XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h, XMM20i, XMM20j, XMM20k, XMM20l, XMM20m, XMM20n, XMM20o, XMM20p); +- +-reg_class xmm21_reg(XMM21, XMM21b, XMM21c, XMM21d); +-reg_class ymm21_reg(XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h); +-reg_class zmm21_reg(XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h, XMM21i, XMM21j, XMM21k, XMM21l, XMM21m, XMM21n, XMM21o, XMM21p); +- +-reg_class xmm22_reg(XMM22, XMM22b, XMM22c, XMM22d); +-reg_class ymm22_reg(XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h); +-reg_class zmm22_reg(XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h, XMM22i, XMM22j, XMM22k, XMM22l, XMM22m, XMM22n, XMM22o, XMM22p); +- +-reg_class xmm23_reg(XMM23, XMM23b, XMM23c, XMM23d); +-reg_class ymm23_reg(XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h); +-reg_class zmm23_reg(XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h, XMM23i, XMM23j, XMM23k, XMM23l, XMM23m, XMM23n, XMM23o, XMM23p); +- +-reg_class xmm24_reg(XMM24, XMM24b, XMM24c, XMM24d); +-reg_class ymm24_reg(XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h); +-reg_class zmm24_reg(XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h, XMM24i, XMM24j, XMM24k, XMM24l, XMM24m, XMM24n, XMM24o, XMM24p); +- +-reg_class xmm25_reg(XMM25, XMM25b, XMM25c, XMM25d); +-reg_class ymm25_reg(XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h); +-reg_class zmm25_reg(XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h, XMM25i, XMM25j, XMM25k, XMM25l, XMM25m, XMM25n, XMM25o, XMM25p); +- +-reg_class xmm26_reg(XMM26, XMM26b, XMM26c, XMM26d); +-reg_class ymm26_reg(XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h); +-reg_class zmm26_reg(XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h, XMM26i, XMM26j, XMM26k, XMM26l, XMM26m, XMM26n, XMM26o, XMM26p); +- +-reg_class xmm27_reg(XMM27, XMM27b, XMM27c, XMM27d); +-reg_class ymm27_reg(XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h); +-reg_class zmm27_reg(XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h, XMM27i, XMM27j, XMM27k, XMM27l, XMM27m, XMM27n, XMM27o, XMM27p); +- +-reg_class xmm28_reg(XMM28, XMM28b, XMM28c, XMM28d); +-reg_class ymm28_reg(XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h); +-reg_class zmm28_reg(XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h, XMM28i, XMM28j, XMM28k, XMM28l, XMM28m, XMM28n, XMM28o, XMM28p); +- +-reg_class xmm29_reg(XMM29, XMM29b, XMM29c, XMM29d); +-reg_class ymm29_reg(XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h); +-reg_class zmm29_reg(XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h, XMM29i, XMM29j, XMM29k, XMM29l, XMM29m, XMM29n, XMM29o, XMM29p); +- +-reg_class xmm30_reg(XMM30, XMM30b, XMM30c, XMM30d); +-reg_class ymm30_reg(XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h); +-reg_class zmm30_reg(XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h, XMM30i, XMM30j, XMM30k, XMM30l, XMM30m, XMM30n, XMM30o, XMM30p); +- +-reg_class xmm31_reg(XMM31, XMM31b, XMM31c, XMM31d); +-reg_class ymm31_reg(XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h); +-reg_class zmm31_reg(XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h, XMM31i, XMM31j, XMM31k, XMM31l, XMM31m, XMM31n, XMM31o, XMM31p); +- +-#endif +- + %} + + +@@ -1735,8 +1603,8 @@ static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo + return (UseAVX > 2) ? 6 : 4; + } + +-static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load, +- int stack_offset, int reg, uint ireg, outputStream* st) { ++int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load, ++ int stack_offset, int reg, uint ireg, outputStream* st) { + // In 64-bit VM size calculation is very complex. Emitting instructions + // into scratch buffer is used to get size in 64-bit VM. + LP64_ONLY( assert(!do_size, "this method calculates size only for 32-bit VM"); ) +diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad +index 97c396875..f795fb823 100644 +--- a/src/hotspot/cpu/x86/x86_64.ad ++++ b/src/hotspot/cpu/x86/x86_64.ad +@@ -541,6 +541,7 @@ reg_class int_rdi_reg(RDI); + source_hpp %{ + + #include "gc/z/c2/zBarrierSetC2.hpp" ++#include "gc/z/zThreadLocalData.hpp" + + %} + +@@ -1088,8 +1089,8 @@ static enum RC rc_class(OptoReg::Name reg) + static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo, + int src_hi, int dst_hi, uint ireg, outputStream* st); + +-static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load, +- int stack_offset, int reg, uint ireg, outputStream* st); ++int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load, ++ int stack_offset, int reg, uint ireg, outputStream* st); + + static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset, + int dst_offset, uint ireg, outputStream* st) { +@@ -1800,6 +1801,19 @@ const RegMask Matcher::method_handle_invoke_SP_save_mask() { + return NO_REG_mask(); + } + ++static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) { ++ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, weak); ++ __ testptr(ref, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); ++ __ jcc(Assembler::notZero, *stub->entry()); ++ __ bind(*stub->continuation()); ++} ++ ++static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) { ++ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, false /* weak */); ++ __ jmp(*stub->entry()); ++ __ bind(*stub->continuation()); ++} ++ + %} + + //----------ENCODING BLOCK----------------------------------------------------- +@@ -4284,136 +4298,6 @@ operand cmpOpUCF2() %{ + %} + %} + +-// Operands for bound floating pointer register arguments +-operand rxmm0() %{ +- constraint(ALLOC_IN_RC(xmm0_reg)); match(VecX); +- predicate((UseSSE > 0) && (UseAVX<= 2)); format%{%} interface(REG_INTER); +-%} +-operand rxmm1() %{ +- constraint(ALLOC_IN_RC(xmm1_reg)); match(VecX); +- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +-%} +-operand rxmm2() %{ +- constraint(ALLOC_IN_RC(xmm2_reg)); match(VecX); +- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +-%} +-operand rxmm3() %{ +- constraint(ALLOC_IN_RC(xmm3_reg)); match(VecX); +- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +-%} +-operand rxmm4() %{ +- constraint(ALLOC_IN_RC(xmm4_reg)); match(VecX); +- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +-%} +-operand rxmm5() %{ +- constraint(ALLOC_IN_RC(xmm5_reg)); match(VecX); +- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +-%} +-operand rxmm6() %{ +- constraint(ALLOC_IN_RC(xmm6_reg)); match(VecX); +- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +-%} +-operand rxmm7() %{ +- constraint(ALLOC_IN_RC(xmm7_reg)); match(VecX); +- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +-%} +-operand rxmm8() %{ +- constraint(ALLOC_IN_RC(xmm8_reg)); match(VecX); +- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +-%} +-operand rxmm9() %{ +- constraint(ALLOC_IN_RC(xmm9_reg)); match(VecX); +- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +-%} +-operand rxmm10() %{ +- constraint(ALLOC_IN_RC(xmm10_reg)); match(VecX); +- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +-%} +-operand rxmm11() %{ +- constraint(ALLOC_IN_RC(xmm11_reg)); match(VecX); +- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +-%} +-operand rxmm12() %{ +- constraint(ALLOC_IN_RC(xmm12_reg)); match(VecX); +- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +-%} +-operand rxmm13() %{ +- constraint(ALLOC_IN_RC(xmm13_reg)); match(VecX); +- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +-%} +-operand rxmm14() %{ +- constraint(ALLOC_IN_RC(xmm14_reg)); match(VecX); +- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +-%} +-operand rxmm15() %{ +- constraint(ALLOC_IN_RC(xmm15_reg)); match(VecX); +- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +-%} +-operand rxmm16() %{ +- constraint(ALLOC_IN_RC(xmm16_reg)); match(VecX); +- predicate(UseAVX == 3); format%{%} interface(REG_INTER); +-%} +-operand rxmm17() %{ +- constraint(ALLOC_IN_RC(xmm17_reg)); match(VecX); +- predicate(UseAVX == 3); format%{%} interface(REG_INTER); +-%} +-operand rxmm18() %{ +- constraint(ALLOC_IN_RC(xmm18_reg)); match(VecX); +- predicate(UseAVX == 3); format%{%} interface(REG_INTER); +-%} +-operand rxmm19() %{ +- constraint(ALLOC_IN_RC(xmm19_reg)); match(VecX); +- predicate(UseAVX == 3); format%{%} interface(REG_INTER); +-%} +-operand rxmm20() %{ +- constraint(ALLOC_IN_RC(xmm20_reg)); match(VecX); +- predicate(UseAVX == 3); format%{%} interface(REG_INTER); +-%} +-operand rxmm21() %{ +- constraint(ALLOC_IN_RC(xmm21_reg)); match(VecX); +- predicate(UseAVX == 3); format%{%} interface(REG_INTER); +-%} +-operand rxmm22() %{ +- constraint(ALLOC_IN_RC(xmm22_reg)); match(VecX); +- predicate(UseAVX == 3); format%{%} interface(REG_INTER); +-%} +-operand rxmm23() %{ +- constraint(ALLOC_IN_RC(xmm23_reg)); match(VecX); +- predicate(UseAVX == 3); format%{%} interface(REG_INTER); +-%} +-operand rxmm24() %{ +- constraint(ALLOC_IN_RC(xmm24_reg)); match(VecX); +- predicate(UseAVX == 3); format%{%} interface(REG_INTER); +-%} +-operand rxmm25() %{ +- constraint(ALLOC_IN_RC(xmm25_reg)); match(VecX); +- predicate(UseAVX == 3); format%{%} interface(REG_INTER); +-%} +-operand rxmm26() %{ +- constraint(ALLOC_IN_RC(xmm26_reg)); match(VecX); +- predicate(UseAVX == 3); format%{%} interface(REG_INTER); +-%} +-operand rxmm27() %{ +- constraint(ALLOC_IN_RC(xmm27_reg)); match(VecX); +- predicate(UseAVX == 3); format%{%} interface(REG_INTER); +-%} +-operand rxmm28() %{ +- constraint(ALLOC_IN_RC(xmm28_reg)); match(VecX); +- predicate(UseAVX == 3); format%{%} interface(REG_INTER); +-%} +-operand rxmm29() %{ +- constraint(ALLOC_IN_RC(xmm29_reg)); match(VecX); +- predicate(UseAVX == 3); format%{%} interface(REG_INTER); +-%} +-operand rxmm30() %{ +- constraint(ALLOC_IN_RC(xmm30_reg)); match(VecX); +- predicate(UseAVX == 3); format%{%} interface(REG_INTER); +-%} +-operand rxmm31() %{ +- constraint(ALLOC_IN_RC(xmm31_reg)); match(VecX); +- predicate(UseAVX == 3); format%{%} interface(REG_INTER); +-%} +- + //----------OPERAND CLASSES---------------------------------------------------- + // Operand Classes are groups of operands that are used as to simplify + // instruction definitions by not requiring the AD writer to specify separate +@@ -5306,6 +5190,7 @@ instruct loadRange(rRegI dst, memory mem) + instruct loadP(rRegP dst, memory mem) + %{ + match(Set dst (LoadP mem)); ++ predicate(n->as_Load()->barrier_data() == 0); + + ins_cost(125); // XXX + format %{ "movq $dst, $mem\t# ptr" %} +@@ -7526,6 +7411,7 @@ instruct storePConditional(memory heap_top_ptr, + rax_RegP oldval, rRegP newval, + rFlagsReg cr) + %{ ++ predicate(n->as_LoadStore()->barrier_data() == 0); + match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval))); + + format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) " +@@ -7577,7 +7463,7 @@ instruct compareAndSwapP(rRegI res, + rax_RegP oldval, rRegP newval, + rFlagsReg cr) + %{ +- predicate(VM_Version::supports_cx8()); ++ predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0); + match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval))); + match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval))); + effect(KILL cr, KILL oldval); +@@ -7819,7 +7705,7 @@ instruct compareAndExchangeP( + rax_RegP oldval, rRegP newval, + rFlagsReg cr) + %{ +- predicate(VM_Version::supports_cx8()); ++ predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0); + match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval))); + effect(KILL cr); + +@@ -7964,6 +7850,7 @@ instruct xchgL( memory mem, rRegL newval) %{ + + instruct xchgP( memory mem, rRegP newval) %{ + match(Set newval (GetAndSetP mem newval)); ++ predicate(n->as_LoadStore()->barrier_data() == 0); + format %{ "XCHGQ $newval,[$mem]" %} + ins_encode %{ + __ xchgq($newval$$Register, $mem$$Address); +@@ -11614,6 +11501,7 @@ instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2) + instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2) + %{ + match(Set cr (CmpP op1 (LoadP op2))); ++ predicate(n->in(2)->as_Load()->barrier_data() == 0); + + ins_cost(500); // XXX + format %{ "cmpq $op1, $op2\t# ptr" %} +@@ -11639,7 +11527,8 @@ instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2) + // and raw pointers have no anti-dependencies. + instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2) + %{ +- predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none); ++ predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none && ++ n->in(2)->as_Load()->barrier_data() == 0); + match(Set cr (CmpP op1 (LoadP op2))); + + format %{ "cmpq $op1, $op2\t# raw ptr" %} +@@ -11664,7 +11553,8 @@ instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero) + // any compare to a zero should be eq/neq. + instruct testP_mem(rFlagsReg cr, memory op, immP0 zero) + %{ +- predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL)); ++ predicate((!UseCompressedOops || (Universe::narrow_oop_base() != NULL)) && ++ n->in(1)->as_Load()->barrier_data() == 0); + match(Set cr (CmpP (LoadP op) zero)); + + ins_cost(500); // XXX +@@ -11677,7 +11567,9 @@ instruct testP_mem(rFlagsReg cr, memory op, immP0 zero) + + instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero) + %{ +- predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL)); ++ predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && ++ (Universe::narrow_klass_base() == NULL) && ++ n->in(1)->as_Load()->barrier_data() == 0); + match(Set cr (CmpP (LoadP mem) zero)); + + format %{ "cmpq R12, $mem\t# ptr (R12_heapbase==0)" %} +@@ -12632,274 +12524,126 @@ instruct RethrowException() + // Execute ZGC load barrier (strong) slow path + // + +-// When running without XMM regs +-instruct loadBarrierSlowRegNoVec(rRegP dst, memory mem, rFlagsReg cr) %{ ++// Load Pointer ++instruct zLoadP(rRegP dst, memory mem, rFlagsReg cr) ++%{ ++ predicate(UseZGC && n->as_Load()->barrier_data() == ZLoadBarrierStrong); ++ match(Set dst (LoadP mem)); ++ effect(KILL cr, TEMP dst); + +- match(Set dst (LoadBarrierSlowReg mem)); +- predicate(MaxVectorSize < 16 && !n->as_LoadBarrierSlowReg()->is_weak()); ++ ins_cost(125); + +- effect(DEF dst, KILL cr); ++ format %{ "movq $dst, $mem" %} + +- format %{"LoadBarrierSlowRegNoVec $dst, $mem" %} + ins_encode %{ +-#if INCLUDE_ZGC +- Register d = $dst$$Register; +- ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); +- +- assert(d != r12, "Can't be R12!"); +- assert(d != r15, "Can't be R15!"); +- assert(d != rsp, "Can't be RSP!"); +- +- __ lea(d, $mem$$Address); +- __ call(RuntimeAddress(bs->load_barrier_slow_stub(d))); +-#else +- ShouldNotReachHere(); +-#endif ++ __ movptr($dst$$Register, $mem$$Address); ++ if (barrier_data() != ZLoadBarrierElided) { ++ z_load_barrier(_masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, false /* weak */); ++ } + %} +- ins_pipe(pipe_slow); +-%} + +-// For XMM and YMM enabled processors +-instruct loadBarrierSlowRegXmmAndYmm(rRegP dst, memory mem, rFlagsReg cr, +- rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3, +- rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7, +- rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11, +- rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{ ++ ins_pipe(ialu_reg_mem); ++%} + +- match(Set dst (LoadBarrierSlowReg mem)); +- predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16) && !n->as_LoadBarrierSlowReg()->is_weak()); ++// Load Weak Pointer ++instruct zLoadWeakP(rRegP dst, memory mem, rFlagsReg cr) ++%{ ++ predicate(UseZGC && n->as_Load()->barrier_data() == ZLoadBarrierWeak); ++ match(Set dst (LoadP mem)); ++ effect(KILL cr, TEMP dst); + +- effect(DEF dst, KILL cr, +- KILL x0, KILL x1, KILL x2, KILL x3, +- KILL x4, KILL x5, KILL x6, KILL x7, +- KILL x8, KILL x9, KILL x10, KILL x11, +- KILL x12, KILL x13, KILL x14, KILL x15); ++ ins_cost(125); + +- format %{"LoadBarrierSlowRegXmm $dst, $mem" %} ++ format %{ "movq $dst, $mem" %} + ins_encode %{ +-#if INCLUDE_ZGC +- Register d = $dst$$Register; +- ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); +- +- assert(d != r12, "Can't be R12!"); +- assert(d != r15, "Can't be R15!"); +- assert(d != rsp, "Can't be RSP!"); +- +- __ lea(d, $mem$$Address); +- __ call(RuntimeAddress(bs->load_barrier_slow_stub(d))); +-#else +- ShouldNotReachHere(); +-#endif ++ __ movptr($dst$$Register, $mem$$Address); ++ z_load_barrier(_masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, true /* weak */); + %} +- ins_pipe(pipe_slow); +-%} + +-// For ZMM enabled processors +-instruct loadBarrierSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr, +- rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3, +- rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7, +- rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11, +- rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15, +- rxmm16 x16, rxmm17 x17, rxmm18 x18, rxmm19 x19, +- rxmm20 x20, rxmm21 x21, rxmm22 x22, rxmm23 x23, +- rxmm24 x24, rxmm25 x25, rxmm26 x26, rxmm27 x27, +- rxmm28 x28, rxmm29 x29, rxmm30 x30, rxmm31 x31) %{ +- +- match(Set dst (LoadBarrierSlowReg mem)); +- predicate((UseAVX == 3) && (MaxVectorSize >= 16) && !n->as_LoadBarrierSlowReg()->is_weak()); +- +- effect(DEF dst, KILL cr, +- KILL x0, KILL x1, KILL x2, KILL x3, +- KILL x4, KILL x5, KILL x6, KILL x7, +- KILL x8, KILL x9, KILL x10, KILL x11, +- KILL x12, KILL x13, KILL x14, KILL x15, +- KILL x16, KILL x17, KILL x18, KILL x19, +- KILL x20, KILL x21, KILL x22, KILL x23, +- KILL x24, KILL x25, KILL x26, KILL x27, +- KILL x28, KILL x29, KILL x30, KILL x31); +- +- format %{"LoadBarrierSlowRegZmm $dst, $mem" %} +- ins_encode %{ +-#if INCLUDE_ZGC +- Register d = $dst$$Register; +- ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); +- +- assert(d != r12, "Can't be R12!"); +- assert(d != r15, "Can't be R15!"); +- assert(d != rsp, "Can't be RSP!"); +- +- __ lea(d, $mem$$Address); +- __ call(RuntimeAddress(bs->load_barrier_slow_stub(d))); +-#else +- ShouldNotReachHere(); +-#endif +- %} +- ins_pipe(pipe_slow); ++ ins_pipe(ialu_reg_mem); + %} + +-// +-// Execute ZGC load barrier (weak) slow path +-// +- +-// When running without XMM regs +-instruct loadBarrierWeakSlowRegNoVec(rRegP dst, memory mem, rFlagsReg cr) %{ ++instruct zCompareAndExchangeP(memory mem, rax_RegP oldval, rRegP newval, rRegP tmp, rFlagsReg cr) %{ ++ match(Set oldval (CompareAndExchangeP mem (Binary oldval newval))); ++ predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); ++ effect(KILL cr, TEMP tmp); + +- match(Set dst (LoadBarrierSlowReg mem)); +- predicate(MaxVectorSize < 16 && n->as_LoadBarrierSlowReg()->is_weak()); ++ format %{ "lock\n\t" ++ "cmpxchgq $newval, $mem" %} + +- effect(DEF dst, KILL cr); +- +- format %{"LoadBarrierSlowRegNoVec $dst, $mem" %} + ins_encode %{ +-#if INCLUDE_ZGC +- Register d = $dst$$Register; +- ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); +- +- assert(d != r12, "Can't be R12!"); +- assert(d != r15, "Can't be R15!"); +- assert(d != rsp, "Can't be RSP!"); +- +- __ lea(d, $mem$$Address); +- __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d))); +-#else +- ShouldNotReachHere(); +-#endif ++ if (barrier_data() != ZLoadBarrierElided) { ++ __ movptr($tmp$$Register, $oldval$$Register); ++ } ++ __ lock(); ++ __ cmpxchgptr($newval$$Register, $mem$$Address); ++ if (barrier_data() != ZLoadBarrierElided) { ++ Label good; ++ __ testptr($oldval$$Register, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); ++ __ jcc(Assembler::zero, good); ++ z_load_barrier_slow_path(_masm, this, $mem$$Address, $oldval$$Register, $tmp$$Register); ++ __ movptr($oldval$$Register, $tmp$$Register); ++ __ lock(); ++ __ cmpxchgptr($newval$$Register, $mem$$Address); ++ __ bind(good); ++ } + %} +- ins_pipe(pipe_slow); +-%} + +-// For XMM and YMM enabled processors +-instruct loadBarrierWeakSlowRegXmmAndYmm(rRegP dst, memory mem, rFlagsReg cr, +- rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3, +- rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7, +- rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11, +- rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{ +- +- match(Set dst (LoadBarrierSlowReg mem)); +- predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16) && n->as_LoadBarrierSlowReg()->is_weak()); +- +- effect(DEF dst, KILL cr, +- KILL x0, KILL x1, KILL x2, KILL x3, +- KILL x4, KILL x5, KILL x6, KILL x7, +- KILL x8, KILL x9, KILL x10, KILL x11, +- KILL x12, KILL x13, KILL x14, KILL x15); +- +- format %{"LoadBarrierWeakSlowRegXmm $dst, $mem" %} +- ins_encode %{ +-#if INCLUDE_ZGC +- Register d = $dst$$Register; +- ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); +- +- assert(d != r12, "Can't be R12!"); +- assert(d != r15, "Can't be R15!"); +- assert(d != rsp, "Can't be RSP!"); +- +- __ lea(d,$mem$$Address); +- __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d))); +-#else +- ShouldNotReachHere(); +-#endif +- %} +- ins_pipe(pipe_slow); ++ ins_pipe(pipe_cmpxchg); + %} + +-// For ZMM enabled processors +-instruct loadBarrierWeakSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr, +- rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3, +- rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7, +- rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11, +- rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15, +- rxmm16 x16, rxmm17 x17, rxmm18 x18, rxmm19 x19, +- rxmm20 x20, rxmm21 x21, rxmm22 x22, rxmm23 x23, +- rxmm24 x24, rxmm25 x25, rxmm26 x26, rxmm27 x27, +- rxmm28 x28, rxmm29 x29, rxmm30 x30, rxmm31 x31) %{ +- +- match(Set dst (LoadBarrierSlowReg mem)); +- predicate((UseAVX == 3) && (MaxVectorSize >= 16) && n->as_LoadBarrierSlowReg()->is_weak()); +- +- effect(DEF dst, KILL cr, +- KILL x0, KILL x1, KILL x2, KILL x3, +- KILL x4, KILL x5, KILL x6, KILL x7, +- KILL x8, KILL x9, KILL x10, KILL x11, +- KILL x12, KILL x13, KILL x14, KILL x15, +- KILL x16, KILL x17, KILL x18, KILL x19, +- KILL x20, KILL x21, KILL x22, KILL x23, +- KILL x24, KILL x25, KILL x26, KILL x27, +- KILL x28, KILL x29, KILL x30, KILL x31); +- +- format %{"LoadBarrierWeakSlowRegZmm $dst, $mem" %} +- ins_encode %{ +-#if INCLUDE_ZGC +- Register d = $dst$$Register; +- ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); + +- assert(d != r12, "Can't be R12!"); +- assert(d != r15, "Can't be R15!"); +- assert(d != rsp, "Can't be RSP!"); ++instruct zCompareAndSwapP(rRegI res, memory mem, rRegP newval, rRegP tmp, rFlagsReg cr, rax_RegP oldval) %{ ++ match(Set res (CompareAndSwapP mem (Binary oldval newval))); ++ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); ++ predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); ++ effect(KILL cr, KILL oldval, TEMP tmp); ++ ++ format %{ "lock\n\t" ++ "cmpxchgq $newval, $mem\n\t" ++ "sete $res\n\t" ++ "movzbl $res, $res" %} + +- __ lea(d,$mem$$Address); +- __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d))); +-#else +- ShouldNotReachHere(); +-#endif ++ ins_encode %{ ++ if (barrier_data() != ZLoadBarrierElided) { ++ __ movptr($tmp$$Register, $oldval$$Register); ++ } ++ __ lock(); ++ __ cmpxchgptr($newval$$Register, $mem$$Address); ++ if (barrier_data() != ZLoadBarrierElided) { ++ Label good; ++ __ testptr($oldval$$Register, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); ++ __ jcc(Assembler::zero, good); ++ z_load_barrier_slow_path(_masm, this, $mem$$Address, $oldval$$Register, $tmp$$Register); ++ __ movptr($oldval$$Register, $tmp$$Register); ++ __ lock(); ++ __ cmpxchgptr($newval$$Register, $mem$$Address); ++ __ bind(good); ++ __ cmpptr($tmp$$Register, $oldval$$Register); ++ } ++ __ setb(Assembler::equal, $res$$Register); ++ __ movzbl($res$$Register, $res$$Register); + %} +- ins_pipe(pipe_slow); +-%} +- +-// Specialized versions of compareAndExchangeP that adds a keepalive that is consumed +-// but doesn't affect output. +- +-instruct z_compareAndExchangeP( +- memory mem_ptr, +- rax_RegP oldval, rRegP newval, rRegP keepalive, +- rFlagsReg cr) %{ +- predicate(VM_Version::supports_cx8()); +- match(Set oldval (ZCompareAndExchangeP (Binary mem_ptr keepalive) (Binary oldval newval))); +- effect(KILL cr); + +- format %{ "cmpxchgq $mem_ptr,$newval\t# " +- "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %} +- opcode(0x0F, 0xB1); +- ins_encode(lock_prefix, +- REX_reg_mem_wide(newval, mem_ptr), +- OpcP, OpcS, +- reg_mem(newval, mem_ptr) // lock cmpxchg +- ); +- ins_pipe( pipe_cmpxchg ); ++ ins_pipe(pipe_cmpxchg); + %} + +-instruct z_compareAndSwapP(rRegI res, +- memory mem_ptr, +- rax_RegP oldval, rRegP newval, rRegP keepalive, +- rFlagsReg cr) %{ +- predicate(VM_Version::supports_cx8()); +- match(Set res (ZCompareAndSwapP (Binary mem_ptr keepalive) (Binary oldval newval))); +- match(Set res (ZWeakCompareAndSwapP (Binary mem_ptr keepalive) (Binary oldval newval))); +- effect(KILL cr, KILL oldval); ++instruct zXChgP(memory mem, rRegP newval, rFlagsReg cr) %{ ++ match(Set newval (GetAndSetP mem newval)); ++ predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); ++ effect(KILL cr); + +- format %{ "cmpxchgq $mem_ptr,$newval\t# " +- "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" +- "sete $res\n\t" +- "movzbl $res, $res" %} +- opcode(0x0F, 0xB1); +- ins_encode(lock_prefix, +- REX_reg_mem_wide(newval, mem_ptr), +- OpcP, OpcS, +- reg_mem(newval, mem_ptr), +- REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete +- REX_reg_breg(res, res), // movzbl +- Opcode(0xF), Opcode(0xB6), reg_reg(res, res)); +- ins_pipe( pipe_cmpxchg ); +-%} ++ format %{ "xchgq $newval, $mem" %} + +-instruct z_xchgP( memory mem, rRegP newval, rRegP keepalive) %{ +- match(Set newval (ZGetAndSetP mem (Binary newval keepalive))); +- format %{ "XCHGQ $newval,[$mem]" %} + ins_encode %{ +- __ xchgq($newval$$Register, $mem$$Address); ++ __ xchgptr($newval$$Register, $mem$$Address); ++ if (barrier_data() != ZLoadBarrierElided) { ++ z_load_barrier(_masm, this, Address(noreg, 0), $newval$$Register, noreg /* tmp */, false /* weak */); ++ } + %} +- ins_pipe( pipe_cmpxchg ); +-%} ++ ++ ins_pipe(pipe_cmpxchg); ++%} + + // ============================================================================ + // This name is KNOWN by the ADLC and cannot be changed. +diff --git a/src/hotspot/share/adlc/formssel.cpp b/src/hotspot/share/adlc/formssel.cpp +index 2845cf6d8..c41ac52c7 100644 +--- a/src/hotspot/share/adlc/formssel.cpp ++++ b/src/hotspot/share/adlc/formssel.cpp +@@ -774,11 +774,6 @@ bool InstructForm::captures_bottom_type(FormDict &globals) const { + !strcmp(_matrule->_rChild->_opType,"CheckCastPP") || + !strcmp(_matrule->_rChild->_opType,"GetAndSetP") || + !strcmp(_matrule->_rChild->_opType,"GetAndSetN") || +-#if INCLUDE_ZGC +- !strcmp(_matrule->_rChild->_opType,"ZGetAndSetP") || +- !strcmp(_matrule->_rChild->_opType,"ZCompareAndExchangeP") || +- !strcmp(_matrule->_rChild->_opType,"LoadBarrierSlowReg") || +-#endif + !strcmp(_matrule->_rChild->_opType,"CompareAndExchangeP") || + !strcmp(_matrule->_rChild->_opType,"CompareAndExchangeN"))) return true; + else if ( is_ideal_load() == Form::idealP ) return true; +@@ -3506,9 +3501,6 @@ int MatchNode::needs_ideal_memory_edge(FormDict &globals) const { + "StoreCM", + "GetAndSetB", "GetAndSetS", "GetAndAddI", "GetAndSetI", "GetAndSetP", + "GetAndAddB", "GetAndAddS", "GetAndAddL", "GetAndSetL", "GetAndSetN", +-#if INCLUDE_ZGC +- "LoadBarrierSlowReg", "ZGetAndSetP", "ZCompareAndSwapP", "ZCompareAndExchangeP", "ZWeakCompareAndSwapP", +-#endif + "ClearArray" + }; + int cnt = sizeof(needs_ideal_memory_list)/sizeof(char*); +diff --git a/src/hotspot/share/compiler/compilerDirectives.hpp b/src/hotspot/share/compiler/compilerDirectives.hpp +index 5c9fc98e1..e689c77c5 100644 +--- a/src/hotspot/share/compiler/compilerDirectives.hpp ++++ b/src/hotspot/share/compiler/compilerDirectives.hpp +@@ -66,8 +66,7 @@ NOT_PRODUCT(cflags(TraceOptoOutput, bool, TraceOptoOutput, TraceOptoOutput)) + cflags(VectorizeDebug, uintx, 0, VectorizeDebug) \ + cflags(CloneMapDebug, bool, false, CloneMapDebug) \ + cflags(IGVPrintLevel, intx, PrintIdealGraphLevel, IGVPrintLevel) \ +- cflags(MaxNodeLimit, intx, MaxNodeLimit, MaxNodeLimit) \ +-ZGC_ONLY(cflags(ZTraceLoadBarriers, bool, false, ZTraceLoadBarriers)) ++ cflags(MaxNodeLimit, intx, MaxNodeLimit, MaxNodeLimit) + #else + #define compilerdirectives_c2_flags(cflags) + #endif +diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp +index eea74674f..487988bd8 100644 +--- a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp ++++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp +@@ -198,7 +198,7 @@ public: + virtual void clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const; + + // Support for GC barriers emitted during parsing +- virtual bool has_load_barriers() const { return false; } ++ virtual bool has_load_barrier_nodes() const { return false; } + virtual bool is_gc_barrier_node(Node* node) const { return false; } + virtual Node* step_over_gc_barrier(Node* c) const { return c; } + +@@ -213,12 +213,14 @@ public: + // This could for example comprise macro nodes to be expanded during macro expansion. + virtual void* create_barrier_state(Arena* comp_arena) const { return NULL; } + virtual void optimize_loops(PhaseIdealLoop* phase, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const { } +- virtual void barrier_insertion_phase(Compile* C, PhaseIterGVN &igvn) const { } + // If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be + // expanded later, then now is the time to do so. + virtual bool expand_macro_nodes(PhaseMacroExpand* macro) const { return false; } + virtual void verify_gc_barriers(bool post_parse) const {} +- virtual bool needs_anti_dependence_check(const Node* node) const { return true; } ++ ++ virtual void late_barrier_analysis() const { } ++ virtual int estimate_stub_size() const { return 0; } ++ virtual void emit_stubs(CodeBuffer& cb) const { } + }; + + #endif // SHARE_GC_SHARED_C2_BARRIERSETC2_HPP +diff --git a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp +index 1c77a4f63..a12973464 100644 +--- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp ++++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp +@@ -22,444 +22,156 @@ + */ + + #include "precompiled.hpp" +-#include "opto/castnode.hpp" ++#include "classfile/javaClasses.hpp" ++#include "gc/z/c2/zBarrierSetC2.hpp" ++#include "gc/z/zBarrierSet.hpp" ++#include "gc/z/zBarrierSetAssembler.hpp" ++#include "gc/z/zBarrierSetRuntime.hpp" ++#include "opto/block.hpp" + #include "opto/compile.hpp" + #include "opto/graphKit.hpp" +-#include "opto/loopnode.hpp" + #include "opto/machnode.hpp" +-#include "opto/macro.hpp" + #include "opto/memnode.hpp" +-#include "opto/movenode.hpp" + #include "opto/node.hpp" +-#include "opto/phase.hpp" +-#include "opto/phaseX.hpp" ++#include "opto/regalloc.hpp" + #include "opto/rootnode.hpp" +-#include "opto/type.hpp" +-#include "utilities/copy.hpp" + #include "utilities/growableArray.hpp" + #include "utilities/macros.hpp" +-#include "gc/z/zBarrierSet.hpp" +-#include "gc/z/c2/zBarrierSetC2.hpp" +-#include "gc/z/zThreadLocalData.hpp" +-#include "gc/z/zBarrierSetRuntime.hpp" +- +-ZBarrierSetC2State::ZBarrierSetC2State(Arena* comp_arena) +- : _load_barrier_nodes(new (comp_arena) GrowableArray(comp_arena, 8, 0, NULL)) {} +- +-int ZBarrierSetC2State::load_barrier_count() const { +- return _load_barrier_nodes->length(); +-} +- +-void ZBarrierSetC2State::add_load_barrier_node(LoadBarrierNode * n) { +- assert(!_load_barrier_nodes->contains(n), " duplicate entry in expand list"); +- _load_barrier_nodes->append(n); +-} +- +-void ZBarrierSetC2State::remove_load_barrier_node(LoadBarrierNode * n) { +- // this function may be called twice for a node so check +- // that the node is in the array before attempting to remove it +- if (_load_barrier_nodes->contains(n)) { +- _load_barrier_nodes->remove(n); +- } +-} +- +-LoadBarrierNode* ZBarrierSetC2State::load_barrier_node(int idx) const { +- return _load_barrier_nodes->at(idx); +-} + +-void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const { +- return new(comp_arena) ZBarrierSetC2State(comp_arena); +-} ++class ZBarrierSetC2State : public ResourceObj { ++private: ++ GrowableArray* _stubs; ++ Node_Array _live; + +-ZBarrierSetC2State* ZBarrierSetC2::state() const { +- return reinterpret_cast(Compile::current()->barrier_set_state()); +-} ++public: ++ ZBarrierSetC2State(Arena* arena) : ++ _stubs(new (arena) GrowableArray(arena, 8, 0, NULL)), ++ _live(arena) {} + +-bool ZBarrierSetC2::is_gc_barrier_node(Node* node) const { +- // 1. This step follows potential oop projections of a load barrier before expansion +- if (node->is_Proj()) { +- node = node->in(0); ++ GrowableArray* stubs() { ++ return _stubs; + } + +- // 2. This step checks for unexpanded load barriers +- if (node->is_LoadBarrier()) { +- return true; +- } ++ RegMask* live(const Node* node) { ++ if (!node->is_Mach()) { ++ // Don't need liveness for non-MachNodes ++ return NULL; ++ } + +- // 3. This step checks for the phi corresponding to an optimized load barrier expansion +- if (node->is_Phi()) { +- PhiNode* phi = node->as_Phi(); +- Node* n = phi->in(1); +- if (n != NULL && (n->is_LoadBarrierSlowReg())) { +- return true; ++ const MachNode* const mach = node->as_Mach(); ++ if (mach->barrier_data() != ZLoadBarrierStrong && ++ mach->barrier_data() != ZLoadBarrierWeak) { ++ // Don't need liveness data for nodes without barriers ++ return NULL; + } +- } + +- return false; +-} ++ RegMask* live = (RegMask*)_live[node->_idx]; ++ if (live == NULL) { ++ live = new (Compile::current()->comp_arena()->Amalloc_D(sizeof(RegMask))) RegMask(); ++ _live.map(node->_idx, (Node*)live); ++ } + +-void ZBarrierSetC2::register_potential_barrier_node(Node* node) const { +- if (node->is_LoadBarrier()) { +- state()->add_load_barrier_node(node->as_LoadBarrier()); ++ return live; + } +-} ++}; + +-void ZBarrierSetC2::unregister_potential_barrier_node(Node* node) const { +- if (node->is_LoadBarrier()) { +- state()->remove_load_barrier_node(node->as_LoadBarrier()); +- } ++static ZBarrierSetC2State* barrier_set_state() { ++ return reinterpret_cast(Compile::current()->barrier_set_state()); + } + +-void ZBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful) const { +- // Remove useless LoadBarrier nodes +- ZBarrierSetC2State* s = state(); +- for (int i = s->load_barrier_count()-1; i >= 0; i--) { +- LoadBarrierNode* n = s->load_barrier_node(i); +- if (!useful.member(n)) { +- unregister_potential_barrier_node(n); +- } ++ZLoadBarrierStubC2* ZLoadBarrierStubC2::create(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) { ++ ZLoadBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZLoadBarrierStubC2(node, ref_addr, ref, tmp, weak); ++ if (!Compile::current()->in_scratch_emit_size()) { ++ barrier_set_state()->stubs()->append(stub); + } +-} + +-void ZBarrierSetC2::enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const { +- if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) { +- worklist.push(node); +- } ++ return stub; + } + +-static bool load_require_barrier(LoadNode* load) { return ((load->barrier_data() & RequireBarrier) != 0); } +-static bool load_has_weak_barrier(LoadNode* load) { return ((load->barrier_data() & WeakBarrier) != 0); } +-static bool load_has_expanded_barrier(LoadNode* load) { return ((load->barrier_data() & ExpandedBarrier) != 0); } +-static void load_set_expanded_barrier(LoadNode* load) { return load->set_barrier_data(ExpandedBarrier); } +- +-static void load_set_barrier(LoadNode* load, bool weak) { +- if (weak) { +- load->set_barrier_data(WeakBarrier); +- } else { +- load->set_barrier_data(RequireBarrier); +- } ++ZLoadBarrierStubC2::ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) : ++ _node(node), ++ _ref_addr(ref_addr), ++ _ref(ref), ++ _tmp(tmp), ++ _weak(weak), ++ _entry(), ++ _continuation() { ++ assert_different_registers(ref, ref_addr.base()); ++ assert_different_registers(ref, ref_addr.index()); + } + +-// == LoadBarrierNode == +- +-LoadBarrierNode::LoadBarrierNode(Compile* C, +- Node* c, +- Node* mem, +- Node* val, +- Node* adr, +- bool weak) : +- MultiNode(Number_of_Inputs), +- _weak(weak) { +- init_req(Control, c); +- init_req(Memory, mem); +- init_req(Oop, val); +- init_req(Address, adr); +- init_req(Similar, C->top()); +- +- init_class_id(Class_LoadBarrier); +- BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); +- bs->register_potential_barrier_node(this); ++Address ZLoadBarrierStubC2::ref_addr() const { ++ return _ref_addr; + } + +-uint LoadBarrierNode::size_of() const { +- return sizeof(*this); ++Register ZLoadBarrierStubC2::ref() const { ++ return _ref; + } + +-uint LoadBarrierNode::cmp(const Node& n) const { +- ShouldNotReachHere(); +- return 0; ++Register ZLoadBarrierStubC2::tmp() const { ++ return _tmp; + } + +-const Type *LoadBarrierNode::bottom_type() const { +- const Type** floadbarrier = (const Type **)(Compile::current()->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*))); +- Node* in_oop = in(Oop); +- floadbarrier[Control] = Type::CONTROL; +- floadbarrier[Memory] = Type::MEMORY; +- floadbarrier[Oop] = in_oop == NULL ? Type::TOP : in_oop->bottom_type(); +- return TypeTuple::make(Number_of_Outputs, floadbarrier); ++address ZLoadBarrierStubC2::slow_path() const { ++ const DecoratorSet decorators = _weak ? ON_WEAK_OOP_REF : ON_STRONG_OOP_REF; ++ return ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators); + } + +-const TypePtr* LoadBarrierNode::adr_type() const { +- ShouldNotReachHere(); +- return NULL; ++RegMask& ZLoadBarrierStubC2::live() const { ++ return *barrier_set_state()->live(_node); + } + +-const Type *LoadBarrierNode::Value(PhaseGVN *phase) const { +- const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*))); +- const Type* val_t = phase->type(in(Oop)); +- floadbarrier[Control] = Type::CONTROL; +- floadbarrier[Memory] = Type::MEMORY; +- floadbarrier[Oop] = val_t; +- return TypeTuple::make(Number_of_Outputs, floadbarrier); ++Label* ZLoadBarrierStubC2::entry() { ++ // The _entry will never be bound when in_scratch_emit_size() is true. ++ // However, we still need to return a label that is not bound now, but ++ // will eventually be bound. Any lable will do, as it will only act as ++ // a placeholder, so we return the _continuation label. ++ return Compile::current()->in_scratch_emit_size() ? &_continuation : &_entry; + } + +-bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) { +- if (phase != NULL) { +- return phase->is_dominator(d, n); +- } +- +- for (int i = 0; i < 10 && n != NULL; i++) { +- n = IfNode::up_one_dom(n, linear_only); +- if (n == d) { +- return true; +- } +- } +- +- return false; ++Label* ZLoadBarrierStubC2::continuation() { ++ return &_continuation; + } + +-LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) { +- if (is_weak()) { +- // Weak barriers can't be eliminated +- return NULL; +- } +- +- Node* val = in(LoadBarrierNode::Oop); +- if (in(Similar)->is_Proj() && in(Similar)->in(0)->is_LoadBarrier()) { +- LoadBarrierNode* lb = in(Similar)->in(0)->as_LoadBarrier(); +- assert(lb->in(Address) == in(Address), ""); +- // Load barrier on Similar edge dominates so if it now has the Oop field it can replace this barrier. +- if (lb->in(Oop) == in(Oop)) { +- return lb; +- } +- // Follow chain of load barrier through Similar edges +- while (!lb->in(Similar)->is_top()) { +- lb = lb->in(Similar)->in(0)->as_LoadBarrier(); +- assert(lb->in(Address) == in(Address), ""); +- } +- if (lb != in(Similar)->in(0)) { +- return lb; +- } +- } +- for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { +- Node* u = val->fast_out(i); +- if (u != this && u->is_LoadBarrier() && u->in(Oop) == val && u->as_LoadBarrier()->has_true_uses()) { +- Node* this_ctrl = in(LoadBarrierNode::Control); +- Node* other_ctrl = u->in(LoadBarrierNode::Control); +- if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) { +- return u->as_LoadBarrier(); +- } +- } +- } +- +- if (can_be_eliminated()) { +- return NULL; +- } +- +- if (!look_for_similar) { +- return NULL; +- } +- +- Node* addr = in(LoadBarrierNode::Address); +- for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) { +- Node* u = addr->fast_out(i); +- if (u != this && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) { +- Node* this_ctrl = in(LoadBarrierNode::Control); +- Node* other_ctrl = u->in(LoadBarrierNode::Control); +- if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) { +- ResourceMark rm; +- Unique_Node_List wq; +- wq.push(in(LoadBarrierNode::Control)); +- bool ok = true; +- bool dom_found = false; +- for (uint next = 0; next < wq.size(); ++next) { +- Node *n = wq.at(next); +- if (n->is_top()) { +- return NULL; +- } +- assert(n->is_CFG(), ""); +- if (n->is_SafePoint()) { +- ok = false; +- break; +- } +- if (n == u) { +- dom_found = true; +- continue; +- } +- if (n->is_Region()) { +- for (uint i = 1; i < n->req(); i++) { +- Node* m = n->in(i); +- if (m != NULL) { +- wq.push(m); +- } +- } +- } else { +- Node* m = n->in(0); +- if (m != NULL) { +- wq.push(m); +- } +- } +- } +- if (ok) { +- assert(dom_found, ""); +- return u->as_LoadBarrier(); +- } +- break; +- } +- } +- } +- +- return NULL; +-} +- +-void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const { +- // Change to that barrier may affect a dominated barrier so re-push those +- assert(!is_weak(), "sanity"); +- Node* val = in(LoadBarrierNode::Oop); +- +- for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { +- Node* u = val->fast_out(i); +- if (u != this && u->is_LoadBarrier() && u->in(Oop) == val) { +- Node* this_ctrl = in(Control); +- Node* other_ctrl = u->in(Control); +- if (is_dominator(NULL, false, this_ctrl, other_ctrl)) { +- igvn->_worklist.push(u); +- } +- } +- +- Node* addr = in(LoadBarrierNode::Address); +- for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) { +- Node* u = addr->fast_out(i); +- if (u != this && u->is_LoadBarrier() && u->in(Similar)->is_top()) { +- Node* this_ctrl = in(Control); +- Node* other_ctrl = u->in(Control); +- if (is_dominator(NULL, false, this_ctrl, other_ctrl)) { +- igvn->_worklist.push(u); +- } +- } +- } +- } ++void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const { ++ return new (comp_arena) ZBarrierSetC2State(comp_arena); + } + +-Node *LoadBarrierNode::Identity(PhaseGVN *phase) { +- LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false); +- if (dominating_barrier != NULL) { +- assert(!is_weak(), "Weak barriers cant be eliminated"); +- assert(dominating_barrier->in(Oop) == in(Oop), ""); +- return dominating_barrier; +- } +- +- return this; ++void ZBarrierSetC2::late_barrier_analysis() const { ++ analyze_dominating_barriers(); ++ compute_liveness_at_stubs(); + } + +-Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { +- if (remove_dead_region(phase, can_reshape)) { +- return this; +- } +- +- Node *val = in(Oop); +- Node *mem = in(Memory); +- Node *ctrl = in(Control); +- assert(val->Opcode() != Op_DecodeN, ""); ++void ZBarrierSetC2::emit_stubs(CodeBuffer& cb) const { ++ MacroAssembler masm(&cb); ++ GrowableArray* const stubs = barrier_set_state()->stubs(); + +- if (mem->is_MergeMem()) { +- Node *new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw); +- set_req(Memory, new_mem); +- if (mem->outcnt() == 0 && can_reshape) { +- phase->is_IterGVN()->_worklist.push(mem); ++ for (int i = 0; i < stubs->length(); i++) { ++ // Make sure there is enough space in the code buffer ++ if (cb.insts()->maybe_expand_to_ensure_remaining(Compile::MAX_inst_size) && cb.blob() == NULL) { ++ ciEnv::current()->record_failure("CodeCache is full"); ++ return; + } +- return this; +- } + +- LoadBarrierNode *dominating_barrier = NULL; +- if (!is_weak()) { +- dominating_barrier = has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress()); +- if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) { +- assert(in(Address) == dominating_barrier->in(Address), ""); +- set_req(Similar, dominating_barrier->proj_out(Oop)); +- return this; +- } ++ ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, stubs->at(i)); + } +- +- bool eliminate = can_reshape && (dominating_barrier != NULL || !has_true_uses()); +- if (eliminate) { +- if (can_reshape) { +- PhaseIterGVN* igvn = phase->is_IterGVN(); +- Node* out_ctrl = proj_out_or_null(Control); +- Node* out_res = proj_out_or_null(Oop); +- +- if (out_ctrl != NULL) { +- igvn->replace_node(out_ctrl, ctrl); +- } +- +- // That transformation may cause the Similar edge on the load barrier to be invalid +- fix_similar_in_uses(igvn); +- if (out_res != NULL) { +- if (dominating_barrier != NULL) { +- assert(!is_weak(), "Sanity"); +- igvn->replace_node(out_res, dominating_barrier->proj_out(Oop)); +- } else { +- igvn->replace_node(out_res, val); +- } +- } +- } +- return new ConINode(TypeInt::ZERO); +- } +- +- // If the Similar edge is no longer a load barrier, clear it +- Node* similar = in(Similar); +- if (!similar->is_top() && !(similar->is_Proj() && similar->in(0)->is_LoadBarrier())) { +- set_req(Similar, phase->C->top()); +- return this; +- } +- +- if (can_reshape && !is_weak()) { +- // If this barrier is linked through the Similar edge by a +- // dominated barrier and both barriers have the same Oop field, +- // the dominated barrier can go away, so push it for reprocessing. +- // We also want to avoid a barrier to depend on another dominating +- // barrier through its Similar edge that itself depend on another +- // barrier through its Similar edge and rather have the first +- // depend on the third. +- PhaseIterGVN* igvn = phase->is_IterGVN(); +- Node* out_res = proj_out(Oop); +- for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { +- Node* u = out_res->fast_out(i); +- if (u->is_LoadBarrier() && u->in(Similar) == out_res && +- (u->in(Oop) == val || !u->in(Similar)->is_top())) { +- assert(!u->as_LoadBarrier()->is_weak(), "Sanity"); +- igvn->_worklist.push(u); +- } +- } +- +- push_dominated_barriers(igvn); +- } +- +- return NULL; ++ masm.flush(); + } + +-uint LoadBarrierNode::match_edge(uint idx) const { +- ShouldNotReachHere(); +- return 0; +-} ++int ZBarrierSetC2::estimate_stub_size() const { ++ Compile* const C = Compile::current(); ++ BufferBlob* const blob = C->scratch_buffer_blob(); ++ GrowableArray* const stubs = barrier_set_state()->stubs(); ++ int size = 0; + +-void LoadBarrierNode::fix_similar_in_uses(PhaseIterGVN* igvn) { +- Node* out_res = proj_out_or_null(Oop); +- if (out_res == NULL) { +- return; ++ for (int i = 0; i < stubs->length(); i++) { ++ CodeBuffer cb(blob->content_begin(), (address)C->scratch_locs_memory() - blob->content_begin()); ++ MacroAssembler masm(&cb); ++ ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, stubs->at(i)); ++ size += cb.insts_size(); + } + +- for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { +- Node* u = out_res->fast_out(i); +- if (u->is_LoadBarrier() && u->in(Similar) == out_res) { +- igvn->replace_input_of(u, Similar, igvn->C->top()); +- --i; +- --imax; +- } +- } +-} +- +-bool LoadBarrierNode::has_true_uses() const { +- Node* out_res = proj_out_or_null(Oop); +- if (out_res != NULL) { +- for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { +- Node *u = out_res->fast_out(i); +- if (!u->is_LoadBarrier() || u->in(Similar) != out_res) { +- return true; +- } +- } +- } +- return false; ++ return size; + } + + static bool barrier_needed(C2Access access) { +@@ -467,1071 +179,253 @@ static bool barrier_needed(C2Access access) { + } + + Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { +- Node* p = BarrierSetC2::load_at_resolved(access, val_type); +- if (!barrier_needed(access)) { +- return p; ++ Node* result = BarrierSetC2::load_at_resolved(access, val_type); ++ if (barrier_needed(access) && access.raw_access()->is_Mem()) { ++ if ((access.decorators() & ON_WEAK_OOP_REF) != 0) { ++ access.raw_access()->as_Load()->set_barrier_data(ZLoadBarrierWeak); ++ } else { ++ access.raw_access()->as_Load()->set_barrier_data(ZLoadBarrierStrong); ++ } + } + +- bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0; +- if (p->isa_Load()) { +- load_set_barrier(p->as_Load(), weak); +- } +- return p; ++ return result; + } + + Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val, + Node* new_val, const Type* val_type) const { + Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type); +- LoadStoreNode* lsn = result->as_LoadStore(); + if (barrier_needed(access)) { +- lsn->set_has_barrier(); ++ access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong); + } +- return lsn; ++ return result; + } + + Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val, + Node* new_val, const Type* value_type) const { + Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); +- LoadStoreNode* lsn = result->as_LoadStore(); + if (barrier_needed(access)) { +- lsn->set_has_barrier(); ++ access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong); + } +- return lsn; ++ return result; + + } + + Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const { + Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type); +- LoadStoreNode* lsn = result->as_LoadStore(); + if (barrier_needed(access)) { +- lsn->set_has_barrier(); ++ access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong); + } +- return lsn; ++ return result; + } + +-// == Macro Expansion == +- +-// Optimized, low spill, loadbarrier variant using stub specialized on register used +-void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const { +- PhaseIterGVN &igvn = phase->igvn(); +- float unlikely = PROB_UNLIKELY(0.999); +- +- Node* in_ctrl = barrier->in(LoadBarrierNode::Control); +- Node* in_mem = barrier->in(LoadBarrierNode::Memory); +- Node* in_val = barrier->in(LoadBarrierNode::Oop); +- Node* in_adr = barrier->in(LoadBarrierNode::Address); +- +- Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control); +- Node* out_res = barrier->proj_out(LoadBarrierNode::Oop); +- +- assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null"); +- +- Node* jthread = igvn.transform(new ThreadLocalNode()); +- Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset())); +- Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, +- TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), +- MemNode::unordered)); +- Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val)); +- Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask)); +- Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type()))); +- Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool(); +- IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If(); +- Node* then = igvn.transform(new IfTrueNode(iff)); +- Node* elsen = igvn.transform(new IfFalseNode(iff)); +- +- Node* new_loadp = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(), +- (const TypePtr*) in_val->bottom_type(), MemNode::unordered, barrier->is_weak())); +- +- // Create the final region/phi pair to converge cntl/data paths to downstream code +- Node* result_region = igvn.transform(new RegionNode(3)); +- result_region->set_req(1, then); +- result_region->set_req(2, elsen); +- +- Node* result_phi = igvn.transform(new PhiNode(result_region, TypeInstPtr::BOTTOM)); +- result_phi->set_req(1, new_loadp); +- result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop)); +- +- igvn.replace_node(out_ctrl, result_region); +- igvn.replace_node(out_res, result_phi); +- +- assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!"); +- +- igvn.remove_dead_node(barrier); +- igvn.remove_dead_node(out_ctrl); +- igvn.remove_dead_node(out_res); +- assert(is_gc_barrier_node(result_phi), "sanity"); +- assert(step_over_gc_barrier(result_phi) == in_val, "sanity"); +- phase->C->print_method(PHASE_BEFORE_MACRO_EXPANSION, 4, barrier->_idx); +- return; +-} +- +-bool ZBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const { +- Compile* C = Compile::current(); +- PhaseIterGVN &igvn = macro->igvn(); +- ZBarrierSetC2State* s = state(); +- if (s->load_barrier_count() > 0) { +-#ifdef ASSERT +- verify_gc_barriers(false); +-#endif +- igvn.set_delay_transform(true); +- int skipped = 0; +- while (s->load_barrier_count() > skipped) { +- int load_barrier_count = s->load_barrier_count(); +- LoadBarrierNode * n = s->load_barrier_node(load_barrier_count-1-skipped); +- if (igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) { +- // Node is unreachable, so don't try to expand it +- s->remove_load_barrier_node(n); +- continue; +- } +- if (!n->can_be_eliminated()) { +- skipped++; +- continue; +- } +- expand_loadbarrier_node(macro, n); +- assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list"); +- if (C->failing()) return true; +- } +- while (s->load_barrier_count() > 0) { +- int load_barrier_count = s->load_barrier_count(); +- LoadBarrierNode* n = s->load_barrier_node(load_barrier_count - 1); +- assert(!(igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())), "should have been processed already"); +- assert(!n->can_be_eliminated(), "should have been processed already"); +- expand_loadbarrier_node(macro, n); +- assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list"); +- if (C->failing()) return true; +- } +- igvn.set_delay_transform(false); +- igvn.optimize(); +- if (C->failing()) return true; +- } +- return false; ++bool ZBarrierSetC2::array_copy_requires_gc_barriers(BasicType type) const { ++ return type == T_OBJECT || type == T_ARRAY; + } + +-Node* ZBarrierSetC2::step_over_gc_barrier(Node* c) const { +- Node* node = c; +- +- // 1. This step follows potential oop projections of a load barrier before expansion +- if (node->is_Proj()) { +- node = node->in(0); +- } +- +- // 2. This step checks for unexpanded load barriers +- if (node->is_LoadBarrier()) { +- return node->in(LoadBarrierNode::Oop); +- } ++// == Dominating barrier elision == + +- // 3. This step checks for the phi corresponding to an optimized load barrier expansion +- if (node->is_Phi()) { +- PhiNode* phi = node->as_Phi(); +- Node* n = phi->in(1); +- if (n != NULL && (n->is_LoadBarrierSlowReg())) { +- assert(c == node, "projections from step 1 should only be seen before macro expansion"); +- return phi->in(2); ++static bool block_has_safepoint(const Block* block, uint from, uint to) { ++ for (uint i = from; i < to; i++) { ++ if (block->get_node(i)->is_MachSafePoint()) { ++ // Safepoint found ++ return true; + } + } + +- return c; ++ // Safepoint not found ++ return false; + } + +-// == Verification == +- +-#ifdef ASSERT +- +-static bool look_for_barrier(Node* n, bool post_parse, VectorSet& visited) { +- if (visited.test_set(n->_idx)) { +- return true; +- } ++static bool block_has_safepoint(const Block* block) { ++ return block_has_safepoint(block, 0, block->number_of_nodes()); ++} + +- for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { +- Node* u = n->fast_out(i); +- if (u->is_LoadBarrier()) { +- } else if ((u->is_Phi() || u->is_CMove()) && !post_parse) { +- if (!look_for_barrier(u, post_parse, visited)) { +- return false; +- } +- } else if (u->Opcode() == Op_EncodeP || u->Opcode() == Op_DecodeN) { +- if (!look_for_barrier(u, post_parse, visited)) { +- return false; +- } +- } else if (u->Opcode() != Op_SCMemProj) { +- tty->print("bad use"); u->dump(); +- return false; ++static uint block_index(const Block* block, const Node* node) { ++ for (uint j = 0; j < block->number_of_nodes(); ++j) { ++ if (block->get_node(j) == node) { ++ return j; + } + } +- +- return true; ++ ShouldNotReachHere(); ++ return 0; + } + +-void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const { +- ZBarrierSetC2State* s = state(); +- Compile* C = Compile::current(); ++void ZBarrierSetC2::analyze_dominating_barriers() const { + ResourceMark rm; +- VectorSet visited(Thread::current()->resource_area()); +- for (int i = 0; i < s->load_barrier_count(); i++) { +- LoadBarrierNode* n = s->load_barrier_node(i); +- +- // The dominating barrier on the same address if it exists and +- // this barrier must not be applied on the value from the same +- // load otherwise the value is not reloaded before it's used the +- // second time. +- assert(n->in(LoadBarrierNode::Similar)->is_top() || +- (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() && +- n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Address) == n->in(LoadBarrierNode::Address) && +- n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Oop) != n->in(LoadBarrierNode::Oop)), +- "broken similar edge"); +- +- assert(post_parse || n->as_LoadBarrier()->has_true_uses(), +- "found unneeded load barrier"); +- +- // Several load barrier nodes chained through their Similar edge +- // break the code that remove the barriers in final graph reshape. +- assert(n->in(LoadBarrierNode::Similar)->is_top() || +- (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() && +- n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Similar)->is_top()), +- "chain of Similar load barriers"); +- +- if (!n->in(LoadBarrierNode::Similar)->is_top()) { +- ResourceMark rm; +- Unique_Node_List wq; +- Node* other = n->in(LoadBarrierNode::Similar)->in(0); +- wq.push(n); +- bool ok = true; +- bool dom_found = false; +- for (uint next = 0; next < wq.size(); ++next) { +- Node *n = wq.at(next); +- assert(n->is_CFG(), ""); +- assert(!n->is_SafePoint(), ""); +- +- if (n == other) { +- continue; +- } +- +- if (n->is_Region()) { +- for (uint i = 1; i < n->req(); i++) { +- Node* m = n->in(i); +- if (m != NULL) { +- wq.push(m); +- } +- } +- } else { +- Node* m = n->in(0); +- if (m != NULL) { +- wq.push(m); +- } +- } ++ Compile* const C = Compile::current(); ++ PhaseCFG* const cfg = C->cfg(); ++ Block_List worklist; ++ Node_List mem_ops; ++ Node_List barrier_loads; ++ ++ // Step 1 - Find accesses, and track them in lists ++ for (uint i = 0; i < cfg->number_of_blocks(); ++i) { ++ const Block* const block = cfg->get_block(i); ++ for (uint j = 0; j < block->number_of_nodes(); ++j) { ++ const Node* const node = block->get_node(j); ++ if (!node->is_Mach()) { ++ continue; + } +- } + +- if (ZVerifyLoadBarriers) { +- if ((n->is_Load() || n->is_LoadStore()) && n->bottom_type()->make_oopptr() != NULL) { +- visited.Clear(); +- bool found = look_for_barrier(n, post_parse, visited); +- if (!found) { +- n->dump(1); +- n->dump(-3); +- stringStream ss; +- C->method()->print_short_name(&ss); +- tty->print_cr("-%s-", ss.as_string()); +- assert(found, ""); ++ MachNode* const mach = node->as_Mach(); ++ switch (mach->ideal_Opcode()) { ++ case Op_LoadP: ++ case Op_CompareAndExchangeP: ++ case Op_CompareAndSwapP: ++ case Op_GetAndSetP: ++ if (mach->barrier_data() == ZLoadBarrierStrong) { ++ barrier_loads.push(mach); + } +- } +- } +- } +-} +- +-#endif +- +-static void call_catch_cleanup_one(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl); +- +-// This code is cloning all uses of a load that is between a call and the catch blocks, +-// to each use. +- +-static bool fixup_uses_in_catch(PhaseIdealLoop *phase, Node *start_ctrl, Node *node) { +- +- if (!phase->has_ctrl(node)) { +- // This node is floating - doesn't need to be cloned. +- assert(node != start_ctrl, "check"); +- return false; +- } +- +- Node* ctrl = phase->get_ctrl(node); +- if (ctrl != start_ctrl) { +- // We are in a successor block - the node is ok. +- return false; // Unwind +- } +- +- // Process successor nodes +- int outcnt = node->outcnt(); +- for (int i = 0; i < outcnt; i++) { +- Node* n = node->raw_out(0); +- assert(!n->is_LoadBarrier(), "Sanity"); +- // Calling recursively, visiting leafs first +- fixup_uses_in_catch(phase, start_ctrl, n); +- } +- +- // Now all successors are outside +- // - Clone this node to both successors +- int no_succs = node->outcnt(); +- assert(!node->is_Store(), "Stores not expected here"); +- +- // In some very rare cases a load that doesn't need a barrier will end up here +- // Treat it as a LoadP and the insertion of phis will be done correctly. +- if (node->is_Load()) { +- assert(node->as_Load()->barrier_data() == 0, "Sanity"); +- call_catch_cleanup_one(phase, node->as_Load(), phase->get_ctrl(node)); +- } else { +- for (DUIterator_Fast jmax, i = node->fast_outs(jmax); i < jmax; i++) { +- Node* use = node->fast_out(i); +- Node* clone = node->clone(); +- assert(clone->outcnt() == 0, ""); +- +- assert(use->find_edge(node) != -1, "check"); +- phase->igvn().rehash_node_delayed(use); +- use->replace_edge(node, clone); +- +- Node* new_ctrl; +- if (use->is_block_start()) { +- new_ctrl = use; +- } else if (use->is_CFG()) { +- new_ctrl = use->in(0); +- assert (new_ctrl != NULL, ""); +- } else { +- new_ctrl = phase->get_ctrl(use); +- } +- +- phase->set_ctrl(clone, new_ctrl); +- +- if (phase->C->directive()->ZTraceLoadBarriersOption) tty->print_cr(" Clone op %i as %i to control %i", node->_idx, clone->_idx, new_ctrl->_idx); +- phase->igvn().register_new_node_with_optimizer(clone); +- --i, --jmax; +- } +- assert(node->outcnt() == 0, "must be empty now"); +- +- // Node node is dead. +- phase->igvn().remove_dead_node(node); +- } +- return true; // unwind - return if a use was processed +-} +- +-// Clone a load to a specific catch_proj +-static Node* clone_load_to_catchproj(PhaseIdealLoop* phase, Node* load, Node* catch_proj) { +- Node* cloned_load = load->clone(); +- cloned_load->set_req(0, catch_proj); // set explicit control +- phase->set_ctrl(cloned_load, catch_proj); // update +- if (phase->C->directive()->ZTraceLoadBarriersOption) tty->print_cr(" Clone LOAD %i as %i to control %i", load->_idx, cloned_load->_idx, catch_proj->_idx); +- phase->igvn().register_new_node_with_optimizer(cloned_load); +- return cloned_load; +-} +- +-static Node* get_dominating_region(PhaseIdealLoop* phase, Node* node, Node* stop) { +- Node* region = node; +- while (!region->isa_Region()) { +- Node *up = phase->idom(region); +- assert(up != region, "Must not loop"); +- assert(up != stop, "Must not find original control"); +- region = up; +- } +- return region; +-} +- +-// Clone this load to each catch block +-static void call_catch_cleanup_one(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl) { +- bool trace = phase->C->directive()->ZTraceLoadBarriersOption; +- phase->igvn().set_delay_transform(true); +- +- // Verify pre conditions +- assert(ctrl->isa_Proj() && ctrl->in(0)->isa_Call(), "Must be a call proj"); +- assert(ctrl->raw_out(0)->isa_Catch(), "Must be a catch"); +- +- if (ctrl->raw_out(0)->isa_Catch()->outcnt() == 1) { +- if (trace) tty->print_cr("Cleaning up catch: Skipping load %i, call with single catch", load->_idx); +- return; +- } ++ case Op_StoreP: ++ mem_ops.push(mach); ++ break; + +- // Process the loads successor nodes - if any is between +- // the call and the catch blocks, they need to be cloned to. +- // This is done recursively +- int outcnt = load->outcnt(); +- uint index = 0; +- for (int i = 0; i < outcnt; i++) { +- if (index < load->outcnt()) { +- Node *n = load->raw_out(index); +- assert(!n->is_LoadBarrier(), "Sanity"); +- if (!fixup_uses_in_catch(phase, ctrl, n)) { +- // if no successor was cloned, progress to next out. +- index++; ++ default: ++ break; + } + } + } + +- // Now all the loads uses has been cloned down +- // Only thing left is to clone the loads, but they must end up +- // first in the catch blocks. +- +- // We clone the loads oo the catch blocks only when needed. +- // An array is used to map the catch blocks to each lazily cloned load. +- // In that way no extra unnecessary loads are cloned. +- +- // Any use dominated by original block must have an phi and a region added ++ // Step 2 - Find dominating accesses for each load ++ for (uint i = 0; i < barrier_loads.size(); i++) { ++ MachNode* const load = barrier_loads.at(i)->as_Mach(); ++ const TypePtr* load_adr_type = NULL; ++ intptr_t load_offset = 0; ++ const Node* const load_obj = load->get_base_and_disp(load_offset, load_adr_type); ++ Block* const load_block = cfg->get_block_for_node(load); ++ const uint load_index = block_index(load_block, load); + +- Node* catch_node = ctrl->raw_out(0); +- int number_of_catch_projs = catch_node->outcnt(); +- Node** proj_to_load_mapping = NEW_RESOURCE_ARRAY(Node*, number_of_catch_projs); +- Copy::zero_to_bytes(proj_to_load_mapping, sizeof(Node*) * number_of_catch_projs); ++ for (uint j = 0; j < mem_ops.size(); j++) { ++ MachNode* mem = mem_ops.at(j)->as_Mach(); ++ const TypePtr* mem_adr_type = NULL; ++ intptr_t mem_offset = 0; ++ const Node* mem_obj = mem_obj = mem->get_base_and_disp(mem_offset, mem_adr_type); ++ Block* mem_block = cfg->get_block_for_node(mem); ++ uint mem_index = block_index(mem_block, mem); + +- // The phi_map is used to keep track of where phis have already been inserted +- int phi_map_len = phase->C->unique(); +- Node** phi_map = NEW_RESOURCE_ARRAY(Node*, phi_map_len); +- Copy::zero_to_bytes(phi_map, sizeof(Node*) * phi_map_len); +- +- for (unsigned int i = 0; i < load->outcnt(); i++) { +- Node* load_use_control = NULL; +- Node* load_use = load->raw_out(i); +- +- if (phase->has_ctrl(load_use)) { +- load_use_control = phase->get_ctrl(load_use); +- } else { +- load_use_control = load_use->in(0); +- } +- assert(load_use_control != NULL, "sanity"); +- if (trace) tty->print_cr(" Handling use: %i, with control: %i", load_use->_idx, load_use_control->_idx); +- +- // Some times the loads use is a phi. For them we need to determine from which catch block +- // the use is defined. +- bool load_use_is_phi = false; +- unsigned int load_use_phi_index = 0; +- Node* phi_ctrl = NULL; +- if (load_use->is_Phi()) { +- // Find phi input that matches load +- for (unsigned int u = 1; u < load_use->req(); u++) { +- if (load_use->in(u) == load) { +- load_use_is_phi = true; +- load_use_phi_index = u; +- assert(load_use->in(0)->is_Region(), "Region or broken"); +- phi_ctrl = load_use->in(0)->in(u); +- assert(phi_ctrl->is_CFG(), "check"); +- assert(phi_ctrl != load, "check"); +- break; +- } +- } +- assert(load_use_is_phi, "must find"); +- assert(load_use_phi_index > 0, "sanity"); +- } +- +- // For each load use, see which catch projs dominates, create load clone lazily and reconnect +- bool found_dominating_catchproj = false; +- for (int c = 0; c < number_of_catch_projs; c++) { +- Node* catchproj = catch_node->raw_out(c); +- assert(catchproj != NULL && catchproj->isa_CatchProj(), "Sanity"); +- +- if (!phase->is_dominator(catchproj, load_use_control)) { +- if (load_use_is_phi && phase->is_dominator(catchproj, phi_ctrl)) { +- // The loads use is local to the catchproj. +- // fall out and replace load with catch-local load clone. +- } else { +- continue; +- } +- } +- assert(!found_dominating_catchproj, "Max one should match"); +- +- // Clone loads to catch projs +- Node* load_clone = proj_to_load_mapping[c]; +- if (load_clone == NULL) { +- load_clone = clone_load_to_catchproj(phase, load, catchproj); +- proj_to_load_mapping[c] = load_clone; +- } +- phase->igvn().rehash_node_delayed(load_use); +- +- if (load_use_is_phi) { +- // phis are special - the load is defined from a specific control flow +- load_use->set_req(load_use_phi_index, load_clone); +- } else { +- // Multipe edges can be replaced at once - on calls for example +- load_use->replace_edge(load, load_clone); ++ if (load_obj == NodeSentinel || mem_obj == NodeSentinel || ++ load_obj == NULL || mem_obj == NULL || ++ load_offset < 0 || mem_offset < 0) { ++ continue; + } +- --i; // more than one edge can have been removed, but the next is in later iterations +- +- // We could break the for-loop after finding a dominating match. +- // But keep iterating to catch any bad idom early. +- found_dominating_catchproj = true; +- } + +- // We found no single catchproj that dominated the use - The use is at a point after +- // where control flow from multiple catch projs have merged. We will have to create +- // phi nodes before the use and tie the output from the cloned loads together. It +- // can be a single phi or a number of chained phis, depending on control flow +- if (!found_dominating_catchproj) { +- +- // Use phi-control if use is a phi +- if (load_use_is_phi) { +- load_use_control = phi_ctrl; ++ if (mem_obj != load_obj || mem_offset != load_offset) { ++ // Not the same addresses, not a candidate ++ continue; + } +- assert(phase->is_dominator(ctrl, load_use_control), "Common use but no dominator"); + +- // Clone a load on all paths +- for (int c = 0; c < number_of_catch_projs; c++) { +- Node* catchproj = catch_node->raw_out(c); +- Node* load_clone = proj_to_load_mapping[c]; +- if (load_clone == NULL) { +- load_clone = clone_load_to_catchproj(phase, load, catchproj); +- proj_to_load_mapping[c] = load_clone; ++ if (load_block == mem_block) { ++ // Earlier accesses in the same block ++ if (mem_index < load_index && !block_has_safepoint(mem_block, mem_index + 1, load_index)) { ++ load->set_barrier_data(ZLoadBarrierElided); + } +- } +- +- // Move up dominator tree from use until dom front is reached +- Node* next_region = get_dominating_region(phase, load_use_control, ctrl); +- while (phase->idom(next_region) != catch_node) { +- next_region = phase->idom(next_region); +- if (trace) tty->print_cr("Moving up idom to region ctrl %i", next_region->_idx); +- } +- assert(phase->is_dominator(catch_node, next_region), "Sanity"); +- +- // Create or reuse phi node that collect all cloned loads and feed it to the use. +- Node* test_phi = phi_map[next_region->_idx]; +- if ((test_phi != NULL) && test_phi->is_Phi()) { +- // Reuse an already created phi +- if (trace) tty->print_cr(" Using cached Phi %i on load_use %i", test_phi->_idx, load_use->_idx); +- phase->igvn().rehash_node_delayed(load_use); +- load_use->replace_edge(load, test_phi); +- // Now this use is done +- } else { +- // Otherwise we need to create one or more phis +- PhiNode* next_phi = new PhiNode(next_region, load->type()); +- phi_map[next_region->_idx] = next_phi; // cache new phi +- phase->igvn().rehash_node_delayed(load_use); +- load_use->replace_edge(load, next_phi); +- +- int dominators_of_region = 0; +- do { +- // New phi, connect to region and add all loads as in. +- Node* region = next_region; +- assert(region->isa_Region() && region->req() > 2, "Catch dead region nodes"); +- PhiNode* new_phi = next_phi; +- +- if (trace) tty->print_cr("Created Phi %i on load %i with control %i", new_phi->_idx, load->_idx, region->_idx); +- +- // Need to add all cloned loads to the phi, taking care that the right path is matched +- dominators_of_region = 0; // reset for new region +- for (unsigned int reg_i = 1; reg_i < region->req(); reg_i++) { +- Node* region_pred = region->in(reg_i); +- assert(region_pred->is_CFG(), "check"); +- bool pred_has_dominator = false; +- for (int c = 0; c < number_of_catch_projs; c++) { +- Node* catchproj = catch_node->raw_out(c); +- if (phase->is_dominator(catchproj, region_pred)) { +- new_phi->set_req(reg_i, proj_to_load_mapping[c]); +- if (trace) tty->print_cr(" - Phi in(%i) set to load %i", reg_i, proj_to_load_mapping[c]->_idx); +- pred_has_dominator = true; +- dominators_of_region++; +- break; +- } +- } +- +- // Sometimes we need to chain several phis. +- if (!pred_has_dominator) { +- assert(dominators_of_region <= 1, "More than one region can't require extra phi"); +- if (trace) tty->print_cr(" - Region %i pred %i not dominated by catch proj", region->_idx, region_pred->_idx); +- // Continue search on on this region_pred +- // - walk up to next region +- // - create a new phi and connect to first new_phi +- next_region = get_dominating_region(phase, region_pred, ctrl); +- +- // Lookup if there already is a phi, create a new otherwise +- Node* test_phi = phi_map[next_region->_idx]; +- if ((test_phi != NULL) && test_phi->is_Phi()) { +- next_phi = test_phi->isa_Phi(); +- dominators_of_region++; // record that a match was found and that we are done +- if (trace) tty->print_cr(" Using cached phi Phi %i on control %i", next_phi->_idx, next_region->_idx); +- } else { +- next_phi = new PhiNode(next_region, load->type()); +- phi_map[next_region->_idx] = next_phi; +- } +- new_phi->set_req(reg_i, next_phi); +- } ++ } else if (mem_block->dominates(load_block)) { ++ // Dominating block? Look around for safepoints ++ ResourceMark rm; ++ Block_List stack; ++ VectorSet visited(Thread::current()->resource_area()); ++ stack.push(load_block); ++ bool safepoint_found = block_has_safepoint(load_block); ++ while (!safepoint_found && stack.size() > 0) { ++ Block* block = stack.pop(); ++ if (visited.test_set(block->_pre_order)) { ++ continue; ++ } ++ if (block_has_safepoint(block)) { ++ safepoint_found = true; ++ break; ++ } ++ if (block == mem_block) { ++ continue; + } + +- new_phi->set_req(0, region); +- phase->igvn().register_new_node_with_optimizer(new_phi); +- phase->set_ctrl(new_phi, region); +- +- assert(dominators_of_region != 0, "Must have found one this iteration"); +- } while (dominators_of_region == 1); +- } +- --i; +- } +- } // end of loop over uses +- +- assert(load->outcnt() == 0, "All uses should be handled"); +- phase->igvn().remove_dead_node(load); +- phase->C->print_method(PHASE_CALL_CATCH_CLEANUP, 4, load->_idx); +- +- // Now we should be home +- phase->igvn().set_delay_transform(false); +-} +- +-// Sort out the loads that are between a call ant its catch blocks +-static void process_catch_cleanup_candidate(PhaseIdealLoop* phase, LoadNode* load) { +- bool trace = phase->C->directive()->ZTraceLoadBarriersOption; +- +- Node* ctrl = phase->get_ctrl(load); +- if (!ctrl->is_Proj() || (ctrl->in(0) == NULL) || !ctrl->in(0)->isa_Call()) { +- return; +- } +- +- Node* catch_node = ctrl->isa_Proj()->raw_out(0); +- if (catch_node->is_Catch()) { +- if (catch_node->outcnt() > 1) { +- call_catch_cleanup_one(phase, load, ctrl); +- } else { +- if (trace) tty->print_cr("Call catch cleanup with only one catch: load %i ", load->_idx); +- } +- } +-} +- +-void ZBarrierSetC2::barrier_insertion_phase(Compile* C, PhaseIterGVN& igvn) const { +- ResourceMark rm; +- PhaseIdealLoop v(igvn, false, false, false, true); +- if (C->failing()) return; +-} +- +-void ZBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const { +- // First make sure all loads between call and catch are moved to the catch block +- clean_catch_blocks(phase); +- +- // Then expand barriers on all loads +- insert_load_barriers(phase); +- +- // Handle all Unsafe that need barriers. +- insert_barriers_on_unsafe(phase); +- +- phase->C->clear_major_progress(); +-} +- +-static bool can_simplify_cas(LoadStoreNode* node) { +- if (node->isa_LoadStoreConditional()) { +- Node *expected_in = node->as_LoadStoreConditional()->in(LoadStoreConditionalNode::ExpectedIn); +- return (expected_in->get_ptr_type() == TypePtr::NULL_PTR); +- } else { +- return false; +- } +-} +- +-static void insert_barrier_before_unsafe(PhaseIdealLoop* phase, LoadStoreNode* old_node) { +- +- Compile *C = phase->C; +- PhaseIterGVN &igvn = phase->igvn(); +- LoadStoreNode* zclone = NULL; +- bool is_weak = false; +- +- Node *in_ctrl = old_node->in(MemNode::Control); +- Node *in_mem = old_node->in(MemNode::Memory); +- Node *in_adr = old_node->in(MemNode::Address); +- Node *in_val = old_node->in(MemNode::ValueIn); +- const TypePtr *adr_type = old_node->adr_type(); +- const TypePtr* load_type = TypeOopPtr::BOTTOM; // The type for the load we are adding +- +- switch (old_node->Opcode()) { +- case Op_CompareAndExchangeP: { +- zclone = new ZCompareAndExchangePNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn), +- adr_type, old_node->get_ptr_type(), ((CompareAndExchangeNode*)old_node)->order()); +- load_type = old_node->bottom_type()->is_ptr(); +- break; +- } +- case Op_WeakCompareAndSwapP: { +- if (can_simplify_cas(old_node)) { +- break; +- } +- is_weak = true; +- zclone = new ZWeakCompareAndSwapPNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn), +- ((CompareAndSwapNode*)old_node)->order()); +- adr_type = TypePtr::BOTTOM; +- break; +- } +- case Op_CompareAndSwapP: { +- if (can_simplify_cas(old_node)) { +- break; +- } +- zclone = new ZCompareAndSwapPNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn), +- ((CompareAndSwapNode*)old_node)->order()); +- adr_type = TypePtr::BOTTOM; +- break; +- } +- case Op_GetAndSetP: { +- zclone = new ZGetAndSetPNode(in_ctrl, in_mem, in_adr, in_val, old_node->adr_type(), old_node->get_ptr_type()); +- load_type = old_node->bottom_type()->is_ptr(); +- break; +- } +- } +- if (zclone != NULL) { +- igvn.register_new_node_with_optimizer(zclone, old_node); +- +- // Make load +- LoadPNode *load = new LoadPNode(NULL, in_mem, in_adr, adr_type, load_type, MemNode::unordered, +- LoadNode::DependsOnlyOnTest); +- load_set_expanded_barrier(load); +- igvn.register_new_node_with_optimizer(load); +- igvn.replace_node(old_node, zclone); +- +- Node *barrier = new LoadBarrierNode(C, NULL, in_mem, load, in_adr, is_weak); +- Node *barrier_val = new ProjNode(barrier, LoadBarrierNode::Oop); +- Node *barrier_ctrl = new ProjNode(barrier, LoadBarrierNode::Control); +- +- igvn.register_new_node_with_optimizer(barrier); +- igvn.register_new_node_with_optimizer(barrier_val); +- igvn.register_new_node_with_optimizer(barrier_ctrl); +- +- // loop over all of in_ctrl usages and move to barrier_ctrl +- for (DUIterator_Last imin, i = in_ctrl->last_outs(imin); i >= imin; --i) { +- Node *use = in_ctrl->last_out(i); +- uint l; +- for (l = 0; use->in(l) != in_ctrl; l++) {} +- igvn.replace_input_of(use, l, barrier_ctrl); +- } +- +- load->set_req(MemNode::Control, in_ctrl); +- barrier->set_req(LoadBarrierNode::Control, in_ctrl); +- zclone->add_req(barrier_val); // add req as keep alive. +- +- C->print_method(PHASE_ADD_UNSAFE_BARRIER, 4, zclone->_idx); +- } +-} +- +-void ZBarrierSetC2::insert_barriers_on_unsafe(PhaseIdealLoop* phase) const { +- Compile *C = phase->C; +- PhaseIterGVN &igvn = phase->igvn(); +- uint new_ids = C->unique(); +- VectorSet visited(Thread::current()->resource_area()); +- GrowableArray nodeStack(Thread::current()->resource_area(), 0, 0, NULL); +- nodeStack.push(C->root()); +- visited.test_set(C->root()->_idx); +- +- // Traverse all nodes, visit all unsafe ops that require a barrier +- while (nodeStack.length() > 0) { +- Node *n = nodeStack.pop(); +- +- bool is_old_node = (n->_idx < new_ids); // don't process nodes that were created during cleanup +- if (is_old_node) { +- if (n->is_LoadStore()) { +- LoadStoreNode* lsn = n->as_LoadStore(); +- if (lsn->has_barrier()) { +- BasicType bt = lsn->in(MemNode::Address)->bottom_type()->basic_type(); +- assert ((bt == T_OBJECT || bt == T_ARRAY), "Sanity test"); +- insert_barrier_before_unsafe(phase, lsn); +- } +- } +- } +- for (uint i = 0; i < n->len(); i++) { +- if (n->in(i)) { +- if (!visited.test_set(n->in(i)->_idx)) { +- nodeStack.push(n->in(i)); ++ // Push predecessor blocks ++ for (uint p = 1; p < block->num_preds(); ++p) { ++ Block* pred = cfg->get_block_for_node(block->pred(p)); ++ stack.push(pred); ++ } + } +- } +- } +- } +- +- igvn.optimize(); +- C->print_method(PHASE_ADD_UNSAFE_BARRIER, 2); +-} + +- +-// The purpose of ZBarrierSetC2::clean_catch_blocks is to prepare the IR for +-// splicing in load barrier nodes. +-// +-// The problem is that we might have instructions between a call and its catch nodes. +-// (This is usually handled in PhaseCFG:call_catch_cleanup, which clones mach nodes in +-// already scheduled blocks.) We can't have loads that require barriers there, +-// because we need to splice in new control flow, and that would violate the IR. +-// +-// clean_catch_blocks find all Loads that require a barrier and clone them and any +-// dependent instructions to each use. The loads must be in the beginning of the catch block +-// before any store. +-// +-// Sometimes the loads use will be at a place dominated by all catch blocks, then we need +-// a load in each catch block, and a Phi at the dominated use. +- +-void ZBarrierSetC2::clean_catch_blocks(PhaseIdealLoop* phase) const { +- +- Compile *C = phase->C; +- uint new_ids = C->unique(); +- PhaseIterGVN &igvn = phase->igvn(); +- VectorSet visited(Thread::current()->resource_area()); +- GrowableArray nodeStack(Thread::current()->resource_area(), 0, 0, NULL); +- nodeStack.push(C->root()); +- visited.test_set(C->root()->_idx); +- +- // Traverse all nodes, visit all loads that require a barrier +- while(nodeStack.length() > 0) { +- Node *n = nodeStack.pop(); +- +- for (uint i = 0; i < n->len(); i++) { +- if (n->in(i)) { +- if (!visited.test_set(n->in(i)->_idx)) { +- nodeStack.push(n->in(i)); ++ if (!safepoint_found) { ++ load->set_barrier_data(ZLoadBarrierElided); + } + } + } +- +- bool is_old_node = (n->_idx < new_ids); // don't process nodes that were created during cleanup +- if (n->is_Load() && is_old_node) { +- LoadNode* load = n->isa_Load(); +- // only care about loads that will have a barrier +- if (load_require_barrier(load)) { +- process_catch_cleanup_candidate(phase, load); +- } +- } + } +- +- C->print_method(PHASE_CALL_CATCH_CLEANUP, 2); + } + +-class DomDepthCompareClosure : public CompareClosure { +- PhaseIdealLoop* _phase; +- +-public: +- DomDepthCompareClosure(PhaseIdealLoop* phase) : _phase(phase) { } +- +- int do_compare(LoadNode* const &n1, LoadNode* const &n2) { +- int d1 = _phase->dom_depth(_phase->get_ctrl(n1)); +- int d2 = _phase->dom_depth(_phase->get_ctrl(n2)); +- if (d1 == d2) { +- // Compare index if the depth is the same, ensures all entries are unique. +- return n1->_idx - n2->_idx; +- } else { +- return d2 - d1; +- } +- } +-}; +- +-// Traverse graph and add all loadPs to list, sorted by dom depth +-void gather_loadnodes_sorted(PhaseIdealLoop* phase, GrowableArray* loadList) { +- +- VectorSet visited(Thread::current()->resource_area()); +- GrowableArray nodeStack(Thread::current()->resource_area(), 0, 0, NULL); +- DomDepthCompareClosure ddcc(phase); ++// == Reduced spilling optimization == + +- nodeStack.push(phase->C->root()); +- while(nodeStack.length() > 0) { +- Node *n = nodeStack.pop(); +- if (visited.test(n->_idx)) { +- continue; +- } +- +- if (n->isa_Load()) { +- LoadNode *load = n->as_Load(); +- if (load_require_barrier(load)) { +- assert(phase->get_ctrl(load) != NULL, "sanity"); +- assert(phase->dom_depth(phase->get_ctrl(load)) != 0, "sanity"); +- loadList->insert_sorted(&ddcc, load); +- } +- } ++void ZBarrierSetC2::compute_liveness_at_stubs() const { ++ ResourceMark rm; ++ Compile* const C = Compile::current(); ++ Arena* const A = Thread::current()->resource_area(); ++ PhaseCFG* const cfg = C->cfg(); ++ PhaseRegAlloc* const regalloc = C->regalloc(); ++ RegMask* const live = NEW_ARENA_ARRAY(A, RegMask, cfg->number_of_blocks() * sizeof(RegMask)); ++ ZBarrierSetAssembler* const bs = ZBarrierSet::assembler(); ++ Block_List worklist; ++ ++ for (uint i = 0; i < cfg->number_of_blocks(); ++i) { ++ new ((void*)(live + i)) RegMask(); ++ worklist.push(cfg->get_block(i)); ++ } ++ ++ while (worklist.size() > 0) { ++ const Block* const block = worklist.pop(); ++ RegMask& old_live = live[block->_pre_order]; ++ RegMask new_live; ++ ++ // Initialize to union of successors ++ for (uint i = 0; i < block->_num_succs; i++) { ++ const uint succ_id = block->_succs[i]->_pre_order; ++ new_live.OR(live[succ_id]); ++ } ++ ++ // Walk block backwards, computing liveness ++ for (int i = block->number_of_nodes() - 1; i >= 0; --i) { ++ const Node* const node = block->get_node(i); ++ ++ // Remove def bits ++ const OptoReg::Name first = bs->refine_register(node, regalloc->get_reg_first(node)); ++ const OptoReg::Name second = bs->refine_register(node, regalloc->get_reg_second(node)); ++ if (first != OptoReg::Bad) { ++ new_live.Remove(first); ++ } ++ if (second != OptoReg::Bad) { ++ new_live.Remove(second); ++ } ++ ++ // Add use bits ++ for (uint j = 1; j < node->req(); ++j) { ++ const Node* const use = node->in(j); ++ const OptoReg::Name first = bs->refine_register(use, regalloc->get_reg_first(use)); ++ const OptoReg::Name second = bs->refine_register(use, regalloc->get_reg_second(use)); ++ if (first != OptoReg::Bad) { ++ new_live.Insert(first); ++ } + +- visited.set(n->_idx); +- for (uint i = 0; i < n->req(); i++) { +- if (n->in(i)) { +- if (!visited.test(n->in(i)->_idx)) { +- nodeStack.push(n->in(i)); ++ if (second != OptoReg::Bad) { ++ new_live.Insert(second); + } + } +- } +- } +-} +- +-// Add LoadBarriers to all LoadPs +-void ZBarrierSetC2::insert_load_barriers(PhaseIdealLoop* phase) const { +- bool trace = phase->C->directive()->ZTraceLoadBarriersOption; +- GrowableArray loadList(Thread::current()->resource_area(), 0, 0, NULL); +- gather_loadnodes_sorted(phase, &loadList); +- PhaseIterGVN &igvn = phase->igvn(); +- int count = 0; +- +- for (GrowableArrayIterator loadIter = loadList.begin(); loadIter != loadList.end(); ++loadIter) { +- LoadNode *load = *loadIter; + +- if (load_has_expanded_barrier(load)) { +- continue; +- } +- +- do { +- // Insert a barrier on a loadP +- // if another load is found that needs to be expanded first, retry on that one +- LoadNode* result = insert_one_loadbarrier(phase, load, phase->get_ctrl(load)); +- while (result != NULL) { +- result = insert_one_loadbarrier(phase, result, phase->get_ctrl(result)); ++ // If this node tracks liveness, update it ++ RegMask* const regs = barrier_set_state()->live(node); ++ if (regs != NULL) { ++ regs->OR(new_live); + } +- } while (!load_has_expanded_barrier(load)); +- } +- phase->C->print_method(PHASE_INSERT_BARRIER, 2); +-} +- +-void push_antidependent_stores(PhaseIdealLoop* phase, Node_Stack& nodestack, LoadNode* start_load) { +- // push all stores on the same mem, that can_alias +- // Any load found must be handled first +- PhaseIterGVN &igvn = phase->igvn(); +- int load_alias_idx = igvn.C->get_alias_index(start_load->adr_type()); +- +- Node *mem = start_load->in(1); +- for (DUIterator_Fast imax, u = mem->fast_outs(imax); u < imax; u++) { +- Node *mem_use = mem->fast_out(u); +- +- if (mem_use == start_load) continue; +- if (!mem_use->is_Store()) continue; +- if (!phase->has_ctrl(mem_use)) continue; +- if (phase->get_ctrl(mem_use) != phase->get_ctrl(start_load)) continue; +- +- // add any aliasing store in this block +- StoreNode *store = mem_use->isa_Store(); +- const TypePtr *adr_type = store->adr_type(); +- if (igvn.C->can_alias(adr_type, load_alias_idx)) { +- nodestack.push(store, 0); +- } +- } +-} +- +-LoadNode* ZBarrierSetC2::insert_one_loadbarrier(PhaseIdealLoop* phase, LoadNode* start_load, Node* ctrl) const { +- bool trace = phase->C->directive()->ZTraceLoadBarriersOption; +- PhaseIterGVN &igvn = phase->igvn(); +- +- // Check for other loadPs at the same loop depth that is reachable by a DFS +- // - if found - return it. It needs to be inserted first +- // - otherwise proceed and insert barrier +- +- VectorSet visited(Thread::current()->resource_area()); +- Node_Stack nodestack(100); +- +- nodestack.push(start_load, 0); +- push_antidependent_stores(phase, nodestack, start_load); +- +- while(!nodestack.is_empty()) { +- Node* n = nodestack.node(); // peek +- nodestack.pop(); +- if (visited.test(n->_idx)) { +- continue; +- } +- +- if (n->is_Load() && n != start_load && load_require_barrier(n->as_Load()) && !load_has_expanded_barrier(n->as_Load())) { +- // Found another load that needs a barrier in the same block. Must expand later loads first. +- if (trace) tty->print_cr(" * Found LoadP %i on DFS", n->_idx); +- return n->as_Load(); // return node that should be expanded first + } + +- if (!phase->has_ctrl(n)) continue; +- if (phase->get_ctrl(n) != phase->get_ctrl(start_load)) continue; +- if (n->is_Phi()) continue; +- +- visited.set(n->_idx); +- // push all children +- for (DUIterator_Fast imax, ii = n->fast_outs(imax); ii < imax; ii++) { +- Node* c = n->fast_out(ii); +- if (c != NULL) { +- nodestack.push(c, 0); +- } +- } +- } +- insert_one_loadbarrier_inner(phase, start_load, ctrl, visited); +- return NULL; +-} +- +-void ZBarrierSetC2::insert_one_loadbarrier_inner(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl, VectorSet visited2) const { +- PhaseIterGVN &igvn = phase->igvn(); +- Compile* C = igvn.C; +- bool trace = C->directive()->ZTraceLoadBarriersOption; +- +- // create barrier +- Node* barrier = new LoadBarrierNode(C, NULL, load->in(LoadNode::Memory), NULL, load->in(LoadNode::Address), load_has_weak_barrier(load)); +- Node* barrier_val = new ProjNode(barrier, LoadBarrierNode::Oop); +- Node* barrier_ctrl = new ProjNode(barrier, LoadBarrierNode::Control); +- +- if (trace) tty->print_cr("Insert load %i with barrier: %i and ctrl : %i", load->_idx, barrier->_idx, ctrl->_idx); +- +- // Splice control +- // - insert barrier control diamond between loads ctrl and ctrl successor on path to block end. +- // - If control successor is a catch, step over to next. +- Node* ctrl_succ = NULL; +- for (DUIterator_Fast imax, j = ctrl->fast_outs(imax); j < imax; j++) { +- Node* tmp = ctrl->fast_out(j); +- +- // - CFG nodes is the ones we are going to splice (1 only!) +- // - Phi nodes will continue to hang from the region node! +- // - self loops should be skipped +- if (tmp->is_Phi() || tmp == ctrl) { +- continue; +- } +- +- if (tmp->is_CFG()) { +- assert(ctrl_succ == NULL, "There can be only one"); +- ctrl_succ = tmp; +- continue; +- } +- } +- +- // Now splice control +- assert(ctrl_succ != load, "sanity"); +- assert(ctrl_succ != NULL, "Broken IR"); +- bool found = false; +- for(uint k = 0; k < ctrl_succ->req(); k++) { +- if (ctrl_succ->in(k) == ctrl) { +- assert(!found, "sanity"); +- if (trace) tty->print_cr(" Move CFG ctrl_succ %i to barrier_ctrl", ctrl_succ->_idx); +- igvn.replace_input_of(ctrl_succ, k, barrier_ctrl); +- found = true; +- k--; +- } +- } +- +- // For all successors of ctrl - move all visited to become successors of barrier_ctrl instead +- for (DUIterator_Fast imax, r = ctrl->fast_outs(imax); r < imax; r++) { +- Node* tmp = ctrl->fast_out(r); +- if (visited2.test(tmp->_idx) && (tmp != load)) { +- if (trace) tty->print_cr(" Move ctrl_succ %i to barrier_ctrl", tmp->_idx); +- igvn.replace_input_of(tmp, 0, barrier_ctrl); +- --r; --imax; +- } +- } +- +- // Move the loads user to the barrier +- for (DUIterator_Fast imax, i = load->fast_outs(imax); i < imax; i++) { +- Node* u = load->fast_out(i); +- if (u->isa_LoadBarrier()) { +- continue; +- } +- +- // find correct input - replace with iterator? +- for(uint j = 0; j < u->req(); j++) { +- if (u->in(j) == load) { +- igvn.replace_input_of(u, j, barrier_val); +- --i; --imax; // Adjust the iterator of the *outer* loop +- break; // some nodes (calls) might have several uses from the same node ++ // Now at block top, see if we have any changes ++ new_live.SUBTRACT(old_live); ++ if (new_live.is_NotEmpty()) { ++ // Liveness has refined, update and propagate to prior blocks ++ old_live.OR(new_live); ++ for (uint i = 1; i < block->num_preds(); ++i) { ++ Block* const pred = cfg->get_block_for_node(block->pred(i)); ++ worklist.push(pred); + } + } + } +- +- // Connect barrier to load and control +- barrier->set_req(LoadBarrierNode::Oop, load); +- barrier->set_req(LoadBarrierNode::Control, ctrl); +- +- igvn.rehash_node_delayed(load); +- igvn.register_new_node_with_optimizer(barrier); +- igvn.register_new_node_with_optimizer(barrier_val); +- igvn.register_new_node_with_optimizer(barrier_ctrl); +- load_set_expanded_barrier(load); +- +- C->print_method(PHASE_INSERT_BARRIER, 3, load->_idx); +-} +- +-// The bad_mask in the ThreadLocalData shouldn't have an anti-dep-check. +-// The bad_mask address if of type TypeRawPtr, but that will alias +-// InitializeNodes until the type system is expanded. +-bool ZBarrierSetC2::needs_anti_dependence_check(const Node* node) const { +- MachNode* mnode = node->as_Mach(); +- if (mnode != NULL) { +- intptr_t offset = 0; +- const TypePtr *adr_type2 = NULL; +- const Node* base = mnode->get_base_and_disp(offset, adr_type2); +- if ((base != NULL) && +- (base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_ThreadLocal) && +- (offset == in_bytes(ZThreadLocalData::address_bad_mask_offset()))) { +- return false; +- } +- } +- return true; + } +diff --git a/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp b/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp +index bef6a7048..accae1f72 100644 +--- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp ++++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp +@@ -29,134 +29,38 @@ + #include "opto/node.hpp" + #include "utilities/growableArray.hpp" + +-class ZCompareAndSwapPNode : public CompareAndSwapPNode { +-public: +- ZCompareAndSwapPNode(Node* c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapPNode(c, mem, adr, val, ex, mem_ord) { } +- virtual int Opcode() const; +-}; +- +-class ZWeakCompareAndSwapPNode : public WeakCompareAndSwapPNode { +-public: +- ZWeakCompareAndSwapPNode(Node* c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : WeakCompareAndSwapPNode(c, mem, adr, val, ex, mem_ord) { } +- virtual int Opcode() const; +-}; +- +-class ZCompareAndExchangePNode : public CompareAndExchangePNode { +-public: +- ZCompareAndExchangePNode(Node* c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, const Type* t, MemNode::MemOrd mem_ord) : CompareAndExchangePNode(c, mem, adr, val, ex, at, t, mem_ord) { } +- virtual int Opcode() const; +-}; +- +-class ZGetAndSetPNode : public GetAndSetPNode { +-public: +- ZGetAndSetPNode(Node* c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* t) : GetAndSetPNode(c, mem, adr, val, at, t) { } +- virtual int Opcode() const; +-}; +- +-class LoadBarrierNode : public MultiNode { +-private: +- bool _weak; +- +- static bool is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n); +- void push_dominated_barriers(PhaseIterGVN* igvn) const; +- +-public: +- enum { +- Control, +- Memory, +- Oop, +- Address, +- Number_of_Outputs = Address, +- Similar, +- Number_of_Inputs +- }; +- +- LoadBarrierNode(Compile* C, +- Node* c, +- Node* mem, +- Node* val, +- Node* adr, +- bool weak); +- +- virtual int Opcode() const; +- virtual uint size_of() const; +- virtual uint cmp(const Node& n) const; +- virtual const Type *bottom_type() const; +- virtual const TypePtr* adr_type() const; +- virtual const Type *Value(PhaseGVN *phase) const; +- virtual Node *Identity(PhaseGVN *phase); +- virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); +- virtual uint match_edge(uint idx) const; +- +- LoadBarrierNode* has_dominating_barrier(PhaseIdealLoop* phase, +- bool linear_only, +- bool look_for_similar); +- +- void fix_similar_in_uses(PhaseIterGVN* igvn); ++const uint8_t ZLoadBarrierStrong = 1; ++const uint8_t ZLoadBarrierWeak = 2; ++const uint8_t ZLoadBarrierElided = 3; + +- bool has_true_uses() const; +- +- bool can_be_eliminated() const { +- return !in(Similar)->is_top(); +- } +- +- bool is_weak() const { +- return _weak; +- } +-}; +- +-class LoadBarrierSlowRegNode : public LoadPNode { +-private: +- bool _is_weak; +-public: +- LoadBarrierSlowRegNode(Node *c, +- Node *mem, +- Node *adr, +- const TypePtr *at, +- const TypePtr* t, +- MemOrd mo, +- bool weak = false, +- ControlDependency control_dependency = DependsOnlyOnTest) +- : LoadPNode(c, mem, adr, at, t, mo, control_dependency), _is_weak(weak) { +- init_class_id(Class_LoadBarrierSlowReg); +- } +- +- virtual const char * name() { +- return "LoadBarrierSlowRegNode"; +- } +- +- virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) { +- return NULL; +- } +- +- virtual int Opcode() const; +- bool is_weak() { return _is_weak; } +-}; +- +-class ZBarrierSetC2State : public ResourceObj { ++class ZLoadBarrierStubC2 : public ResourceObj { + private: +- // List of load barrier nodes which need to be expanded before matching +- GrowableArray* _load_barrier_nodes; +- ++ const MachNode* _node; ++ const Address _ref_addr; ++ const Register _ref; ++ const Register _tmp; ++ const bool _weak; ++ Label _entry; ++ Label _continuation; ++ ++ ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak); ++ + public: +- ZBarrierSetC2State(Arena* comp_arena); +- int load_barrier_count() const; +- void add_load_barrier_node(LoadBarrierNode* n); +- void remove_load_barrier_node(LoadBarrierNode* n); +- LoadBarrierNode* load_barrier_node(int idx) const; +-}; +- +-enum BarrierInfo { +- NoBarrier = 0, +- RequireBarrier = 1, +- WeakBarrier = 3, // Inclusive with RequireBarrier +- ExpandedBarrier = 4 ++ static ZLoadBarrierStubC2* create(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak); ++ ++ Address ref_addr() const; ++ Register ref() const; ++ Register tmp() const; ++ address slow_path() const; ++ RegMask& live() const; ++ Label* entry(); ++ Label* continuation(); + }; + + class ZBarrierSetC2 : public BarrierSetC2 { + private: +- ZBarrierSetC2State* state() const; +- void expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const; ++ void compute_liveness_at_stubs() const; ++ void analyze_dominating_barriers() const; + + protected: + virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const; +@@ -174,38 +78,11 @@ protected: + + public: + virtual void* create_barrier_state(Arena* comp_arena) const; +- virtual bool has_load_barriers() const { return true; } +- virtual bool is_gc_barrier_node(Node* node) const; +- +- virtual void register_potential_barrier_node(Node* node) const; +- virtual void unregister_potential_barrier_node(Node* node) const; +- virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const { } +- virtual void enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const; +- virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful) const; +- +- virtual bool array_copy_requires_gc_barriers(BasicType type) const { return true; } +- virtual Node* step_over_gc_barrier(Node* c) const; +- // If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be +- // expanded later, then now is the time to do so. +- virtual bool expand_macro_nodes(PhaseMacroExpand* macro) const; +- virtual bool needs_anti_dependence_check(const Node* node) const; +- +-#ifdef ASSERT +- virtual void verify_gc_barriers(bool post_parse) const; +-#endif +- +- // Load barrier insertion and expansion external +- virtual void barrier_insertion_phase(Compile* C, PhaseIterGVN &igvn) const; +- virtual void optimize_loops(PhaseIdealLoop* phase, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const; +- +-private: +- // Load barrier insertion and expansion internal +- void insert_barriers_on_unsafe(PhaseIdealLoop* phase) const; +- void clean_catch_blocks(PhaseIdealLoop* phase) const; +- void insert_load_barriers(PhaseIdealLoop* phase) const; +- LoadNode* insert_one_loadbarrier(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl) const; +- void insert_one_loadbarrier_inner(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl, VectorSet visited) const; ++ virtual bool array_copy_requires_gc_barriers(BasicType type) const; + ++ virtual void late_barrier_analysis() const; ++ virtual int estimate_stub_size() const; ++ virtual void emit_stubs(CodeBuffer& cb) const; + }; + + #endif // SHARE_GC_Z_C2_ZBARRIERSETC2_HPP +diff --git a/src/hotspot/share/gc/z/zBarrierSetAssembler.hpp b/src/hotspot/share/gc/z/zBarrierSetAssembler.hpp +index 52133c073..9d07f9e8c 100644 +--- a/src/hotspot/share/gc/z/zBarrierSetAssembler.hpp ++++ b/src/hotspot/share/gc/z/zBarrierSetAssembler.hpp +@@ -1,5 +1,5 @@ + /* +- * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. ++ * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it +@@ -24,10 +24,7 @@ + #ifndef SHARE_GC_Z_ZBARRIERSETASSEMBLER_HPP + #define SHARE_GC_Z_ZBARRIERSETASSEMBLER_HPP + +-#include "asm/macroAssembler.hpp" + #include "gc/shared/barrierSetAssembler.hpp" +-#include "oops/accessDecorators.hpp" +-#include "utilities/globalDefinitions.hpp" + #include "utilities/macros.hpp" + + class ZBarrierSetAssemblerBase : public BarrierSetAssembler { +diff --git a/src/hotspot/share/opto/classes.cpp b/src/hotspot/share/opto/classes.cpp +index e8ac3a61d..eec705013 100644 +--- a/src/hotspot/share/opto/classes.cpp ++++ b/src/hotspot/share/opto/classes.cpp +@@ -48,9 +48,6 @@ + #include "opto/subnode.hpp" + #include "opto/vectornode.hpp" + #include "utilities/macros.hpp" +-#if INCLUDE_ZGC +-#include "gc/z/c2/zBarrierSetC2.hpp" +-#endif + + // ---------------------------------------------------------------------------- + // Build a table of virtual functions to map from Nodes to dense integer +diff --git a/src/hotspot/share/opto/classes.hpp b/src/hotspot/share/opto/classes.hpp +index f6b588685..2fae8dd26 100644 +--- a/src/hotspot/share/opto/classes.hpp ++++ b/src/hotspot/share/opto/classes.hpp +@@ -189,17 +189,6 @@ macro(LoadP) + macro(LoadN) + macro(LoadRange) + macro(LoadS) +-#if INCLUDE_ZGC +-#define zgcmacro(x) macro(x) +-#else +-#define zgcmacro(x) optionalmacro(x) +-#endif +-zgcmacro(LoadBarrier) +-zgcmacro(LoadBarrierSlowReg) +-zgcmacro(ZCompareAndSwapP) +-zgcmacro(ZWeakCompareAndSwapP) +-zgcmacro(ZCompareAndExchangeP) +-zgcmacro(ZGetAndSetP) + macro(Lock) + macro(Loop) + macro(LoopLimit) +diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp +index ba61967bc..3145d1e2f 100644 +--- a/src/hotspot/share/opto/compile.cpp ++++ b/src/hotspot/share/opto/compile.cpp +@@ -79,9 +79,6 @@ + #if INCLUDE_G1GC + #include "gc/g1/g1ThreadLocalData.hpp" + #endif // INCLUDE_G1GC +-#if INCLUDE_ZGC +-#include "gc/z/c2/zBarrierSetC2.hpp" +-#endif + + + // -------------------- Compile::mach_constant_base_node ----------------------- +@@ -636,16 +633,12 @@ debug_only( int Compile::_debug_idx = 100000; ) + Compile::Compile( ciEnv* ci_env, C2Compiler* compiler, ciMethod* target, int osr_bci, + bool subsume_loads, bool do_escape_analysis, bool eliminate_boxing, DirectiveSet* directive) + : Phase(Compiler), +- _env(ci_env), +- _directive(directive), +- _log(ci_env->log()), + _compile_id(ci_env->compile_id()), + _save_argument_registers(false), + _stub_name(NULL), + _stub_function(NULL), + _stub_entry_point(NULL), + _method(target), +- _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())), + _entry_bci(osr_bci), + _initial_gvn(NULL), + _for_igvn(NULL), +@@ -665,8 +658,6 @@ Compile::Compile( ciEnv* ci_env, C2Compiler* compiler, ciMethod* target, int osr + _inner_loops(0), + _scratch_const_size(-1), + _in_scratch_emit_size(false), +- _dead_node_list(comp_arena()), +- _dead_node_count(0), + #ifndef PRODUCT + _trace_opto_output(directive->TraceOptoOutputOption), + _in_dump_cnt(0), +@@ -674,9 +665,15 @@ Compile::Compile( ciEnv* ci_env, C2Compiler* compiler, ciMethod* target, int osr + #endif + _congraph(NULL), + _comp_arena(mtCompiler), ++ _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())), ++ _env(ci_env), ++ _directive(directive), ++ _log(ci_env->log()), + _node_arena(mtCompiler), + _old_arena(mtCompiler), + _Compile_types(mtCompiler), ++ _dead_node_list(comp_arena()), ++ _dead_node_count(0), + _replay_inline_data(NULL), + _late_inlines(comp_arena(), 2, 0, NULL), + _string_late_inlines(comp_arena(), 2, 0, NULL), +@@ -959,9 +956,6 @@ Compile::Compile( ciEnv* ci_env, + bool return_pc, + DirectiveSet* directive) + : Phase(Compiler), +- _env(ci_env), +- _directive(directive), +- _log(ci_env->log()), + _compile_id(0), + _save_argument_registers(save_arg_registers), + _method(NULL), +@@ -991,6 +985,10 @@ Compile::Compile( ciEnv* ci_env, + _printer(NULL), + #endif + _comp_arena(mtCompiler), ++ _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())), ++ _env(ci_env), ++ _directive(directive), ++ _log(ci_env->log()), + _node_arena(mtCompiler), + _old_arena(mtCompiler), + _Compile_types(mtCompiler), +@@ -2366,13 +2364,6 @@ void Compile::Optimize() { + bs->verify_gc_barriers(false); + #endif + +- bs->barrier_insertion_phase(C, igvn); +- if (failing()) return; +- +-#ifdef ASSERT +- bs->verify_gc_barriers(false); +-#endif +- + { + TracePhase tp("macroExpand", &timers[_t_macroExpand]); + PhaseMacroExpand mex(igvn); +@@ -2933,29 +2924,6 @@ void Compile::final_graph_reshaping_impl( Node *n, Final_Reshape_Counts &frc) { + break; + } + +-#if INCLUDE_ZGC +- case Op_LoadBarrier: { +- assert(0, "There should be no load barriers left"); +- } +- case Op_ZGetAndSetP: +- case Op_ZCompareAndExchangeP: +- case Op_ZCompareAndSwapP: +- case Op_ZWeakCompareAndSwapP: +- case Op_LoadBarrierSlowReg: { +-#ifdef ASSERT +- if (VerifyOptoOopOffsets) { +- MemNode *mem = n->as_Mem(); +- // Check to see if address types have grounded out somehow. +- const TypeInstPtr *tp = mem->in(MemNode::Address)->bottom_type()->isa_instptr(); +- ciInstanceKlass *k = tp->klass()->as_instance_klass(); +- bool oop_offset_is_sane = k->contains_field_offset(tp->offset()); +- assert(!tp || oop_offset_is_sane, ""); +- } +-#endif +- break; +- } +-#endif +- + case Op_AddP: { // Assert sane base pointers + Node *addp = n->in(AddPNode::Address); + assert( !addp->is_AddP() || +diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp +index d58758db7..bb5622221 100644 +--- a/src/hotspot/share/opto/compile.hpp ++++ b/src/hotspot/share/opto/compile.hpp +@@ -55,7 +55,6 @@ class ConnectionGraph; + class IdealGraphPrinter; + class InlineTree; + class Int_Array; +-class LoadBarrierNode; + class Matcher; + class MachConstantNode; + class MachConstantBaseNode; +@@ -1168,11 +1167,7 @@ class Compile : public Phase { + bool in_scratch_emit_size() const { return _in_scratch_emit_size; } + + enum ScratchBufferBlob { +-#if defined(PPC64) + MAX_inst_size = 2048, +-#else +- MAX_inst_size = 1024, +-#endif + MAX_locs_size = 128, // number of relocInfo elements + MAX_const_size = 128, + MAX_stubs_size = 128 +@@ -1247,14 +1242,30 @@ class Compile : public Phase { + // Process an OopMap Element while emitting nodes + void Process_OopMap_Node(MachNode *mach, int code_offset); + ++ class BufferSizingData { ++ public: ++ int _stub; ++ int _code; ++ int _const; ++ int _reloc; ++ ++ BufferSizingData() : ++ _stub(0), ++ _code(0), ++ _const(0), ++ _reloc(0) ++ { }; ++ }; ++ + // Initialize code buffer +- CodeBuffer* init_buffer(uint* blk_starts); ++ void estimate_buffer_size(int& const_req); ++ CodeBuffer* init_buffer(BufferSizingData& buf_sizes); + + // Write out basic block data to code buffer + void fill_buffer(CodeBuffer* cb, uint* blk_starts); + + // Determine which variable sized branches can be shortened +- void shorten_branches(uint* blk_starts, int& code_size, int& reloc_size, int& stub_size); ++ void shorten_branches(uint* blk_starts, BufferSizingData& buf_sizes); + + // Compute the size of first NumberOfLoopInstrToAlign instructions + // at the head of a loop. +diff --git a/src/hotspot/share/opto/escape.cpp b/src/hotspot/share/opto/escape.cpp +index 47c83766e..934e2acb9 100644 +--- a/src/hotspot/share/opto/escape.cpp ++++ b/src/hotspot/share/opto/escape.cpp +@@ -487,13 +487,6 @@ void ConnectionGraph::add_node_to_connection_graph(Node *n, Unique_Node_List *de + add_local_var_and_edge(n, PointsToNode::NoEscape, + n->in(0), delayed_worklist); + } +-#if INCLUDE_ZGC +- else if (UseZGC) { +- if (n->as_Proj()->_con == LoadBarrierNode::Oop && n->in(0)->is_LoadBarrier()) { +- add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0)->in(LoadBarrierNode::Oop), delayed_worklist); +- } +- } +-#endif + break; + } + case Op_Rethrow: // Exception object escapes +@@ -701,14 +694,6 @@ void ConnectionGraph::add_final_edges(Node *n) { + add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), NULL); + break; + } +-#if INCLUDE_ZGC +- else if (UseZGC) { +- if (n->as_Proj()->_con == LoadBarrierNode::Oop && n->in(0)->is_LoadBarrier()) { +- add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0)->in(LoadBarrierNode::Oop), NULL); +- break; +- } +- } +-#endif + ELSE_FAIL("Op_Proj"); + } + case Op_Rethrow: // Exception object escapes +diff --git a/src/hotspot/share/opto/lcm.cpp b/src/hotspot/share/opto/lcm.cpp +index 05ec9fa9f..16b80bfc3 100644 +--- a/src/hotspot/share/opto/lcm.cpp ++++ b/src/hotspot/share/opto/lcm.cpp +@@ -169,7 +169,6 @@ void PhaseCFG::implicit_null_check(Block* block, Node *proj, Node *val, int allo + case Op_LoadI: + case Op_LoadL: + case Op_LoadP: +- case Op_LoadBarrierSlowReg: + case Op_LoadN: + case Op_LoadS: + case Op_LoadKlass: +diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp +index d99e2f81b..da81c3a5d 100644 +--- a/src/hotspot/share/opto/loopnode.cpp ++++ b/src/hotspot/share/opto/loopnode.cpp +@@ -997,18 +997,6 @@ void LoopNode::verify_strip_mined(int expect_skeleton) const { + } + } + +- if (UseZGC && !inner_out->in(0)->is_CountedLoopEnd()) { +- // In some very special cases there can be a load that has no other uses than the +- // counted loop safepoint. Then its loadbarrier will be placed between the inner +- // loop exit and the safepoint. This is very rare +- +- Node* ifnode = inner_out->in(1)->in(0); +- // Region->IfTrue->If == Region->Iffalse->If +- if (ifnode == inner_out->in(2)->in(0)) { +- inner_out = ifnode->in(0); +- } +- } +- + CountedLoopEndNode* cle = inner_out->in(0)->as_CountedLoopEnd(); + assert(cle == inner->loopexit_or_null(), "mismatch"); + bool has_skeleton = outer_le->in(1)->bottom_type()->singleton() && outer_le->in(1)->bottom_type()->is_int()->get_con() == 0; +@@ -4230,7 +4218,6 @@ void PhaseIdealLoop::build_loop_late_post( Node *n ) { + case Op_LoadL: + case Op_LoadS: + case Op_LoadP: +- case Op_LoadBarrierSlowReg: + case Op_LoadN: + case Op_LoadRange: + case Op_LoadD_unaligned: +diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp +index 411cf0841..3a1034244 100644 +--- a/src/hotspot/share/opto/loopopts.cpp ++++ b/src/hotspot/share/opto/loopopts.cpp +@@ -41,9 +41,6 @@ + #include "opto/rootnode.hpp" + #include "opto/subnode.hpp" + #include "utilities/macros.hpp" +-#if INCLUDE_ZGC +-#include "gc/z/c2/zBarrierSetC2.hpp" +-#endif + + //============================================================================= + //------------------------------split_thru_phi--------------------------------- +diff --git a/src/hotspot/share/opto/machnode.hpp b/src/hotspot/share/opto/machnode.hpp +index bceaf1df1..6794e8ab2 100644 +--- a/src/hotspot/share/opto/machnode.hpp ++++ b/src/hotspot/share/opto/machnode.hpp +@@ -197,7 +197,7 @@ public: + // ADLC inherit from this class. + class MachNode : public Node { + public: +- MachNode() : Node((uint)0), _num_opnds(0), _opnds(NULL) { ++ MachNode() : Node((uint)0), _barrier(0), _num_opnds(0), _opnds(NULL) { + init_class_id(Class_Mach); + } + // Required boilerplate +@@ -211,6 +211,10 @@ public: + // no constant base node input. + virtual uint mach_constant_base_node_input() const { return (uint)-1; } + ++ uint8_t barrier_data() const { return _barrier; } ++ void set_barrier_data(uint data) { _barrier = data; } ++ ++ + // Copy inputs and operands to new node of instruction. + // Called from cisc_version() and short_branch_version(). + // !!!! The method's body is defined in ad_.cpp file. +@@ -255,6 +259,9 @@ public: + // output have choices - but they must use the same choice. + virtual uint two_adr( ) const { return 0; } + ++ // The GC might require some barrier metadata for machine code emission. ++ uint8_t _barrier; ++ + // Array of complex operand pointers. Each corresponds to zero or + // more leafs. Must be set by MachNode constructor to point to an + // internal array of MachOpers. The MachOper array is sized by +diff --git a/src/hotspot/share/opto/matcher.cpp b/src/hotspot/share/opto/matcher.cpp +index bdcf9c424..c75854e65 100644 +--- a/src/hotspot/share/opto/matcher.cpp ++++ b/src/hotspot/share/opto/matcher.cpp +@@ -1745,6 +1745,13 @@ MachNode *Matcher::ReduceInst( State *s, int rule, Node *&mem ) { + _shared_nodes.map(leaf->_idx, ex); + } + ++ // Have mach nodes inherit GC barrier data ++ if (leaf->is_LoadStore()) { ++ mach->set_barrier_data(leaf->as_LoadStore()->barrier_data()); ++ } else if (leaf->is_Mem()) { ++ mach->set_barrier_data(leaf->as_Mem()->barrier_data()); ++ } ++ + return ex; + } + +@@ -2164,17 +2171,6 @@ void Matcher::find_shared( Node *n ) { + case Op_SafePoint: + mem_op = true; + break; +-#if INCLUDE_ZGC +- case Op_CallLeaf: +- if (UseZGC) { +- if (n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr() || +- n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr()) { +- mem_op = true; +- mem_addr_idx = TypeFunc::Parms+1; +- } +- break; +- } +-#endif + default: + if( n->is_Store() ) { + // Do match stores, despite no ideal reg +@@ -2279,33 +2275,6 @@ void Matcher::find_shared( Node *n ) { + n->del_req(LoadStoreConditionalNode::ExpectedIn); + break; + } +-#if INCLUDE_ZGC +- case Op_ZCompareAndExchangeP: +- case Op_ZCompareAndSwapP: +- case Op_ZWeakCompareAndSwapP: { +- Node *mem = n->in(MemNode::Address); +- Node *keepalive = n->in(5); +- Node *pair1 = new BinaryNode(mem, keepalive); +- +- Node *newval = n->in(MemNode::ValueIn); +- Node *oldval = n->in(LoadStoreConditionalNode::ExpectedIn); +- Node *pair2 = new BinaryNode(oldval, newval); +- +- n->set_req(MemNode::Address, pair1); +- n->set_req(MemNode::ValueIn, pair2); +- n->del_req(5); +- n->del_req(LoadStoreConditionalNode::ExpectedIn); +- break; +- } +- case Op_ZGetAndSetP: { +- Node *keepalive = n->in(4); +- Node *newval = n->in(MemNode::ValueIn); +- Node *pair = new BinaryNode(newval, keepalive); +- n->set_req(MemNode::ValueIn, pair); +- n->del_req(4); +- break; +- } +-#endif // INCLUDE_ZGC + case Op_CMoveD: // Convert trinary to binary-tree + case Op_CMoveF: + case Op_CMoveI: +diff --git a/src/hotspot/share/opto/memnode.cpp b/src/hotspot/share/opto/memnode.cpp +index 6fa851af2..ee0f09e11 100644 +--- a/src/hotspot/share/opto/memnode.cpp ++++ b/src/hotspot/share/opto/memnode.cpp +@@ -49,9 +49,6 @@ + #include "utilities/copy.hpp" + #include "utilities/macros.hpp" + #include "utilities/vmError.hpp" +-#if INCLUDE_ZGC +-#include "gc/z/c2/zBarrierSetC2.hpp" +-#endif + + // Portions of code courtesy of Clifford Click + +@@ -2812,7 +2809,7 @@ LoadStoreNode::LoadStoreNode( Node *c, Node *mem, Node *adr, Node *val, const Ty + : Node(required), + _type(rt), + _adr_type(at), +- _has_barrier(false) ++ _barrier(0) + { + init_req(MemNode::Control, c ); + init_req(MemNode::Memory , mem); +diff --git a/src/hotspot/share/opto/memnode.hpp b/src/hotspot/share/opto/memnode.hpp +index 084a91c49..7468abdbc 100644 +--- a/src/hotspot/share/opto/memnode.hpp ++++ b/src/hotspot/share/opto/memnode.hpp +@@ -43,6 +43,8 @@ private: + bool _unaligned_access; // Unaligned access from unsafe + bool _mismatched_access; // Mismatched access from unsafe: byte read in integer array for instance + bool _unsafe_access; // Access of unsafe origin. ++ uint8_t _barrier; // Bit field with barrier information ++ + protected: + #ifdef ASSERT + const TypePtr* _adr_type; // What kind of memory is being addressed? +@@ -62,18 +64,30 @@ public: + unset // The memory ordering is not set (used for testing) + } MemOrd; + protected: +- MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at ) +- : Node(c0,c1,c2 ), _unaligned_access(false), _mismatched_access(false), _unsafe_access(false) { ++ MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at ) : ++ Node(c0,c1,c2), ++ _unaligned_access(false), ++ _mismatched_access(false), ++ _unsafe_access(false), ++ _barrier(0) { + init_class_id(Class_Mem); + debug_only(_adr_type=at; adr_type();) + } +- MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at, Node *c3 ) +- : Node(c0,c1,c2,c3), _unaligned_access(false), _mismatched_access(false), _unsafe_access(false) { ++ MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at, Node *c3 ) : ++ Node(c0,c1,c2,c3), ++ _unaligned_access(false), ++ _mismatched_access(false), ++ _unsafe_access(false), ++ _barrier(0) { + init_class_id(Class_Mem); + debug_only(_adr_type=at; adr_type();) + } +- MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at, Node *c3, Node *c4) +- : Node(c0,c1,c2,c3,c4), _unaligned_access(false), _mismatched_access(false), _unsafe_access(false) { ++ MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at, Node *c3, Node *c4) : ++ Node(c0,c1,c2,c3,c4), ++ _unaligned_access(false), ++ _mismatched_access(false), ++ _unsafe_access(false), ++ _barrier(0) { + init_class_id(Class_Mem); + debug_only(_adr_type=at; adr_type();) + } +@@ -125,6 +139,9 @@ public: + #endif + } + ++ uint8_t barrier_data() { return _barrier; } ++ void set_barrier_data(uint8_t barrier_data) { _barrier = barrier_data; } ++ + // Search through memory states which precede this node (load or store). + // Look for an exact match for the address, with no intervening + // aliased stores. +@@ -182,8 +199,6 @@ private: + // this field. + const MemOrd _mo; + +- uint _barrier; // Bit field with barrier information +- + protected: + virtual uint cmp(const Node &n) const; + virtual uint size_of() const; // Size is bigger +@@ -195,7 +210,7 @@ protected: + public: + + LoadNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const Type *rt, MemOrd mo, ControlDependency control_dependency) +- : MemNode(c,mem,adr,at), _type(rt), _mo(mo), _control_dependency(control_dependency), _barrier(0) { ++ : MemNode(c,mem,adr,at), _control_dependency(control_dependency), _mo(mo), _type(rt) { + init_class_id(Class_Load); + } + inline bool is_unordered() const { return !is_acquire(); } +@@ -264,10 +279,6 @@ public: + Node* convert_to_unsigned_load(PhaseGVN& gvn); + Node* convert_to_signed_load(PhaseGVN& gvn); + +- void copy_barrier_info(const Node* src) { _barrier = src->as_Load()->_barrier; } +- uint barrier_data() { return _barrier; } +- void set_barrier_data(uint barrier_data) { _barrier |= barrier_data; } +- + #ifndef PRODUCT + virtual void dump_spec(outputStream *st) const; + #endif +@@ -816,7 +827,7 @@ class LoadStoreNode : public Node { + private: + const Type* const _type; // What kind of value is loaded? + const TypePtr* _adr_type; // What kind of memory is being addressed? +- bool _has_barrier; ++ uint8_t _barrier; // Bit field with barrier information + virtual uint size_of() const; // Size is bigger + public: + LoadStoreNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* rt, uint required ); +@@ -829,8 +840,9 @@ public: + + bool result_not_used() const; + MemBarNode* trailing_membar() const; +- void set_has_barrier() { _has_barrier = true; }; +- bool has_barrier() const { return _has_barrier; }; ++ ++ uint8_t barrier_data() { return _barrier; } ++ void set_barrier_data(uint8_t barrier_data) { _barrier = barrier_data; } + }; + + class LoadStoreConditionalNode : public LoadStoreNode { +@@ -882,6 +894,7 @@ public: + MemNode::MemOrd order() const { + return _mem_ord; + } ++ virtual uint size_of() const { return sizeof(*this); } + }; + + class CompareAndExchangeNode : public LoadStoreNode { +@@ -899,6 +912,7 @@ public: + MemNode::MemOrd order() const { + return _mem_ord; + } ++ virtual uint size_of() const { return sizeof(*this); } + }; + + //------------------------------CompareAndSwapBNode--------------------------- +diff --git a/src/hotspot/share/opto/node.cpp b/src/hotspot/share/opto/node.cpp +index 2ff9bb2cd..ed5524b83 100644 +--- a/src/hotspot/share/opto/node.cpp ++++ b/src/hotspot/share/opto/node.cpp +@@ -546,9 +546,6 @@ Node *Node::clone() const { + if (n->is_SafePoint()) { + n->as_SafePoint()->clone_replaced_nodes(); + } +- if (n->is_Load()) { +- n->as_Load()->copy_barrier_info(this); +- } + return n; // Return the clone + } + +@@ -1468,10 +1465,6 @@ bool Node::needs_anti_dependence_check() const { + if( req() < 2 || (_flags & Flag_needs_anti_dependence_check) == 0 ) { + return false; + } +- BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); +- if (!bs->needs_anti_dependence_check(this)) { +- return false; +- } + return in(1)->bottom_type()->has_memory(); + } + +diff --git a/src/hotspot/share/opto/node.hpp b/src/hotspot/share/opto/node.hpp +index 49f415a31..9c0aedcad 100644 +--- a/src/hotspot/share/opto/node.hpp ++++ b/src/hotspot/share/opto/node.hpp +@@ -82,8 +82,6 @@ class JVMState; + class JumpNode; + class JumpProjNode; + class LoadNode; +-class LoadBarrierNode; +-class LoadBarrierSlowRegNode; + class LoadStoreNode; + class LoadStoreConditionalNode; + class LockNode; +@@ -641,7 +639,6 @@ public: + DEFINE_CLASS_ID(MemBar, Multi, 3) + DEFINE_CLASS_ID(Initialize, MemBar, 0) + DEFINE_CLASS_ID(MemBarStoreStore, MemBar, 1) +- DEFINE_CLASS_ID(LoadBarrier, Multi, 4) + + DEFINE_CLASS_ID(Mach, Node, 1) + DEFINE_CLASS_ID(MachReturn, Mach, 0) +@@ -691,7 +688,6 @@ public: + DEFINE_CLASS_ID(Mem, Node, 4) + DEFINE_CLASS_ID(Load, Mem, 0) + DEFINE_CLASS_ID(LoadVector, Load, 0) +- DEFINE_CLASS_ID(LoadBarrierSlowReg, Load, 1) + DEFINE_CLASS_ID(Store, Mem, 1) + DEFINE_CLASS_ID(StoreVector, Store, 0) + DEFINE_CLASS_ID(LoadStore, Mem, 2) +@@ -834,8 +830,6 @@ public: + DEFINE_CLASS_QUERY(Load) + DEFINE_CLASS_QUERY(LoadStore) + DEFINE_CLASS_QUERY(LoadStoreConditional) +- DEFINE_CLASS_QUERY(LoadBarrier) +- DEFINE_CLASS_QUERY(LoadBarrierSlowReg) + DEFINE_CLASS_QUERY(Lock) + DEFINE_CLASS_QUERY(Loop) + DEFINE_CLASS_QUERY(Mach) +diff --git a/src/hotspot/share/opto/output.cpp b/src/hotspot/share/opto/output.cpp +index b8f72ffca..4319f9fd1 100644 +--- a/src/hotspot/share/opto/output.cpp ++++ b/src/hotspot/share/opto/output.cpp +@@ -31,6 +31,8 @@ + #include "compiler/compileBroker.hpp" + #include "compiler/compilerDirectives.hpp" + #include "compiler/oopMap.hpp" ++#include "gc/shared/barrierSet.hpp" ++#include "gc/shared/c2/barrierSetC2.hpp" + #include "memory/allocation.inline.hpp" + #include "opto/ad.hpp" + #include "opto/callnode.hpp" +@@ -114,35 +116,35 @@ void Compile::Output() { + } + } + ++ // Keeper of sizing aspects ++ BufferSizingData buf_sizes = BufferSizingData(); ++ ++ // Initialize code buffer ++ estimate_buffer_size(buf_sizes._const); ++ if (failing()) return; ++ ++ // Pre-compute the length of blocks and replace ++ // long branches with short if machine supports it. ++ // Must be done before ScheduleAndBundle due to SPARC delay slots ++ + uint* blk_starts = NEW_RESOURCE_ARRAY(uint, _cfg->number_of_blocks() + 1); + blk_starts[0] = 0; + +- // Initialize code buffer and process short branches. +- CodeBuffer* cb = init_buffer(blk_starts); ++ shorten_branches(blk_starts, buf_sizes); + +- if (cb == NULL || failing()) { ++ ScheduleAndBundle(); ++ if (failing()) { + return; + } + +- ScheduleAndBundle(); +- +-#ifndef PRODUCT +- if (trace_opto_output()) { +- tty->print("\n---- After ScheduleAndBundle ----\n"); +- for (uint i = 0; i < _cfg->number_of_blocks(); i++) { +- tty->print("\nBB#%03d:\n", i); +- Block* block = _cfg->get_block(i); +- for (uint j = 0; j < block->number_of_nodes(); j++) { +- Node* n = block->get_node(j); +- OptoReg::Name reg = _regalloc->get_reg_first(n); +- tty->print(" %-6s ", reg >= 0 && reg < REG_COUNT ? Matcher::regName[reg] : ""); +- n->dump(); +- } +- } +- } +-#endif ++ // Late barrier analysis must be done after schedule and bundle ++ // Otherwise liveness based spilling will fail ++ BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); ++ bs->late_barrier_analysis(); + +- if (failing()) { ++ // Complete sizing of codebuffer ++ CodeBuffer* cb = init_buffer(buf_sizes); ++ if (cb == NULL || failing()) { + return; + } + +@@ -223,7 +225,7 @@ void Compile::compute_loop_first_inst_sizes() { + + // The architecture description provides short branch variants for some long + // branch instructions. Replace eligible long branches with short branches. +-void Compile::shorten_branches(uint* blk_starts, int& code_size, int& reloc_size, int& stub_size) { ++void Compile::shorten_branches(uint* blk_starts, BufferSizingData& buf_sizes) { + // Compute size of each block, method size, and relocation information size + uint nblocks = _cfg->number_of_blocks(); + +@@ -241,11 +243,11 @@ void Compile::shorten_branches(uint* blk_starts, int& code_size, int& reloc_size + bool has_short_branch_candidate = false; + + // Initialize the sizes to 0 +- code_size = 0; // Size in bytes of generated code +- stub_size = 0; // Size in bytes of all stub entries ++ int code_size = 0; // Size in bytes of generated code ++ int stub_size = 0; // Size in bytes of all stub entries + // Size in bytes of all relocation entries, including those in local stubs. + // Start with 2-bytes of reloc info for the unvalidated entry point +- reloc_size = 1; // Number of relocation entries ++ int reloc_size = 1; // Number of relocation entries + + // Make three passes. The first computes pessimistic blk_starts, + // relative jmp_offset and reloc_size information. The second performs +@@ -479,6 +481,10 @@ void Compile::shorten_branches(uint* blk_starts, int& code_size, int& reloc_size + // a relocation index. + // The CodeBuffer will expand the locs array if this estimate is too low. + reloc_size *= 10 / sizeof(relocInfo); ++ ++ buf_sizes._reloc = reloc_size; ++ buf_sizes._code = code_size; ++ buf_sizes._stub = stub_size; + } + + //------------------------------FillLocArray----------------------------------- +@@ -490,8 +496,8 @@ static LocationValue *new_loc_value( PhaseRegAlloc *ra, OptoReg::Name regnum, Lo + // This should never have accepted Bad before + assert(OptoReg::is_valid(regnum), "location must be valid"); + return (OptoReg::is_reg(regnum)) +- ? new LocationValue(Location::new_reg_loc(l_type, OptoReg::as_VMReg(regnum)) ) +- : new LocationValue(Location::new_stk_loc(l_type, ra->reg2offset(regnum))); ++ ? new LocationValue(Location::new_reg_loc(l_type, OptoReg::as_VMReg(regnum)) ) ++ : new LocationValue(Location::new_stk_loc(l_type, ra->reg2offset(regnum))); + } + + +@@ -610,12 +616,12 @@ void Compile::FillLocArray( int idx, MachSafePointNode* sfpt, Node *local, + } + #endif //_LP64 + else if( (t->base() == Type::FloatBot || t->base() == Type::FloatCon) && +- OptoReg::is_reg(regnum) ) { ++ OptoReg::is_reg(regnum) ) { + array->append(new_loc_value( _regalloc, regnum, Matcher::float_in_double() +- ? Location::float_in_dbl : Location::normal )); ++ ? Location::float_in_dbl : Location::normal )); + } else if( t->base() == Type::Int && OptoReg::is_reg(regnum) ) { + array->append(new_loc_value( _regalloc, regnum, Matcher::int_in_long +- ? Location::int_in_long : Location::normal )); ++ ? Location::int_in_long : Location::normal )); + } else if( t->base() == Type::NarrowOop ) { + array->append(new_loc_value( _regalloc, regnum, Location::narrowoop )); + } else { +@@ -626,48 +632,48 @@ void Compile::FillLocArray( int idx, MachSafePointNode* sfpt, Node *local, + + // No register. It must be constant data. + switch (t->base()) { +- case Type::Half: // Second half of a double +- ShouldNotReachHere(); // Caller should skip 2nd halves +- break; +- case Type::AnyPtr: +- array->append(new ConstantOopWriteValue(NULL)); +- break; +- case Type::AryPtr: +- case Type::InstPtr: // fall through +- array->append(new ConstantOopWriteValue(t->isa_oopptr()->const_oop()->constant_encoding())); +- break; +- case Type::NarrowOop: +- if (t == TypeNarrowOop::NULL_PTR) { ++ case Type::Half: // Second half of a double ++ ShouldNotReachHere(); // Caller should skip 2nd halves ++ break; ++ case Type::AnyPtr: + array->append(new ConstantOopWriteValue(NULL)); +- } else { +- array->append(new ConstantOopWriteValue(t->make_ptr()->isa_oopptr()->const_oop()->constant_encoding())); +- } +- break; +- case Type::Int: +- array->append(new ConstantIntValue(t->is_int()->get_con())); +- break; +- case Type::RawPtr: +- // A return address (T_ADDRESS). +- assert((intptr_t)t->is_ptr()->get_con() < (intptr_t)0x10000, "must be a valid BCI"); ++ break; ++ case Type::AryPtr: ++ case Type::InstPtr: // fall through ++ array->append(new ConstantOopWriteValue(t->isa_oopptr()->const_oop()->constant_encoding())); ++ break; ++ case Type::NarrowOop: ++ if (t == TypeNarrowOop::NULL_PTR) { ++ array->append(new ConstantOopWriteValue(NULL)); ++ } else { ++ array->append(new ConstantOopWriteValue(t->make_ptr()->isa_oopptr()->const_oop()->constant_encoding())); ++ } ++ break; ++ case Type::Int: ++ array->append(new ConstantIntValue(t->is_int()->get_con())); ++ break; ++ case Type::RawPtr: ++ // A return address (T_ADDRESS). ++ assert((intptr_t)t->is_ptr()->get_con() < (intptr_t)0x10000, "must be a valid BCI"); + #ifdef _LP64 +- // Must be restored to the full-width 64-bit stack slot. +- array->append(new ConstantLongValue(t->is_ptr()->get_con())); ++ // Must be restored to the full-width 64-bit stack slot. ++ array->append(new ConstantLongValue(t->is_ptr()->get_con())); + #else +- array->append(new ConstantIntValue(t->is_ptr()->get_con())); ++ array->append(new ConstantIntValue(t->is_ptr()->get_con())); + #endif +- break; +- case Type::FloatCon: { +- float f = t->is_float_constant()->getf(); +- array->append(new ConstantIntValue(jint_cast(f))); +- break; +- } +- case Type::DoubleCon: { +- jdouble d = t->is_double_constant()->getd(); ++ break; ++ case Type::FloatCon: { ++ float f = t->is_float_constant()->getf(); ++ array->append(new ConstantIntValue(jint_cast(f))); ++ break; ++ } ++ case Type::DoubleCon: { ++ jdouble d = t->is_double_constant()->getd(); + #ifdef _LP64 +- array->append(new ConstantIntValue((jint)0)); +- array->append(new ConstantDoubleValue(d)); ++ array->append(new ConstantIntValue((jint)0)); ++ array->append(new ConstantDoubleValue(d)); + #else +- // Repack the double as two jints. ++ // Repack the double as two jints. + // The convention the interpreter uses is that the second local + // holds the first raw word of the native double representation. + // This is actually reasonable, since locals and stack arrays +@@ -679,15 +685,15 @@ void Compile::FillLocArray( int idx, MachSafePointNode* sfpt, Node *local, + array->append(new ConstantIntValue(acc.words[1])); + array->append(new ConstantIntValue(acc.words[0])); + #endif +- break; +- } +- case Type::Long: { +- jlong d = t->is_long()->get_con(); ++ break; ++ } ++ case Type::Long: { ++ jlong d = t->is_long()->get_con(); + #ifdef _LP64 +- array->append(new ConstantIntValue((jint)0)); +- array->append(new ConstantLongValue(d)); ++ array->append(new ConstantIntValue((jint)0)); ++ array->append(new ConstantLongValue(d)); + #else +- // Repack the long as two jints. ++ // Repack the long as two jints. + // The convention the interpreter uses is that the second local + // holds the first raw word of the native double representation. + // This is actually reasonable, since locals and stack arrays +@@ -699,14 +705,14 @@ void Compile::FillLocArray( int idx, MachSafePointNode* sfpt, Node *local, + array->append(new ConstantIntValue(acc.words[1])); + array->append(new ConstantIntValue(acc.words[0])); + #endif +- break; +- } +- case Type::Top: // Add an illegal value here +- array->append(new LocationValue(Location())); +- break; +- default: +- ShouldNotReachHere(); +- break; ++ break; ++ } ++ case Type::Top: // Add an illegal value here ++ array->append(new LocationValue(Location())); ++ break; ++ default: ++ ShouldNotReachHere(); ++ break; + } + } + +@@ -871,58 +877,58 @@ void Compile::Process_OopMap_Node(MachNode *mach, int current_offset) { + + // A simplified version of Process_OopMap_Node, to handle non-safepoints. + class NonSafepointEmitter { +- Compile* C; +- JVMState* _pending_jvms; +- int _pending_offset; ++ Compile* C; ++ JVMState* _pending_jvms; ++ int _pending_offset; + +- void emit_non_safepoint(); ++ void emit_non_safepoint(); + + public: +- NonSafepointEmitter(Compile* compile) { +- this->C = compile; +- _pending_jvms = NULL; +- _pending_offset = 0; +- } ++ NonSafepointEmitter(Compile* compile) { ++ this->C = compile; ++ _pending_jvms = NULL; ++ _pending_offset = 0; ++ } + +- void observe_instruction(Node* n, int pc_offset) { +- if (!C->debug_info()->recording_non_safepoints()) return; ++ void observe_instruction(Node* n, int pc_offset) { ++ if (!C->debug_info()->recording_non_safepoints()) return; + +- Node_Notes* nn = C->node_notes_at(n->_idx); +- if (nn == NULL || nn->jvms() == NULL) return; +- if (_pending_jvms != NULL && +- _pending_jvms->same_calls_as(nn->jvms())) { +- // Repeated JVMS? Stretch it up here. +- _pending_offset = pc_offset; +- } else { ++ Node_Notes* nn = C->node_notes_at(n->_idx); ++ if (nn == NULL || nn->jvms() == NULL) return; + if (_pending_jvms != NULL && +- _pending_offset < pc_offset) { +- emit_non_safepoint(); +- } +- _pending_jvms = NULL; +- if (pc_offset > C->debug_info()->last_pc_offset()) { +- // This is the only way _pending_jvms can become non-NULL: +- _pending_jvms = nn->jvms(); ++ _pending_jvms->same_calls_as(nn->jvms())) { ++ // Repeated JVMS? Stretch it up here. + _pending_offset = pc_offset; ++ } else { ++ if (_pending_jvms != NULL && ++ _pending_offset < pc_offset) { ++ emit_non_safepoint(); ++ } ++ _pending_jvms = NULL; ++ if (pc_offset > C->debug_info()->last_pc_offset()) { ++ // This is the only way _pending_jvms can become non-NULL: ++ _pending_jvms = nn->jvms(); ++ _pending_offset = pc_offset; ++ } + } + } +- } + + // Stay out of the way of real safepoints: +- void observe_safepoint(JVMState* jvms, int pc_offset) { +- if (_pending_jvms != NULL && +- !_pending_jvms->same_calls_as(jvms) && +- _pending_offset < pc_offset) { +- emit_non_safepoint(); ++ void observe_safepoint(JVMState* jvms, int pc_offset) { ++ if (_pending_jvms != NULL && ++ !_pending_jvms->same_calls_as(jvms) && ++ _pending_offset < pc_offset) { ++ emit_non_safepoint(); ++ } ++ _pending_jvms = NULL; + } +- _pending_jvms = NULL; +- } + +- void flush_at_end() { +- if (_pending_jvms != NULL) { +- emit_non_safepoint(); ++ void flush_at_end() { ++ if (_pending_jvms != NULL) { ++ emit_non_safepoint(); ++ } ++ _pending_jvms = NULL; + } +- _pending_jvms = NULL; +- } + }; + + void NonSafepointEmitter::emit_non_safepoint() { +@@ -952,15 +958,11 @@ void NonSafepointEmitter::emit_non_safepoint() { + } + + //------------------------------init_buffer------------------------------------ +-CodeBuffer* Compile::init_buffer(uint* blk_starts) { ++void Compile::estimate_buffer_size(int& const_req) { + + // Set the initially allocated size +- int code_req = initial_code_capacity; +- int locs_req = initial_locs_capacity; +- int stub_req = initial_stub_capacity; +- int const_req = initial_const_capacity; ++ const_req = initial_const_capacity; + +- int pad_req = NativeCall::instruction_size; + // The extra spacing after the code is necessary on some platforms. + // Sometimes we need to patch in a jump after the last instruction, + // if the nmethod has been deoptimized. (See 4932387, 4894843.) +@@ -972,7 +974,7 @@ CodeBuffer* Compile::init_buffer(uint* blk_starts) { + + // Compute prolog code size + _method_size = 0; +- _frame_slots = OptoReg::reg2stack(_matcher->_old_SP)+_regalloc->_framesize; ++ _frame_slots = OptoReg::reg2stack(_matcher->_old_SP) + _regalloc->_framesize; + #if defined(IA64) && !defined(AIX) + if (save_argument_registers()) { + // 4815101: this is a stub with implicit and unknown precision fp args. +@@ -1021,12 +1023,18 @@ CodeBuffer* Compile::init_buffer(uint* blk_starts) { + // Initialize the space for the BufferBlob used to find and verify + // instruction size in MachNode::emit_size() + init_scratch_buffer_blob(const_req); +- if (failing()) return NULL; // Out of memory ++} + +- // Pre-compute the length of blocks and replace +- // long branches with short if machine supports it. +- shorten_branches(blk_starts, code_req, locs_req, stub_req); ++CodeBuffer* Compile::init_buffer(BufferSizingData& buf_sizes) { + ++ int stub_req = buf_sizes._stub; ++ int code_req = buf_sizes._code; ++ int const_req = buf_sizes._const; ++ ++ int pad_req = NativeCall::instruction_size; ++ ++ BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); ++ stub_req += bs->estimate_stub_size(); + // nmethod and CodeBuffer count stubs & constants as part of method's code. + // class HandlerImpl is platform-specific and defined in the *.ad files. + int exception_handler_req = HandlerImpl::size_exception_handler() + MAX_stubs_size; // add marginal slop for handler +@@ -1038,18 +1046,18 @@ CodeBuffer* Compile::init_buffer(uint* blk_starts) { + code_req = const_req = stub_req = exception_handler_req = deopt_handler_req = 0x10; // force expansion + + int total_req = +- const_req + +- code_req + +- pad_req + +- stub_req + +- exception_handler_req + +- deopt_handler_req; // deopt handler ++ const_req + ++ code_req + ++ pad_req + ++ stub_req + ++ exception_handler_req + ++ deopt_handler_req; // deopt handler + + if (has_method_handle_invokes()) + total_req += deopt_handler_req; // deopt MH handler + + CodeBuffer* cb = code_buffer(); +- cb->initialize(total_req, locs_req); ++ cb->initialize(total_req, buf_sizes._reloc); + + // Have we run out of code space? + if ((cb->blob() == NULL) || (!CompileBroker::should_compile_new_jobs())) { +@@ -1263,12 +1271,12 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) { + Process_OopMap_Node(mach, current_offset); + } // End if safepoint + +- // If this is a null check, then add the start of the previous instruction to the list ++ // If this is a null check, then add the start of the previous instruction to the list + else if( mach->is_MachNullCheck() ) { + inct_starts[inct_cnt++] = previous_offset; + } + +- // If this is a branch, then fill in the label with the target BB's label ++ // If this is a branch, then fill in the label with the target BB's label + else if (mach->is_MachBranch()) { + // This requires the TRUE branch target be in succs[0] + uint block_num = block->non_connector_successor(0)->_pre_order; +@@ -1279,8 +1287,8 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) { + bool delay_slot_is_used = valid_bundle_info(n) && + node_bundling(n)->use_unconditional_delay(); + if (!delay_slot_is_used && mach->may_be_short_branch()) { +- assert(delay_slot == NULL, "not expecting delay slot node"); +- int br_size = n->size(_regalloc); ++ assert(delay_slot == NULL, "not expecting delay slot node"); ++ int br_size = n->size(_regalloc); + int offset = blk_starts[block_num] - current_offset; + if (block_num >= i) { + // Current and following block's offset are not +@@ -1338,7 +1346,7 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) { + } + } + #ifdef ASSERT +- // Check that oop-store precedes the card-mark ++ // Check that oop-store precedes the card-mark + else if (mach->ideal_Opcode() == Op_StoreCM) { + uint storeCM_idx = j; + int count = 0; +@@ -1507,6 +1515,10 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) { + } + #endif + ++ BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); ++ bs->emit_stubs(*cb); ++ if (failing()) return; ++ + #ifndef PRODUCT + // Information on the size of the method, without the extraneous code + Scheduling::increment_method_size(cb->insts_size()); +@@ -1671,20 +1683,20 @@ uint Scheduling::_total_instructions_per_bundle[Pipeline::_max_instrs_per_cycle+ + // Initializer for class Scheduling + + Scheduling::Scheduling(Arena *arena, Compile &compile) +- : _arena(arena), +- _cfg(compile.cfg()), +- _regalloc(compile.regalloc()), +- _reg_node(arena), +- _bundle_instr_count(0), +- _bundle_cycle_number(0), +- _scheduled(arena), +- _available(arena), +- _next_node(NULL), +- _bundle_use(0, 0, resource_count, &_bundle_use_elements[0]), +- _pinch_free_list(arena) ++ : _arena(arena), ++ _cfg(compile.cfg()), ++ _regalloc(compile.regalloc()), ++ _reg_node(arena), ++ _bundle_instr_count(0), ++ _bundle_cycle_number(0), ++ _scheduled(arena), ++ _available(arena), ++ _next_node(NULL), ++ _bundle_use(0, 0, resource_count, &_bundle_use_elements[0]), ++ _pinch_free_list(arena) + #ifndef PRODUCT +- , _branches(0) +- , _unconditional_delays(0) ++ , _branches(0) ++ , _unconditional_delays(0) + #endif + { + // Create a MachNopNode +@@ -1763,8 +1775,8 @@ void Scheduling::step_and_clear() { + _bundle_use.reset(); + + memcpy(_bundle_use_elements, +- Pipeline_Use::elaborated_elements, +- sizeof(Pipeline_Use::elaborated_elements)); ++ Pipeline_Use::elaborated_elements, ++ sizeof(Pipeline_Use::elaborated_elements)); + } + + // Perform instruction scheduling and bundling over the sequence of +@@ -1791,6 +1803,22 @@ void Compile::ScheduleAndBundle() { + // Walk backwards over each basic block, computing the needed alignment + // Walk over all the basic blocks + scheduling.DoScheduling(); ++ ++#ifndef PRODUCT ++ if (trace_opto_output()) { ++ tty->print("\n---- After ScheduleAndBundle ----\n"); ++ for (uint i = 0; i < _cfg->number_of_blocks(); i++) { ++ tty->print("\nBB#%03d:\n", i); ++ Block* block = _cfg->get_block(i); ++ for (uint j = 0; j < block->number_of_nodes(); j++) { ++ Node* n = block->get_node(j); ++ OptoReg::Name reg = _regalloc->get_reg_first(n); ++ tty->print(" %-6s ", reg >= 0 && reg < REG_COUNT ? Matcher::regName[reg] : ""); ++ n->dump(); ++ } ++ } ++ } ++#endif + } + + // Compute the latency of all the instructions. This is fairly simple, +@@ -1859,7 +1887,7 @@ bool Scheduling::NodeFitsInBundle(Node *n) { + #ifndef PRODUCT + if (_cfg->C->trace_opto_output()) + tty->print("# NodeFitsInBundle [%4d]: FALSE; latency %4d > %d\n", +- n->_idx, _current_latency[n_idx], _bundle_cycle_number); ++ n->_idx, _current_latency[n_idx], _bundle_cycle_number); + #endif + return (false); + } +@@ -2084,12 +2112,12 @@ void Scheduling::AddNodeToBundle(Node *n, const Block *bb) { + // Don't allow safepoints in the branch shadow, that will + // cause a number of difficulties + if ( avail_pipeline->instructionCount() == 1 && +- !avail_pipeline->hasMultipleBundles() && +- !avail_pipeline->hasBranchDelay() && +- Pipeline::instr_has_unit_size() && +- d->size(_regalloc) == Pipeline::instr_unit_size() && +- NodeFitsInBundle(d) && +- !node_bundling(d)->used_in_delay()) { ++ !avail_pipeline->hasMultipleBundles() && ++ !avail_pipeline->hasBranchDelay() && ++ Pipeline::instr_has_unit_size() && ++ d->size(_regalloc) == Pipeline::instr_unit_size() && ++ NodeFitsInBundle(d) && ++ !node_bundling(d)->used_in_delay()) { + + if (d->is_Mach() && !d->is_MachSafePoint()) { + // A node that fits in the delay slot was found, so we need to +@@ -2134,13 +2162,13 @@ void Scheduling::AddNodeToBundle(Node *n, const Block *bb) { + // step of the bundles + if (!NodeFitsInBundle(n)) { + #ifndef PRODUCT +- if (_cfg->C->trace_opto_output()) +- tty->print("# *** STEP(branch won't fit) ***\n"); ++ if (_cfg->C->trace_opto_output()) ++ tty->print("# *** STEP(branch won't fit) ***\n"); + #endif + // Update the state information +- _bundle_instr_count = 0; +- _bundle_cycle_number += 1; +- _bundle_use.step(1); ++ _bundle_instr_count = 0; ++ _bundle_cycle_number += 1; ++ _bundle_use.step(1); + } + } + +@@ -2186,8 +2214,8 @@ void Scheduling::AddNodeToBundle(Node *n, const Block *bb) { + #ifndef PRODUCT + if (_cfg->C->trace_opto_output()) + tty->print("# *** STEP(%d >= %d instructions) ***\n", +- instruction_count + _bundle_instr_count, +- Pipeline::_max_instrs_per_cycle); ++ instruction_count + _bundle_instr_count, ++ Pipeline::_max_instrs_per_cycle); + #endif + step(1); + } +@@ -2393,7 +2421,7 @@ void Scheduling::DoScheduling() { + } + assert(!last->is_Mach() || last->as_Mach()->ideal_Opcode() != Op_Con, ""); + if( last->is_Catch() || +- (last->is_Mach() && last->as_Mach()->ideal_Opcode() == Op_Halt) ) { ++ (last->is_Mach() && last->as_Mach()->ideal_Opcode() == Op_Halt) ) { + // There might be a prior call. Skip it. + while (_bb_start < _bb_end && bb->get_node(--_bb_end)->is_MachProj()); + } else if( last->is_MachNullCheck() ) { +@@ -2463,7 +2491,7 @@ void Scheduling::DoScheduling() { + } + #endif + #ifdef ASSERT +- verify_good_schedule(bb,"after block local scheduling"); ++ verify_good_schedule(bb,"after block local scheduling"); + #endif + } + +@@ -2811,31 +2839,31 @@ void Scheduling::ComputeRegisterAntidependencies(Block *b) { + // + void Scheduling::garbage_collect_pinch_nodes() { + #ifndef PRODUCT +- if (_cfg->C->trace_opto_output()) tty->print("Reclaimed pinch nodes:"); +-#endif +- int trace_cnt = 0; +- for (uint k = 0; k < _reg_node.Size(); k++) { +- Node* pinch = _reg_node[k]; +- if ((pinch != NULL) && pinch->Opcode() == Op_Node && +- // no predecence input edges +- (pinch->req() == pinch->len() || pinch->in(pinch->req()) == NULL) ) { +- cleanup_pinch(pinch); +- _pinch_free_list.push(pinch); +- _reg_node.map(k, NULL); ++ if (_cfg->C->trace_opto_output()) tty->print("Reclaimed pinch nodes:"); ++#endif ++ int trace_cnt = 0; ++ for (uint k = 0; k < _reg_node.Size(); k++) { ++ Node* pinch = _reg_node[k]; ++ if ((pinch != NULL) && pinch->Opcode() == Op_Node && ++ // no predecence input edges ++ (pinch->req() == pinch->len() || pinch->in(pinch->req()) == NULL) ) { ++ cleanup_pinch(pinch); ++ _pinch_free_list.push(pinch); ++ _reg_node.map(k, NULL); + #ifndef PRODUCT +- if (_cfg->C->trace_opto_output()) { +- trace_cnt++; +- if (trace_cnt > 40) { +- tty->print("\n"); +- trace_cnt = 0; +- } +- tty->print(" %d", pinch->_idx); ++ if (_cfg->C->trace_opto_output()) { ++ trace_cnt++; ++ if (trace_cnt > 40) { ++ tty->print("\n"); ++ trace_cnt = 0; + } +-#endif ++ tty->print(" %d", pinch->_idx); + } ++#endif + } ++ } + #ifndef PRODUCT +- if (_cfg->C->trace_opto_output()) tty->print("\n"); ++ if (_cfg->C->trace_opto_output()) tty->print("\n"); + #endif + } + +@@ -2872,19 +2900,19 @@ void Scheduling::dump_available() const { + void Scheduling::print_statistics() { + // Print the size added by nops for bundling + tty->print("Nops added %d bytes to total of %d bytes", +- _total_nop_size, _total_method_size); ++ _total_nop_size, _total_method_size); + if (_total_method_size > 0) + tty->print(", for %.2f%%", +- ((double)_total_nop_size) / ((double) _total_method_size) * 100.0); ++ ((double)_total_nop_size) / ((double) _total_method_size) * 100.0); + tty->print("\n"); + + // Print the number of branch shadows filled + if (Pipeline::_branch_has_delay_slot) { + tty->print("Of %d branches, %d had unconditional delay slots filled", +- _total_branches, _total_unconditional_delays); ++ _total_branches, _total_unconditional_delays); + if (_total_branches > 0) + tty->print(", for %.2f%%", +- ((double)_total_unconditional_delays) / ((double)_total_branches) * 100.0); ++ ((double)_total_unconditional_delays) / ((double)_total_branches) * 100.0); + tty->print("\n"); + } + +@@ -2898,6 +2926,6 @@ void Scheduling::print_statistics() { + + if (total_bundles > 0) + tty->print("Average ILP (excluding nops) is %.2f\n", +- ((double)total_instructions) / ((double)total_bundles)); ++ ((double)total_instructions) / ((double)total_bundles)); + } + #endif +diff --git a/src/hotspot/share/opto/output.hpp b/src/hotspot/share/opto/output.hpp +index ab3c1a304..ec3cc2981 100644 +--- a/src/hotspot/share/opto/output.hpp ++++ b/src/hotspot/share/opto/output.hpp +@@ -49,10 +49,7 @@ class Pipeline_Use; + + // Define the initial sizes for allocation of the resizable code buffer + enum { +- initial_code_capacity = 16 * 1024, +- initial_stub_capacity = 4 * 1024, +- initial_const_capacity = 4 * 1024, +- initial_locs_capacity = 3 * 1024 ++ initial_const_capacity = 4 * 1024 + }; + + //------------------------------Scheduling---------------------------------- +diff --git a/src/hotspot/share/opto/phaseX.cpp b/src/hotspot/share/opto/phaseX.cpp +index 880efe991..4ea04b63c 100644 +--- a/src/hotspot/share/opto/phaseX.cpp ++++ b/src/hotspot/share/opto/phaseX.cpp +@@ -1655,14 +1655,14 @@ void PhaseIterGVN::add_users_to_worklist( Node *n ) { + // of the mirror load depends on the type of 'n'. See LoadNode::Value(). + // LoadBarrier?(LoadP(LoadP(AddP(foo:Klass, #java_mirror)))) + BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); +- bool has_load_barriers = bs->has_load_barriers(); ++ bool has_load_barrier_nodes = bs->has_load_barrier_nodes(); + + if (use_op == Op_LoadP && use->bottom_type()->isa_rawptr()) { + for (DUIterator_Fast i2max, i2 = use->fast_outs(i2max); i2 < i2max; i2++) { + Node* u = use->fast_out(i2); + const Type* ut = u->bottom_type(); + if (u->Opcode() == Op_LoadP && ut->isa_instptr()) { +- if (has_load_barriers) { ++ if (has_load_barrier_nodes) { + // Search for load barriers behind the load + for (DUIterator_Fast i3max, i3 = u->fast_outs(i3max); i3 < i3max; i3++) { + Node* b = u->fast_out(i3); +@@ -1813,14 +1813,14 @@ void PhaseCCP::analyze() { + // Loading the java mirror from a Klass requires two loads and the type + // of the mirror load depends on the type of 'n'. See LoadNode::Value(). + BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); +- bool has_load_barriers = bs->has_load_barriers(); ++ bool has_load_barrier_nodes = bs->has_load_barrier_nodes(); + + if (m_op == Op_LoadP && m->bottom_type()->isa_rawptr()) { + for (DUIterator_Fast i2max, i2 = m->fast_outs(i2max); i2 < i2max; i2++) { + Node* u = m->fast_out(i2); + const Type* ut = u->bottom_type(); + if (u->Opcode() == Op_LoadP && ut->isa_instptr() && ut != type(u)) { +- if (has_load_barriers) { ++ if (has_load_barrier_nodes) { + // Search for load barriers behind the load + for (DUIterator_Fast i3max, i3 = u->fast_outs(i3max); i3 < i3max; i3++) { + Node* b = u->fast_out(i3); +diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp +index b40587ba5..e0bf8410a 100644 +--- a/src/hotspot/share/opto/vectornode.cpp ++++ b/src/hotspot/share/opto/vectornode.cpp +@@ -252,7 +252,6 @@ void VectorNode::vector_operands(Node* n, uint* start, uint* end) { + case Op_LoadI: case Op_LoadL: + case Op_LoadF: case Op_LoadD: + case Op_LoadP: case Op_LoadN: +- case Op_LoadBarrierSlowReg: + *start = 0; + *end = 0; // no vector operands + break; +-- +2.12.3 + diff --git a/ZGC-aarch64-not-using-zr-register-avoid-sigill-in-Ma.patch b/ZGC-aarch64-not-using-zr-register-avoid-sigill-in-Ma.patch new file mode 100644 index 0000000..bab06da --- /dev/null +++ b/ZGC-aarch64-not-using-zr-register-avoid-sigill-in-Ma.patch @@ -0,0 +1,97 @@ +From 425112071e77e2fb599d1f96ce48689d45461261 Mon Sep 17 00:00:00 2001 +Date: Mon, 17 Feb 2020 18:55:47 +0800 +Subject: [PATCH] ZGC: aarch64: not using zr register avoid sigill in + MacroAssembler::push_fp and pop_fp + +Summary: : +LLT: jtreg +Bug url: NA +--- + src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp | 46 +++++++++++++--------- + 1 file changed, 24 insertions(+), 22 deletions(-) +diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +index 611f13b0e..6db979b57 100644 +--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp ++++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +@@ -2100,57 +2100,59 @@ int MacroAssembler::pop(unsigned int bitset, Register stack) { + // Push lots of registers in the bit set supplied. Don't push sp. + // Return the number of words pushed + int MacroAssembler::push_fp(unsigned int bitset, Register stack) { +- int words_pushed = 0; +- + // Scan bitset to accumulate register pairs + unsigned char regs[32]; + int count = 0; ++ int i = 0; + for (int reg = 0; reg <= 31; reg++) { + if (1 & bitset) + regs[count++] = reg; + bitset >>= 1; + } +- regs[count++] = zr->encoding_nocheck(); +- count &= ~1; // Only push an even number of regs + +- // Always pushing full 128 bit registers. +- if (count) { +- stpq(as_FloatRegister(regs[0]), as_FloatRegister(regs[1]), Address(pre(stack, -count * wordSize * 2))); +- words_pushed += 2; ++ if (!count) { ++ return 0; + } +- for (int i = 2; i < count; i += 2) { ++ ++ add(stack, stack, -count * wordSize * 2); ++ ++ if (count & 1) { ++ strq(as_FloatRegister(regs[0]), Address(stack)); ++ i += 1; ++ } ++ ++ for (; i < count; i += 2) { + stpq(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize * 2)); +- words_pushed += 2; + } + +- assert(words_pushed == count, "oops, pushed != count"); + return count; + } + + int MacroAssembler::pop_fp(unsigned int bitset, Register stack) { +- int words_pushed = 0; +- + // Scan bitset to accumulate register pairs + unsigned char regs[32]; + int count = 0; ++ int i = 0; + for (int reg = 0; reg <= 31; reg++) { + if (1 & bitset) + regs[count++] = reg; + bitset >>= 1; + } +- regs[count++] = zr->encoding_nocheck(); +- count &= ~1; + +- for (int i = 2; i < count; i += 2) { +- ldpq(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize * 2)); +- words_pushed += 2; ++ if (!count) { ++ return 0; + } +- if (count) { +- ldpq(as_FloatRegister(regs[0]), as_FloatRegister(regs[1]), Address(post(stack, count * wordSize * 2))); +- words_pushed += 2; ++ ++ if (count & 1) { ++ ldrq(as_FloatRegister(regs[0]), Address(stack)); ++ i += 1; + } + +- assert(words_pushed == count, "oops, pushed != count"); ++ for (; i < count; i += 2) { ++ ldpq(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize * 2)); ++ } ++ ++ add(stack, stack, count * wordSize * 2); + + return count; + } diff --git a/change-vendor-to-openEuler_Community.patch b/change-vendor-to-openEuler_Community.patch new file mode 100644 index 0000000..f43d132 --- /dev/null +++ b/change-vendor-to-openEuler_Community.patch @@ -0,0 +1,54 @@ +From 1ab27951c430bbe0820321c7194e89f63b9ac78b Mon Sep 17 00:00:00 2001 +Date: Tue, 30 Jul 2019 14:20:57 +0000 +Subject: [PATCH] change vendor to Huawei Technologies Co., LTD + +Summary: change VENDOR to openEuler Community, as + well as VENDOR_URL & VENDOR_URL_BUG +LLT: java -XshowSettings:properties +Bug url: NA +--- + src/hotspot/share/runtime/vm_version.cpp | 6 +----- + src/java.base/share/native/libjava/System.c | 6 +++--- + 2 files changed, 4 insertions(+), 8 deletions(-) + +diff --git a/src/hotspot/share/runtime/vm_version.cpp b/src/hotspot/share/runtime/vm_version.cpp +index eb05460ad..bb070895f 100644 +--- a/src/hotspot/share/runtime/vm_version.cpp ++++ b/src/hotspot/share/runtime/vm_version.cpp +@@ -114,11 +114,7 @@ const char* Abstract_VM_Version::vm_name() { + + + const char* Abstract_VM_Version::vm_vendor() { +-#ifdef VENDOR +- return VENDOR; +-#else +- return "Oracle Corporation"; +-#endif ++ return "openEuler Community"; + } + + +diff --git a/src/java.base/share/native/libjava/System.c b/src/java.base/share/native/libjava/System.c +index 3a33f1179..65a44d5a1 100644 +--- a/src/java.base/share/native/libjava/System.c ++++ b/src/java.base/share/native/libjava/System.c +@@ -110,13 +110,13 @@ Java_java_lang_System_identityHashCode(JNIEnv *env, jobject this, jobject x) + + /* Third party may overwrite these values. */ + #ifndef VENDOR +-#define VENDOR "Oracle Corporation" ++#define VENDOR "openEuler Community" + #endif + #ifndef VENDOR_URL +-#define VENDOR_URL "http://java.oracle.com/" ++#define VENDOR_URL "https://openeuler.org/" + #endif + #ifndef VENDOR_URL_BUG +-#define VENDOR_URL_BUG "http://bugreport.java.com/bugreport/" ++#define VENDOR_URL_BUG "https://openeuler.org/" + #endif + + #ifdef JAVA_SPECIFICATION_VENDOR /* Third party may NOT overwrite this. */ +-- +2.19.0 + diff --git a/freetype-seeks-to-index-at-the-end-of-the-fo.patch b/freetype-seeks-to-index-at-the-end-of-the-fo.patch new file mode 100644 index 0000000..99bd08c --- /dev/null +++ b/freetype-seeks-to-index-at-the-end-of-the-fo.patch @@ -0,0 +1,84 @@ +From f8e686fda7c681d1b27e266d3f954d6441aee83a Mon Sep 17 00:00:00 2001 +Date: Thu, 13 Feb 2020 19:07:20 +0000 +Subject: [PATCH] 8227662: freetype seeks to index at the end of the font data + +Summary: : freetype seeks to index at the end of the font data +LLT: test/jdk/java/awt/FontMetrics/SpaceAdvance.java +Bug url: https://bugs.openjdk.java.net/browse/JDK-8237381 +--- + .../share/native/libfontmanager/freetypeScaler.c | 2 +- + test/jdk/java/awt/FontMetrics/SpaceAdvance.java | 49 ++++++++++++++++++++++ + 2 files changed, 50 insertions(+), 1 deletion(-) + create mode 100644 test/jdk/java/awt/FontMetrics/SpaceAdvance.java + +diff --git a/src/java.desktop/share/native/libfontmanager/freetypeScaler.c b/src/java.desktop/share/native/libfontmanager/freetypeScaler.c +index bca003d31..cc8432866 100644 +--- a/src/java.desktop/share/native/libfontmanager/freetypeScaler.c ++++ b/src/java.desktop/share/native/libfontmanager/freetypeScaler.c +@@ -161,7 +161,7 @@ static unsigned long ReadTTFontFileFunc(FT_Stream stream, + */ + + if (numBytes == 0) { +- if (offset >= scalerInfo->fileSize) { ++ if (offset > scalerInfo->fileSize) { + return -1; + } else { + return 0; +diff --git a/test/jdk/java/awt/FontMetrics/SpaceAdvance.java b/test/jdk/java/awt/FontMetrics/SpaceAdvance.java +new file mode 100644 +index 000000000..e2c7acb6f +--- /dev/null ++++ b/test/jdk/java/awt/FontMetrics/SpaceAdvance.java +@@ -0,0 +1,49 @@ ++/* ++ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. ++ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++ * ++ * This code is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License version 2 only, as ++ * published by the Free Software Foundation. ++ * ++ * This code is distributed in the hope that it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++ * version 2 for more details (a copy is included in the LICENSE file that ++ * accompanied this code). ++ * ++ * You should have received a copy of the GNU General Public License version ++ * 2 along with this work; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++ * or visit www.oracle.com if you need additional information or have any ++ * questions. ++ */ ++ ++/* ++ * @test ++ * @bug 8227662 ++ */ ++ ++import java.awt.Font; ++import java.awt.FontMetrics ; ++import java.awt.Graphics2D; ++import java.awt.image.BufferedImage; ++ ++public class SpaceAdvance { ++ public static void main(String[] args) throws Exception { ++ ++ BufferedImage bi = new BufferedImage(1,1,1); ++ Graphics2D g2d = bi.createGraphics(); ++ Font font = new Font(Font.DIALOG, Font.PLAIN, 12); ++ if (!font.canDisplay(' ')) { ++ return; ++ } ++ g2d.setFont(font); ++ FontMetrics fm = g2d.getFontMetrics(); ++ if (fm.charWidth(' ') == 0) { ++ throw new RuntimeException("Space has char width of 0"); ++ } ++ } ++} +-- +2.12.3 + diff --git a/java-11-openjdk.spec b/java-11-openjdk.spec new file mode 100644 index 0000000..23dadeb --- /dev/null +++ b/java-11-openjdk.spec @@ -0,0 +1,1736 @@ +# RPM conditionals so as to be able to dynamically produce +# slowdebug/release builds. See: +# http://rpm.org/user_doc/conditional_builds.html +# +# Examples: +# +# Produce release *and* slowdebug builds on x86_64 (default): +# $ rpmbuild -ba java-1.8.0-openjdk.spec +# +# Produce only release builds (no slowdebug builds) on x86_64: +# $ rpmbuild -ba java-1.8.0-openjdk.spec --without slowdebug +# +# Only produce a release build on x86_64: +# $ fedpkg mockbuild --without slowdebug +# +# Only produce a debug build on x86_64: +# $ fedpkg local --without release +# +# Enable slowdebug builds by default on relevant arches. +%bcond_without slowdebug +# Enable release builds by default on relevant arches. +%bcond_without release + +# note: parametrized macros are order-sensitive (unlike not-parametrized) even with normal macros +# also necessary when passing it as parameter to other macros. If not macro, then it is considered a switch +# see the difference between global and define: +# See https://github.com/rpm-software-management/rpm/issues/127 to comments at "pmatilai commented on Aug 18, 2017" +# (initiated in https://bugzilla.redhat.com/show_bug.cgi?id=1482192) +%global debug_suffix_unquoted -slowdebug +# quoted one for shell operations +%global debug_suffix "%{debug_suffix_unquoted}" +%global normal_suffix "" + +# if you want only debug build but providing java build only normal build but set normalbuild_parameter +%global debug_warning This package has full debug on. Install only in need and remove asap. +%global debug_on with full debug on +%global for_debug for packages with debug on + +%if %{with release} +%global include_normal_build 1 +%else +%global include_normal_build 0 +%endif + +%if %{include_normal_build} +%global build_loop1 %{normal_suffix} +%else +%global build_loop1 %{nil} +%endif + +%global aarch64 aarch64 arm64 armv8 +# we need to distinguish between big and little endian PPC64 +%global ppc64le ppc64le +%global ppc64be ppc64 ppc64p7 +%global multilib_arches %{power64} sparc64 x86_64 +%global jit_arches %{ix86} x86_64 sparcv9 sparc64 %{aarch64} %{power64} %{arm} s390x +%global aot_arches x86_64 %{aarch64} + +# By default, we build a debug build during main build on JIT architectures +%if %{with slowdebug} +%ifarch %{jit_arches} +%ifnarch %{arm} +%global include_debug_build 1 +%else +%global include_debug_build 0 +%endif +%else +%global include_debug_build 0 +%endif +%else +%global include_debug_build 0 +%endif + +# On x86_64 and AArch64, we use the Shenandoah HotSpot +%ifarch x86_64 %{aarch64} +%global use_shenandoah_hotspot 1 +%else +%global use_shenandoah_hotspot 0 +%endif + +%if %{include_debug_build} +%global build_loop2 %{debug_suffix} +%else +%global build_loop2 %{nil} +%endif + +# if you disable both builds, then the build fails +%global build_loop %{build_loop1} %{build_loop2} +# note: that order: normal_suffix debug_suffix, in case of both enabled +# is expected in one single case at the end of the build +%global rev_build_loop %{build_loop2} %{build_loop1} + +%ifarch %{jit_arches} +%global bootstrap_build 1 +%else +%global bootstrap_build 1 +%endif + +%if %{bootstrap_build} +%global targets bootcycle-images all docs +%else +%global targets all docs +%endif + + +# Filter out flags from the optflags macro that cause problems with the OpenJDK build +# We filter out -O flags so that the optimization of HotSpot is not lowered from O3 to O2 +# We filter out -Wall which will otherwise cause HotSpot to produce hundreds of thousands of warnings (100+mb logs) +# We replace it with -Wformat (required by -Werror=format-security) and -Wno-cpp to avoid FORTIFY_SOURCE warnings +# We filter out -fexceptions as the HotSpot build explicitly does -fno-exceptions and it's otherwise the default for C++ +%global ourflags %(echo %optflags | sed -e 's|-Wall|-Wformat -Wno-cpp|' | sed -r -e 's|-O[0-9]*||') +%global ourcppflags %(echo %ourflags | sed -e 's|-fexceptions||') +%global ourldflags %{__global_ldflags} + +# With disabled nss is NSS deactivated, so NSS_LIBDIR can contain the wrong path +# the initialization must be here. Later the pkg-config have buggy behavior +# looks like openjdk RPM specific bug +# Always set this so the nss.cfg file is not broken +%global NSS_LIBDIR %(pkg-config --variable=libdir nss) +%global NSS_LIBS %(pkg-config --libs nss) +%global NSS_CFLAGS %(pkg-config --cflags nss-softokn) +# see https://bugzilla.redhat.com/show_bug.cgi?id=1332456 +%global NSSSOFTOKN_BUILDTIME_NUMBER %(pkg-config --modversion nss-softokn || : ) +%global NSS_BUILDTIME_NUMBER %(pkg-config --modversion nss || : ) +# this is workaround for processing of requires during srpm creation +%global NSSSOFTOKN_BUILDTIME_VERSION %(if [ "x%{NSSSOFTOKN_BUILDTIME_NUMBER}" == "x" ] ; then echo "" ;else echo ">= %{NSSSOFTOKN_BUILDTIME_NUMBER}" ;fi) +%global NSS_BUILDTIME_VERSION %(if [ "x%{NSS_BUILDTIME_NUMBER}" == "x" ] ; then echo "" ;else echo ">= %{NSS_BUILDTIME_NUMBER}" ;fi) + + +# fix for https://bugzilla.redhat.com/show_bug.cgi?id=1111349 +%global _privatelibs libsplashscreen[.]so.*|libawt_xawt[.]so.*|libjli[.]so.*|libattach[.]so.*|libawt[.]so.*|libextnet[.]so.*|libawt_headless[.]so.*|libdt_socket[.]so.*|libfontmanager[.]so.*|libinstrument[.]so.*|libj2gss[.]so.*|libj2pcsc[.]so.*|libj2pkcs11[.]so.*|libjaas[.]so.*|libjavajpeg[.]so.*|libjdwp[.]so.*|libjimage[.]so.*|libjsound[.]so.*|liblcms[.]so.*|libmanagement[.]so.*|libmanagement_agent[.]so.*|libmanagement_ext[.]so.*|libmlib_image[.]so.*|libnet[.]so.*|libnio[.]so.*|libprefs[.]so.*|librmi[.]so.*|libsaproc[.]so.*|libsctp[.]so.*|libsunec[.]so.*|libunpack[.]so.*|libzip[.]so.* + +%global __provides_exclude ^(%{_privatelibs})$ +%global __requires_exclude ^(%{_privatelibs})$ + +# In some cases, the arch used by the JDK does +# not match _arch. +# Also, in some cases, the machine name used by SystemTap +# does not match that given by _build_cpu +%ifarch x86_64 +%global archinstall amd64 +%endif +%ifarch ppc +%global archinstall ppc +%endif +%ifarch %{ppc64be} +%global archinstall ppc64 +%endif +%ifarch %{ppc64le} +%global archinstall ppc64le +%endif +%ifarch %{ix86} +%global archinstall i686 +%endif +%ifarch ia64 +%global archinstall ia64 +%endif +%ifarch s390 +%global archinstall s390 +%endif +%ifarch s390x +%global archinstall s390x +%endif +%ifarch %{arm} +%global archinstall arm +%endif +%ifarch %{aarch64} +%global archinstall aarch64 +%endif +# 32 bit sparc, optimized for v9 +%ifarch sparcv9 +%global archinstall sparc +%endif +# 64 bit sparc +%ifarch sparc64 +%global archinstall sparcv9 +%endif +%ifnarch %{jit_arches} +%global archinstall %{_arch} +%endif + + + +%ifarch %{jit_arches} +%global with_systemtap 1 +%else +%global with_systemtap 0 +%endif + +# New Version-String scheme-style defines +%global majorver 11 + +# Standard JPackage naming and versioning defines +%global origin openjdk +%global origin_nice OpenJDK +%global top_level_dir_name %{origin} +%global minorver 0 +%global buildver 28 +%global updatever 6 +# priority must be 7 digits in total +# setting to 1, so debug ones can have 0 +%global priority 00000%{minorver}1 +%global fulljavaver %{majorver}.%{minorver}.%{updatever} + +%global javaver %{majorver} + +# parametrized macros are order-sensitive +%global compatiblename java-%{majorver}-%{origin} +%global fullversion %{compatiblename}-%{version}-%{release} +# images stub +%global jdkimage jdk +# output dir stub +%define buildoutputdir() %{expand:openjdk/build%{?1}} +# we can copy the javadoc to not arched dir, or make it not noarch +%define uniquejavadocdir() %{expand:%{fullversion}%{?1}} +# main id and dir of this jdk +%define uniquesuffix() %{expand:%{fullversion}.%{_arch}%{?1}} + +%global etcjavasubdir %{_sysconfdir}/java/java-%{javaver}-%{origin} +%define etcjavadir() %{expand:%{etcjavasubdir}/%{uniquesuffix -- %{?1}}} +# Standard JPackage directories and symbolic links. +%define sdkdir() %{expand:%{uniquesuffix -- %{?1}}} +%define jrelnk() %{expand:jre-%{javaver}-%{origin}-%{version}-%{release}.%{_arch}%{?1}} + +%define sdkbindir() %{expand:%{_jvmdir}/%{sdkdir -- %{?1}}/bin} +%define jrebindir() %{expand:%{_jvmdir}/%{sdkdir -- %{?1}}/bin} + +%global rpm_state_dir %{_localstatedir}/lib/rpm-state/ + +%if %{with_systemtap} +# Where to install systemtap tapset (links) +# We would like these to be in a package specific sub-dir, +# but currently systemtap doesn't support that, so we have to +# use the root tapset dir for now. To distinguish between 64 +# and 32 bit architectures we place the tapsets under the arch +# specific dir (note that systemtap will only pickup the tapset +# for the primary arch for now). Systemtap uses the machine name +# aka build_cpu as architecture specific directory name. +%global tapsetroot /usr/share/systemtap +%global tapsetdirttapset %{tapsetroot}/tapset/ +%global tapsetdir %{tapsetdirttapset}/%{_build_cpu} +%endif + +# not-duplicated scriptlets for normal/debug packages +%global update_desktop_icons /usr/bin/gtk-update-icon-cache %{_datadir}/icons/hicolor &>/dev/null || : + + +%define post_script() %{expand: +update-desktop-database %{_datadir}/applications &> /dev/null || : +/bin/touch --no-create %{_datadir}/icons/hicolor &>/dev/null || : +exit 0 +} + + +%define post_headless() %{expand: +%ifarch %{jit_arches} +# MetaspaceShared::generate_vtable_methods not implemented for PPC JIT +%ifnarch %{ppc64le} +# see https://bugzilla.redhat.com/show_bug.cgi?id=513605 +%{jrebindir -- %{?1}}/java -Xshare:dump >/dev/null 2>/dev/null +%endif +%endif + +PRIORITY=%{priority} +if [ "%{?1}" == %{debug_suffix} ]; then + let PRIORITY=PRIORITY-1 +fi + +ext=.gz +alternatives \\ + --install %{_bindir}/java java %{jrebindir -- %{?1}}/java $PRIORITY --family %{name}.%{_arch} \\ + --slave %{_jvmdir}/jre jre %{_jvmdir}/%{sdkdir -- %{?1}} \\ + --slave %{_bindir}/jjs jjs %{jrebindir -- %{?1}}/jjs \\ + --slave %{_bindir}/keytool keytool %{jrebindir -- %{?1}}/keytool \\ + --slave %{_bindir}/pack200 pack200 %{jrebindir -- %{?1}}/pack200 \\ + --slave %{_bindir}/rmid rmid %{jrebindir -- %{?1}}/rmid \\ + --slave %{_bindir}/rmiregistry rmiregistry %{jrebindir -- %{?1}}/rmiregistry \\ + --slave %{_bindir}/unpack200 unpack200 %{jrebindir -- %{?1}}/unpack200 \\ + --slave %{_mandir}/man1/java.1$ext java.1$ext \\ + %{_mandir}/man1/java-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/jjs.1$ext jjs.1$ext \\ + %{_mandir}/man1/jjs-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/keytool.1$ext keytool.1$ext \\ + %{_mandir}/man1/keytool-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/pack200.1$ext pack200.1$ext \\ + %{_mandir}/man1/pack200-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/rmid.1$ext rmid.1$ext \\ + %{_mandir}/man1/rmid-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/rmiregistry.1$ext rmiregistry.1$ext \\ + %{_mandir}/man1/rmiregistry-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/unpack200.1$ext unpack200.1$ext \\ + %{_mandir}/man1/unpack200-%{uniquesuffix -- %{?1}}.1$ext + +for X in %{origin} %{javaver} ; do + alternatives --install %{_jvmdir}/jre-"$X" jre_"$X" %{_jvmdir}/%{sdkdir -- %{?1}} $PRIORITY --family %{name}.%{_arch} +done + +update-alternatives --install %{_jvmdir}/jre-%{javaver}-%{origin} jre_%{javaver}_%{origin} %{_jvmdir}/%{jrelnk -- %{?1}} $PRIORITY --family %{name}.%{_arch} + + +update-desktop-database %{_datadir}/applications &> /dev/null || : +/bin/touch --no-create %{_datadir}/icons/hicolor &>/dev/null || : + +# see pretrans where this file is declared +# also see that pretrans is only for non-debug +if [ ! "%{?1}" == %{debug_suffix} ]; then + if [ -f %{_libexecdir}/copy_jdk_configs_fixFiles.sh ] ; then + sh %{_libexecdir}/copy_jdk_configs_fixFiles.sh %{rpm_state_dir}/%{name}.%{_arch} %{_jvmdir}/%{sdkdir -- %{?1}} + fi +fi + +exit 0 +} + +%define postun_script() %{expand: +update-desktop-database %{_datadir}/applications &> /dev/null || : +if [ $1 -eq 0 ] ; then + /bin/touch --no-create %{_datadir}/icons/hicolor &>/dev/null + %{update_desktop_icons} +fi +exit 0 +} + + +%define postun_headless() %{expand: + alternatives --remove java %{jrebindir -- %{?1}}/java + alternatives --remove jre_%{origin} %{_jvmdir}/%{sdkdir -- %{?1}} + alternatives --remove jre_%{javaver} %{_jvmdir}/%{sdkdir -- %{?1}} + alternatives --remove jre_%{javaver}_%{origin} %{_jvmdir}/%{jrelnk -- %{?1}} +} + +%define posttrans_script() %{expand: +%{update_desktop_icons} +} + +%define post_devel() %{expand: + +PRIORITY=%{priority} +if [ "%{?1}" == %{debug_suffix} ]; then + let PRIORITY=PRIORITY-1 +fi + +ext=.gz +alternatives \\ + --install %{_bindir}/javac javac %{sdkbindir -- %{?1}}/javac $PRIORITY --family %{name}.%{_arch} \\ + --slave %{_jvmdir}/java java_sdk %{_jvmdir}/%{sdkdir -- %{?1}} \\ +%ifarch %{aot_arches} + --slave %{_bindir}/jaotc jaotc %{sdkbindir -- %{?1}}/jaotc \\ +%endif + --slave %{_bindir}/jlink jlink %{sdkbindir -- %{?1}}/jlink \\ + --slave %{_bindir}/jmod jmod %{sdkbindir -- %{?1}}/jmod \\ + --slave %{_bindir}/jhsdb jhsdb %{sdkbindir -- %{?1}}/jhsdb \\ + --slave %{_bindir}/jar jar %{sdkbindir -- %{?1}}/jar \\ + --slave %{_bindir}/jarsigner jarsigner %{sdkbindir -- %{?1}}/jarsigner \\ + --slave %{_bindir}/javadoc javadoc %{sdkbindir -- %{?1}}/javadoc \\ + --slave %{_bindir}/javap javap %{sdkbindir -- %{?1}}/javap \\ + --slave %{_bindir}/jcmd jcmd %{sdkbindir -- %{?1}}/jcmd \\ + --slave %{_bindir}/jconsole jconsole %{sdkbindir -- %{?1}}/jconsole \\ + --slave %{_bindir}/jdb jdb %{sdkbindir -- %{?1}}/jdb \\ + --slave %{_bindir}/jdeps jdeps %{sdkbindir -- %{?1}}/jdeps \\ + --slave %{_bindir}/jdeprscan jdeprscan %{sdkbindir -- %{?1}}/jdeprscan \\ + --slave %{_bindir}/jimage jimage %{sdkbindir -- %{?1}}/jimage \\ + --slave %{_bindir}/jinfo jinfo %{sdkbindir -- %{?1}}/jinfo \\ + --slave %{_bindir}/jmap jmap %{sdkbindir -- %{?1}}/jmap \\ + --slave %{_bindir}/jps jps %{sdkbindir -- %{?1}}/jps \\ + --slave %{_bindir}/jrunscript jrunscript %{sdkbindir -- %{?1}}/jrunscript \\ + --slave %{_bindir}/jshell jshell %{sdkbindir -- %{?1}}/jshell \\ + --slave %{_bindir}/jstack jstack %{sdkbindir -- %{?1}}/jstack \\ + --slave %{_bindir}/jstat jstat %{sdkbindir -- %{?1}}/jstat \\ + --slave %{_bindir}/jstatd jstatd %{sdkbindir -- %{?1}}/jstatd \\ + --slave %{_bindir}/rmic rmic %{sdkbindir -- %{?1}}/rmic \\ + --slave %{_bindir}/serialver serialver %{sdkbindir -- %{?1}}/serialver \\ + --slave %{_mandir}/man1/jar.1$ext jar.1$ext \\ + %{_mandir}/man1/jar-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/jarsigner.1$ext jarsigner.1$ext \\ + %{_mandir}/man1/jarsigner-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/javac.1$ext javac.1$ext \\ + %{_mandir}/man1/javac-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/javadoc.1$ext javadoc.1$ext \\ + %{_mandir}/man1/javadoc-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/javap.1$ext javap.1$ext \\ + %{_mandir}/man1/javap-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/jcmd.1$ext jcmd.1$ext \\ + %{_mandir}/man1/jcmd-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/jconsole.1$ext jconsole.1$ext \\ + %{_mandir}/man1/jconsole-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/jdb.1$ext jdb.1$ext \\ + %{_mandir}/man1/jdb-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/jdeps.1$ext jdeps.1$ext \\ + %{_mandir}/man1/jdeps-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/jinfo.1$ext jinfo.1$ext \\ + %{_mandir}/man1/jinfo-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/jmap.1$ext jmap.1$ext \\ + %{_mandir}/man1/jmap-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/jps.1$ext jps.1$ext \\ + %{_mandir}/man1/jps-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/jrunscript.1$ext jrunscript.1$ext \\ + %{_mandir}/man1/jrunscript-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/jstack.1$ext jstack.1$ext \\ + %{_mandir}/man1/jstack-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/jstat.1$ext jstat.1$ext \\ + %{_mandir}/man1/jstat-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/jstatd.1$ext jstatd.1$ext \\ + %{_mandir}/man1/jstatd-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/rmic.1$ext rmic.1$ext \\ + %{_mandir}/man1/rmic-%{uniquesuffix -- %{?1}}.1$ext \\ + --slave %{_mandir}/man1/serialver.1$ext serialver.1$ext \\ + %{_mandir}/man1/serialver-%{uniquesuffix -- %{?1}}.1$ext \\ + +for X in %{origin} %{javaver} ; do + alternatives \\ + --install %{_jvmdir}/java-"$X" java_sdk_"$X" %{_jvmdir}/%{sdkdir -- %{?1}} $PRIORITY --family %{name}.%{_arch} +done + +update-alternatives --install %{_jvmdir}/java-%{javaver}-%{origin} java_sdk_%{javaver}_%{origin} %{_jvmdir}/%{sdkdir -- %{?1}} $PRIORITY --family %{name}.%{_arch} + +update-desktop-database %{_datadir}/applications &> /dev/null || : +/bin/touch --no-create %{_datadir}/icons/hicolor &>/dev/null || : + +exit 0 +} + +%define postun_devel() %{expand: + alternatives --remove javac %{sdkbindir -- %{?1}}/javac + alternatives --remove java_sdk_%{origin} %{_jvmdir}/%{sdkdir -- %{?1}} + alternatives --remove java_sdk_%{javaver} %{_jvmdir}/%{sdkdir -- %{?1}} + alternatives --remove java_sdk_%{javaver}_%{origin} %{_jvmdir}/%{sdkdir -- %{?1}} + +update-desktop-database %{_datadir}/applications &> /dev/null || : + +if [ $1 -eq 0 ] ; then + /bin/touch --no-create %{_datadir}/icons/hicolor &>/dev/null + %{update_desktop_icons} +fi +exit 0 +} + +%define posttrans_devel() %{expand: +%{update_desktop_icons} +} + +%define post_javadoc() %{expand: + +PRIORITY=%{priority} +if [ "%{?1}" == %{debug_suffix} ]; then + let PRIORITY=PRIORITY-1 +fi + +alternatives \\ + --install %{_javadocdir}/java javadocdir %{_javadocdir}/%{uniquejavadocdir -- %{?1}}/api \\ + $PRIORITY --family %{name} +exit 0 +} + +%define postun_javadoc() %{expand: + alternatives --remove javadocdir %{_javadocdir}/%{uniquejavadocdir -- %{?1}}/api +exit 0 +} + +%define post_javadoc_zip() %{expand: + +PRIORITY=%{priority} +if [ "%{?1}" == %{debug_suffix} ]; then + let PRIORITY=PRIORITY-1 +fi + +alternatives \\ + --install %{_javadocdir}/java-zip javadoczip %{_javadocdir}/%{uniquejavadocdir -- %{?1}}.zip \\ + $PRIORITY --family %{name} +exit 0 +} + +%define postun_javadoc_zip() %{expand: + alternatives --remove javadoczip %{_javadocdir}/%{uniquejavadocdir -- %{?1}}.zip +exit 0 +} + +%define files_jre() %{expand: +%{_datadir}/icons/hicolor/*x*/apps/java-%{javaver}-%{origin}.png +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libsplashscreen.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libawt_xawt.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libjawt.so +} + + +%define files_jre_headless() %{expand: +%license %{_jvmdir}/%{sdkdir -- %{?1}}/legal +%dir %{_sysconfdir}/.java/.systemPrefs +%dir %{_sysconfdir}/.java +%dir %{_jvmdir}/%{sdkdir -- %{?1}} +%{_jvmdir}/%{sdkdir -- %{?1}}/release +%{_jvmdir}/%{jrelnk -- %{?1}} +%dir %{_jvmdir}/%{sdkdir -- %{?1}}/bin +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/java +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/jjs +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/keytool +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/pack200 +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/rmid +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/rmiregistry +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/unpack200 +%dir %{_jvmdir}/%{sdkdir -- %{?1}}/lib +%ifarch %{jit_arches} +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/classlist +%endif +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/jexec +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/jrt-fs.jar +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/modules +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/psfont.properties.ja +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/psfontj2d.properties +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/tzdb.dat +%dir %{_jvmdir}/%{sdkdir -- %{?1}}/lib/jli +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/jli/libjli.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/jvm.cfg +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libattach.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libawt.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libextnet.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libjsig.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libawt_headless.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libdt_socket.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libfontmanager.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libinstrument.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libj2gss.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libj2pcsc.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libj2pkcs11.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libjaas.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libjava.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libjavajpeg.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libjdwp.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libjimage.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libjsound.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/liblcms.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libmanagement.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libmanagement_agent.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libmanagement_ext.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libmlib_image.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libnet.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libnio.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libprefs.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/librmi.so +# Zero and S390x don't have SA +%ifarch %{jit_arches} +%ifnarch s390x +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libsaproc.so +%endif +%endif +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libsctp.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libsunec.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libunpack.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libverify.so +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/libzip.so +%dir %{_jvmdir}/%{sdkdir -- %{?1}}/lib/jfr +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/jfr/default.jfc +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/jfr/profile.jfc +%{_mandir}/man1/java-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/jjs-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/keytool-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/pack200-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/rmid-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/rmiregistry-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/unpack200-%{uniquesuffix -- %{?1}}.1* +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/server/ +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/client/ +%ifarch %{jit_arches} +%ifnarch %{power64} +%attr(444, root, root) %ghost %{_jvmdir}/%{sdkdir -- %{?1}}/lib/server/classes.jsa +%attr(444, root, root) %ghost %{_jvmdir}/%{sdkdir -- %{?1}}/lib/client/classes.jsa +%endif +%endif +%dir %{etcjavasubdir} +%dir %{etcjavadir -- %{?1}} +%dir %{etcjavadir -- %{?1}}/lib +%dir %{etcjavadir -- %{?1}}/lib/security +%{etcjavadir -- %{?1}}/lib/security/cacerts +%dir %{etcjavadir -- %{?1}}/conf +%dir %{etcjavadir -- %{?1}}/conf/management +%dir %{etcjavadir -- %{?1}}/conf/security +%dir %{etcjavadir -- %{?1}}/conf/security/policy +%dir %{etcjavadir -- %{?1}}/conf/security/policy/limited +%dir %{etcjavadir -- %{?1}}/conf/security/policy/unlimited +%config(noreplace) %{etcjavadir -- %{?1}}/lib/security/default.policy +%config(noreplace) %{etcjavadir -- %{?1}}/lib/security/blacklisted.certs +%config(noreplace) %{etcjavadir -- %{?1}}/lib/security/public_suffix_list.dat +%config(noreplace) %{etcjavadir -- %{?1}}/conf/security/policy/limited/exempt_local.policy +%config(noreplace) %{etcjavadir -- %{?1}}/conf/security/policy/limited/default_local.policy +%config(noreplace) %{etcjavadir -- %{?1}}/conf/security/policy/limited/default_US_export.policy +%config(noreplace) %{etcjavadir -- %{?1}}/conf/security/policy/unlimited/default_local.policy +%config(noreplace) %{etcjavadir -- %{?1}}/conf/security/policy/unlimited/default_US_export.policy + %{etcjavadir -- %{?1}}/conf/security/policy/README.txt +%config(noreplace) %{etcjavadir -- %{?1}}/conf/security/java.policy +%config(noreplace) %{etcjavadir -- %{?1}}/conf/security/java.security +%config(noreplace) %{etcjavadir -- %{?1}}/conf/logging.properties +%config(noreplace) %{etcjavadir -- %{?1}}/conf/security/nss.cfg +%config(noreplace) %{etcjavadir -- %{?1}}/conf/management/jmxremote.access +# this is conifg template, thus not config-noreplace +%config %{etcjavadir -- %{?1}}/conf/management/jmxremote.password.template +%config(noreplace) %{etcjavadir -- %{?1}}/conf/management/management.properties +%config(noreplace) %{etcjavadir -- %{?1}}/conf/net.properties +%config(noreplace) %{etcjavadir -- %{?1}}/conf/sound.properties +%{_jvmdir}/%{sdkdir -- %{?1}}/conf +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/security +} + +%define files_devel() %{expand: +%dir %{_jvmdir}/%{sdkdir -- %{?1}}/bin +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/jar +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/jarsigner +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/javac +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/javadoc +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/javap +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/jconsole +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/jcmd +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/jdb +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/jdeps +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/jdeprscan +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/jimage +# Zero and S390x don't have SA +%ifarch %{jit_arches} +%ifnarch s390x +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/jhsdb +%endif +%endif +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/jinfo +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/jlink +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/jmap +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/jmod +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/jps +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/jrunscript +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/jshell +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/jstack +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/jstat +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/jstatd +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/rmic +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/serialver +%ifarch %{aot_arches} +%{_jvmdir}/%{sdkdir -- %{?1}}/bin/jaotc +%endif +%{_jvmdir}/%{sdkdir -- %{?1}}/include +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/ct.sym +%if %{with_systemtap} +%{_jvmdir}/%{sdkdir -- %{?1}}/tapset +%endif +%{_datadir}/applications/*jconsole%{?1}.desktop +%{_mandir}/man1/jar-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/jarsigner-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/javac-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/javadoc-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/javap-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/jconsole-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/jcmd-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/jdb-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/jdeps-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/jinfo-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/jmap-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/jps-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/jrunscript-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/jstack-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/jstat-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/jstatd-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/rmic-%{uniquesuffix -- %{?1}}.1* +%{_mandir}/man1/serialver-%{uniquesuffix -- %{?1}}.1* +%if %{with_systemtap} +%dir %{tapsetroot} +%dir %{tapsetdirttapset} +%dir %{tapsetdir} +%{tapsetdir}/*%{_arch}%{?1}.stp +%endif +} + +%define files_jmods() %{expand: +%{_jvmdir}/%{sdkdir -- %{?1}}/jmods +} + +%define files_demo() %{expand: +%license %{_jvmdir}/%{sdkdir -- %{?1}}/legal +%{_jvmdir}/%{sdkdir -- %{?1}}/demo +%{_jvmdir}/%{sdkdir -- %{?1}}/sample +} + +%define files_src() %{expand: +%license %{_jvmdir}/%{sdkdir -- %{?1}}/legal +%{_jvmdir}/%{sdkdir -- %{?1}}/lib/src.zip +} + +%define files_javadoc() %{expand: +%doc %{_javadocdir}/%{uniquejavadocdir -- %{?1}} +%license %{buildoutputdir -- %{?1}}/images/%{jdkimage}/legal +} + +%define files_javadoc_zip() %{expand: +%doc %{_javadocdir}/%{uniquejavadocdir -- %{?1}}.zip +%license %{buildoutputdir -- %{?1}}/images/%{jdkimage}/legal +} + +# not-duplicated requires/provides/obsoletes for normal/debug packages +%define java_rpo() %{expand: +Requires: fontconfig%{?_isa} +Requires: xorg-x11-fonts-Type1 +# Requires rest of java +Requires: %{name}-headless%{?1}%{?_isa} = %{epoch}:%{version}-%{release} +OrderWithRequires: %{name}-headless%{?1}%{?_isa} = %{epoch}:%{version}-%{release} +# for java-X-openjdk package's desktop binding +Recommends: gtk3%{?_isa} + +Provides: java-%{javaver}-%{origin}%{?1} = %{epoch}:%{version}-%{release} + +# Standard JPackage base provides +Provides: jre = %{javaver}%{?1} +Provides: jre-%{origin}%{?1} = %{epoch}:%{version}-%{release} +Provides: jre-%{javaver}%{?1} = %{epoch}:%{version}-%{release} +Provides: jre-%{javaver}-%{origin}%{?1} = %{epoch}:%{version}-%{release} +Provides: java-%{javaver}%{?1} = %{epoch}:%{version}-%{release} +Provides: java-%{origin}%{?1} = %{epoch}:%{version}-%{release} +Provides: java%{?1} = %{epoch}:%{javaver} +} + +%define java_headless_rpo() %{expand: +# Require /etc/pki/java/cacerts +Requires: ca-certificates +# Require javapackages-filesystem for ownership of /usr/lib/jvm/ and macros +Requires: javapackages-filesystem +# Require zone-info data provided by tzdata-java sub-package +Requires: tzdata-java >= 2015d +# libsctp.so.1 is being `dlopen`ed on demand +Requires: lksctp-tools%{?_isa} +# there is a need to depend on the exact version of NSS +Requires: nss%{?_isa} %{NSS_BUILDTIME_VERSION} +Requires: nss-softokn%{?_isa} %{NSSSOFTOKN_BUILDTIME_VERSION} +# tool to copy jdk's configs - should be Recommends only, but then only dnf/yum enforce it, +# not rpm transaction and so no configs are persisted when pure rpm -u is run. It may be +# considered as regression +Requires: copy-jdk-configs >= 3.3 +OrderWithRequires: copy-jdk-configs +# Post requires alternatives to install tool alternatives +Requires(post): %{_sbindir}/alternatives +# in version 1.7 and higher for --family switch +Requires(post): chkconfig >= 1.7 +# Postun requires alternatives to uninstall tool alternatives +Requires(postun): %{_sbindir}/alternatives +# in version 1.7 and higher for --family switch +Requires(postun): chkconfig >= 1.7 +# for optional support of kernel stream control, card reader and printing bindings +Suggests: lksctp-tools%{?_isa}, pcsc-lite-devel%{?_isa}, cups + +# Standard JPackage base provides +Provides: jre-headless%{?1} = %{epoch}:%{javaver} +Provides: jre-%{javaver}-%{origin}-headless%{?1} = %{epoch}:%{version}-%{release} +Provides: jre-%{origin}-headless%{?1} = %{epoch}:%{version}-%{release} +Provides: jre-%{javaver}-headless%{?1} = %{epoch}:%{version}-%{release} +Provides: java-%{javaver}-%{origin}-headless%{?1} = %{epoch}:%{version}-%{release} +Provides: java-%{javaver}-headless%{?1} = %{epoch}:%{version}-%{release} +Provides: java-%{origin}-headless%{?1} = %{epoch}:%{version}-%{release} +Provides: java-headless%{?1} = %{epoch}:%{javaver} + +# https://bugzilla.redhat.com/show_bug.cgi?id=1312019 +Provides: /usr/bin/jjs + +} + +%define java_devel_rpo() %{expand: +# Requires base package +Requires: %{name}%{?1}%{?_isa} = %{epoch}:%{version}-%{release} +OrderWithRequires: %{name}-headless%{?1}%{?_isa} = %{epoch}:%{version}-%{release} +# Post requires alternatives to install tool alternatives +Requires(post): %{_sbindir}/alternatives +# in version 1.7 and higher for --family switch +Requires(post): chkconfig >= 1.7 +# Postun requires alternatives to uninstall tool alternatives +Requires(postun): %{_sbindir}/alternatives +# in version 1.7 and higher for --family switch +Requires(postun): chkconfig >= 1.7 + +# Standard JPackage devel provides +Provides: java-sdk-%{javaver}-%{origin}%{?1} = %{epoch}:%{version} +Provides: java-sdk-%{javaver}%{?1} = %{epoch}:%{version} +#Provides: java-sdk-%%{origin}%%{?1} = %%{epoch}:%%{version} +#Provides: java-sdk%%{?1} = %%{epoch}:%%{javaver} +Provides: java-%{javaver}-devel%{?1} = %{epoch}:%{version} +Provides: java-%{javaver}-%{origin}-devel%{?1} = %{epoch}:%{version} +#Provides: java-devel-%%{origin}%%{?1} = %%{epoch}:%%{version} +#Provides: java-devel%%{?1} = %%{epoch}:%%{javaver} + +} + +%define java_jmods_rpo() %{expand: +# Requires devel package +# as jmods are bytecode, they should be OK without any _isa +Requires: %{name}-devel%{?1} = %{epoch}:%{version}-%{release} +OrderWithRequires: %{name}-headless%{?1} = %{epoch}:%{version}-%{release} + +Provides: java-jmods%{?1} = %{epoch}:%{version}-%{release} +Provides: java-%{javaver}-jmods%{?1} = %{epoch}:%{version}-%{release} +Provides: java-%{javaver}-%{origin}-jmods%{?1} = %{epoch}:%{version}-%{release} + +} + +%define java_demo_rpo() %{expand: +Requires: %{name}%{?1}%{?_isa} = %{epoch}:%{version}-%{release} +OrderWithRequires: %{name}-headless%{?1}%{?_isa} = %{epoch}:%{version}-%{release} + +Provides: java-demo%{?1} = %{epoch}:%{version}-%{release} +Provides: java-%{javaver}-demo%{?1} = %{epoch}:%{version}-%{release} +Provides: java-%{javaver}-%{origin}-demo%{?1} = %{epoch}:%{version}-%{release} + +} + +%define java_javadoc_rpo() %{expand: +OrderWithRequires: %{name}-headless%{?1}%{?_isa} = %{epoch}:%{version}-%{release} +# Post requires alternatives to install javadoc alternative +Requires(post): %{_sbindir}/alternatives +# in version 1.7 and higher for --family switch +Requires(post): chkconfig >= 1.7 +# Postun requires alternatives to uninstall javadoc alternative +Requires(postun): %{_sbindir}/alternatives +# in version 1.7 and higher for --family switch +Requires(postun): chkconfig >= 1.7 + +# Standard JPackage javadoc provides +Provides: java-javadoc%{?1} = %{epoch}:%{version}-%{release} +Provides: java-%{javaver}-javadoc%{?1} = %{epoch}:%{version}-%{release} +Provides: java-%{javaver}-%{origin}-javadoc%{?1} = %{epoch}:%{version}-%{release} +} + +%define java_src_rpo() %{expand: +Requires: %{name}-headless%{?1}%{?_isa} = %{epoch}:%{version}-%{release} + +# Standard JPackage sources provides +Provides: java-src%{?1} = %{epoch}:%{version}-%{release} +Provides: java-%{javaver}-src%{?1} = %{epoch}:%{version}-%{release} +Provides: java-%{javaver}-%{origin}-src%{?1} = %{epoch}:%{version}-%{release} +} + +# Prevent brp-java-repack-jars from being run +%global __jar_repack 0 + +Name: java-%{javaver}-%{origin} +Version: %{fulljavaver} +Release: 2%{?dist} +# java-1.5.0-ibm from jpackage.org set Epoch to 1 for unknown reasons +# and this change was brought into RHEL-4. java-1.5.0-ibm packages +# also included the epoch in their virtual provides. This created a +# situation where in-the-wild java-1.5.0-ibm packages provided "java = +# 1:1.5.0". In RPM terms, "1.6.0 < 1:1.5.0" since 1.6.0 is +# interpreted as 0:1.6.0. So the "java >= 1.6.0" requirement would be +# satisfied by the 1:1.5.0 packages. Thus we need to set the epoch in +# JDK package >= 1.6.0 to 1, and packages referring to JDK virtual +# provides >= 1.6.0 must specify the epoch, "java >= 1:1.6.0". + +Epoch: 1 +Summary: %{origin_nice} Runtime Environment %{majorver} +Group: Development/Languages + +# HotSpot code is licensed under GPLv2 +# JDK library code is licensed under GPLv2 with the Classpath exception +# The Apache license is used in code taken from Apache projects (primarily xalan & xerces) +# DOM levels 2 & 3 and the XML digital signature schemas are licensed under the W3C Software License +# The JSR166 concurrency code is in the public domain +# The BSD and MIT licenses are used for a number of third-party libraries (see ADDITIONAL_LICENSE_INFO) +# The OpenJDK source tree includes: +# - JPEG library (IJG), zlib & libpng (zlib), giflib (MIT), harfbuzz (ISC), +# - freetype (FTL), jline (BSD) and LCMS (MIT) +# - jquery (MIT), jdk.crypto.cryptoki PKCS 11 wrapper (RSA) +# - public_suffix_list.dat from publicsuffix.org (MPLv2.0) +# The test code includes copies of NSS under the Mozilla Public License v2.0 +# The PCSClite headers are under a BSD with advertising license +# The elliptic curve cryptography (ECC) source code is licensed under the LGPLv2.1 or any later version +License: ASL 1.1 and ASL 2.0 and BSD and BSD with advertising and GPL+ and GPLv2 and GPLv2 with exceptions and IJG and LGPLv2+ and MIT and MPLv2.0 and Public Domain and W3C and zlib and ISC and FTL and RSA +URL: http://openjdk.java.net/ + + +Source0: openjdk-%{fulljavaver}-ga.tar.xz + +# Desktop files. Adapted from IcedTea +# Source9: jconsole.desktop.in + +# nss configuration file +# Source11: nss.cfg.in + +############################################ +# +# RPM/distribution specific patches +# +############################################ + +# NSS via SunPKCS11 Provider (disabled comment +# due to memory leak). +# Patch1000: enableCommentedOutSystemNss.patch + +############################################# +# +# OpenJDK specific patches +# +############################################# + +Patch1: change-vendor-to-openEuler_Community.patch +Patch2: 8225648-TESTBUG-java-lang-annotation-loaderLeak-Main.patch +Patch3: 8231584-Deadlock-with-ClassLoader.findLibrary-and-Sy.patch +Patch4: 8214345-infinite-recursion-while-checking-super-clas.patch +Patch5: Add-ability-to-configure-third-port-for-remote-JMX.patch +Patch6: 8214527-AArch64-ZGC-for-Aarch64.patch +Patch7: 8224675-Late-GC-barrier-insertion-for-ZGC.patch +Patch8: freetype-seeks-to-index-at-the-end-of-the-fo.patch +Patch9: ZGC-Redesign-C2-load-barrier-to-expand-on-th.patch +Patch10: ZGC-aarch64-not-using-zr-register-avoid-sigill-in-Ma.patch +Patch11: 8217856-ZGC-Break-out-C2-matching-rules-into-separat.patch +Patch12: 8233073-Make-BitMap-accessors-more-memory-ordering-f.patch +Patch13: 8233061-ZGC-Enforce-memory-ordering-in-segmented-bit.patch +Patch14: Add-loadload-membar-to-avoid-loading-a-incorrect-offset.patch +Patch15: 8226536-Catch-OOM-from-deopt-that-fails-rematerializ.patch +Patch16: prohibition-of-irreducible-loop-in-mergers.patch +Patch17: 8201271-AArch64-Vector-API-for-AArch64.patch +Patch18: 8209375-ZGC-Use-dynamic-base-address-for-mark-stack-.patch +Patch19: VectorAPI-X86-Vector-API-not-supported-on-X86-fix-jt.patch +Patch20: 8209894-ZGC-Cap-number-of-GC-workers-based-on-heap-s.patch +Patch21: 8223347-Integration-of-Vector-API-Fix-VectorAPI-prin.patch +Patch22: 8233506-ZGC-the-load-for-Reference.get-can-be-conver.patch + +BuildRequires: autoconf +BuildRequires: automake +BuildRequires: alsa-lib-devel +BuildRequires: binutils +BuildRequires: cups-devel +BuildRequires: desktop-file-utils +# elfutils only are OK for build without AOT +BuildRequires: elfutils-devel +BuildRequires: fontconfig +BuildRequires: freetype-devel +BuildRequires: giflib-devel +BuildRequires: gcc-c++ +BuildRequires: gdb +BuildRequires: gtk3-devel +BuildRequires: lcms2-devel +BuildRequires: libjpeg-devel +BuildRequires: libpng-devel +BuildRequires: libxslt +BuildRequires: libX11-devel +BuildRequires: libXi-devel +BuildRequires: libXinerama-devel +BuildRequires: libXt-devel +BuildRequires: libXtst-devel +# Requirements for setting up the nss.cfg +BuildRequires: nss-devel +BuildRequires: pkgconfig +BuildRequires: xorg-x11-proto-devel +BuildRequires: zip +BuildRequires: javapackages-filesystem +BuildRequires: java-11-openjdk-devel +# Zero-assembler build requirement +%ifnarch %{jit_arches} +BuildRequires: libffi-devel +%endif +BuildRequires: tzdata-java >= 2015d +# Earlier versions have a bug in tree vectorization on PPC +BuildRequires: gcc >= 4.8.3-8 +# Build requirements for SunEC system NSS support +BuildRequires: nss-softokn-freebl-devel >= 3.16.1 + +%if %{with_systemtap} +BuildRequires: systemtap-sdt-devel +%endif + +# this is always built, also during debug-only build +# when it is built in debug-only this package is just placeholder +%{java_rpo %{nil}} + +%description +The %{origin_nice} runtime environment. + +%if %{include_debug_build} +%package slowdebug +Summary: %{origin_nice} Runtime Environment %{majorver} %{debug_on} +Group: Development/Languages + +%{java_rpo -- %{debug_suffix_unquoted}} +%description slowdebug +The %{origin_nice} runtime environment. +%{debug_warning} +%endif + +%if %{include_normal_build} +%package headless +Summary: %{origin_nice} Headless Runtime Environment %{majorver} +Group: Development/Languages + +%{java_headless_rpo %{nil}} + +%description headless +The %{origin_nice} runtime environment %{majorver} without audio and video support. +%endif + +%if %{include_debug_build} +%package headless-slowdebug +Summary: %{origin_nice} Runtime Environment %{debug_on} +Group: Development/Languages + +%{java_headless_rpo -- %{debug_suffix_unquoted}} + +%description headless-slowdebug +The %{origin_nice} runtime environment %{majorver} without audio and video support. +%{debug_warning} +%endif + +%if %{include_normal_build} +%package devel +Summary: %{origin_nice} Development Environment %{majorver} +Group: Development/Tools + +%{java_devel_rpo %{nil}} + +%description devel +The %{origin_nice} development tools %{majorver}. +%endif + +%if %{include_debug_build} +%package devel-slowdebug +Summary: %{origin_nice} Development Environment %{majorver} %{debug_on} +Group: Development/Tools + +%{java_devel_rpo -- %{debug_suffix_unquoted}} + +%description devel-slowdebug +The %{origin_nice} development tools %{majorver}. +%{debug_warning} +%endif + +%if %{include_normal_build} +%package jmods +Summary: JMods for %{origin_nice} %{majorver} +Group: Development/Tools + +%{java_jmods_rpo %{nil}} + +%description jmods +The JMods for %{origin_nice}. +%endif + +%if %{include_debug_build} +%package jmods-slowdebug +Summary: JMods for %{origin_nice} %{majorver} %{debug_on} +Group: Development/Tools + +%{java_jmods_rpo -- %{debug_suffix_unquoted}} + +%description jmods-slowdebug +The JMods for %{origin_nice} %{majorver}. +%{debug_warning} +%endif + +%if %{include_normal_build} +%package demo +Summary: %{origin_nice} Demos %{majorver} +Group: Development/Languages + +%{java_demo_rpo %{nil}} + +%description demo +The %{origin_nice} demos %{majorver}. +%endif + +%if %{include_debug_build} +%package demo-slowdebug +Summary: %{origin_nice} Demos %{majorver} %{debug_on} +Group: Development/Languages + +%{java_demo_rpo -- %{debug_suffix_unquoted}} + +%description demo-slowdebug +The %{origin_nice} demos %{majorver}. +%{debug_warning} +%endif + +%if %{include_normal_build} +%package src +Summary: %{origin_nice} Source Bundle %{majorver} +Group: Development/Languages + +%{java_src_rpo %{nil}} + +%description src +The java-%{origin}-src sub-package contains the complete %{origin_nice} %{majorver} +class library source code for use by IDE indexers and debuggers. +%endif + +%if %{include_debug_build} +%package src-slowdebug +Summary: %{origin_nice} Source Bundle %{majorver} %{for_debug} +Group: Development/Languages + +%{java_src_rpo -- %{debug_suffix_unquoted}} + +%description src-slowdebug +The java-%{origin}-src-slowdebug sub-package contains the complete %{origin_nice} %{majorver} + class library source code for use by IDE indexers and debuggers. Debugging %{for_debug}. +%endif + +%if %{include_normal_build} +%package javadoc +Summary: %{origin_nice} %{majorver} API documentation +Group: Documentation +Requires: javapackages-filesystem + +%{java_javadoc_rpo %{nil}} + +%description javadoc +The %{origin_nice} %{majorver} API documentation. +%endif + +%if %{include_normal_build} +%package javadoc-zip +Summary: %{origin_nice} %{majorver} API documentation compressed in single archive +Group: Documentation +Requires: javapackages-filesystem + +%{java_javadoc_rpo %{nil}} + +%description javadoc-zip +The %{origin_nice} %{majorver} API documentation compressed in single archive. +%endif + +%if %{include_debug_build} +%package javadoc-slowdebug +Summary: %{origin_nice} %{majorver} API documentation %{for_debug} +Group: Documentation +Requires: javapackages-filesystem + +%{java_javadoc_rpo -- %{debug_suffix_unquoted}} + +%description javadoc-slowdebug +The %{origin_nice} %{majorver} API documentation %{for_debug}. +%endif + +%if %{include_debug_build} +%package javadoc-zip-slowdebug +Summary: %{origin_nice} %{majorver} API documentation compressed in single archive %{for_debug} +Group: Documentation +Requires: javapackages-filesystem + +%{java_javadoc_rpo -- %{debug_suffix_unquoted}} + +%description javadoc-zip-slowdebug +The %{origin_nice} %{majorver} API documentation compressed in single archive %{for_debug}. +%endif + + +%prep +if [ %{include_normal_build} -eq 0 -o %{include_normal_build} -eq 1 ] ; then + echo "include_normal_build is %{include_normal_build}" +else + echo "include_normal_build is %{include_normal_build}, thats invalid. Use 1 for yes or 0 for no" + exit 11 +fi +if [ %{include_debug_build} -eq 0 -o %{include_debug_build} -eq 1 ] ; then + echo "include_debug_build is %{include_debug_build}" +else + echo "include_debug_build is %{include_debug_build}, thats invalid. Use 1 for yes or 0 for no" + exit 12 +fi +if [ %{include_debug_build} -eq 0 -a %{include_normal_build} -eq 0 ] ; then + echo "You have disabled both include_debug_build and include_normal_build. That is a no go." + exit 13 +fi +%setup -q -c -n %{uniquesuffix ""} -T -a 0 +# https://bugzilla.redhat.com/show_bug.cgi?id=1189084 +prioritylength=`expr length %{priority}` +if [ $prioritylength -ne 7 ] ; then + echo "priority must be 7 digits in total, violated" + exit 14 +fi + +# OpenJDK patches +%patch1 -p1 +%patch2 -p1 +%patch3 -p1 +%patch4 -p1 +%patch5 -p1 +%patch6 -p1 +%patch7 -p1 +%patch8 -p1 +%patch9 -p1 +%patch10 -p1 +%patch11 -p1 +%patch12 -p1 +%patch13 -p1 +%patch14 -p1 +%patch15 -p1 +%patch16 -p1 +%patch17 -p1 +%patch18 -p1 +%patch19 -p1 +%patch20 -p1 +%patch21 -p1 +%patch22 -p1 + +# Extract systemtap tapsets +%if %{with_systemtap} +tar --strip-components=1 -x -I xz -f %{SOURCE8} +%if %{include_debug_build} +cp -r tapset tapset%{debug_suffix} +%endif + +for suffix in %{build_loop} ; do + for file in "tapset"$suffix/*.in; do + OUTPUT_FILE=`echo $file | sed -e "s:\.stp\.in$:%{version}-%{release}.%{_arch}.stp:g"` + sed -e "s:@ABS_SERVER_LIBJVM_SO@:%{_jvmdir}/%{sdkdir -- $suffix}/lib/server/libjvm.so:g" $file > $file.1 +# TODO find out which architectures other than i686 have a client vm +%ifarch %{ix86} + sed -e "s:@ABS_CLIENT_LIBJVM_SO@:%{_jvmdir}/%{sdkdir -- $suffix}/lib/client/libjvm.so:g" $file.1 > $OUTPUT_FILE +%else + sed -e "/@ABS_CLIENT_LIBJVM_SO@/d" $file.1 > $OUTPUT_FILE +%endif + sed -i -e "s:@ABS_JAVA_HOME_DIR@:%{_jvmdir}/%{sdkdir -- $suffix}:g" $OUTPUT_FILE + sed -i -e "s:@INSTALL_ARCH_DIR@:%{archinstall}:g" $OUTPUT_FILE + sed -i -e "s:@prefix@:%{_jvmdir}/%{sdkdir -- $suffix}/:g" $OUTPUT_FILE + done +done +# systemtap tapsets ends +%endif + +# Prepare desktop files +for suffix in %{build_loop} ; do +for file in %{SOURCE9}; do + FILE=`basename $file | sed -e s:\.in$::g` + EXT="${FILE##*.}" + NAME="${FILE%.*}" + OUTPUT_FILE=$NAME$suffix.$EXT + sed -e "s:@JAVA_HOME@:%{sdkbindir -- $suffix}:g" $file > $OUTPUT_FILE + sed -i -e "s:@JRE_HOME@:%{jrebindir -- $suffix}:g" $OUTPUT_FILE + sed -i -e "s:@ARCH@:%{version}-%{release}.%{_arch}$suffix:g" $OUTPUT_FILE + sed -i -e "s:@JAVA_MAJOR_VERSION@:%{majorver}:g" $OUTPUT_FILE + sed -i -e "s:@JAVA_VENDOR@:%{origin}:g" $OUTPUT_FILE +done +done + +# Setup nss.cfg +sed -e "s:@NSS_LIBDIR@:%{NSS_LIBDIR}:g" %{SOURCE11} > nss.cfg + + +%build +# How many CPU's do we have? +export NUM_PROC=%(/usr/bin/getconf _NPROCESSORS_ONLN 2> /dev/null || :) +export NUM_PROC=${NUM_PROC:-1} +%if 0%{?_smp_ncpus_max} +# Honor %%_smp_ncpus_max +[ ${NUM_PROC} -gt %{?_smp_ncpus_max} ] && export NUM_PROC=%{?_smp_ncpus_max} +%endif + +%ifarch s390x sparc64 alpha %{power64} %{aarch64} +export ARCH_DATA_MODEL=64 +%endif +%ifarch alpha +export CFLAGS="$CFLAGS -mieee" +%endif + +# We use ourcppflags because the OpenJDK build seems to +# pass EXTRA_CFLAGS to the HotSpot C++ compiler... +# Explicitly set the C++ standard as the default has changed on GCC >= 6 +EXTRA_CFLAGS="%ourcppflags -std=gnu++98 -Wno-error -fno-delete-null-pointer-checks -fno-lifetime-dse" +EXTRA_CPP_FLAGS="%ourcppflags -std=gnu++98 -fno-delete-null-pointer-checks -fno-lifetime-dse" + +%ifarch %{power64} ppc +# fix rpmlint warnings +EXTRA_CFLAGS="$EXTRA_CFLAGS -fno-strict-aliasing" +%endif +export EXTRA_CFLAGS + +for suffix in %{build_loop} ; do +if [ "x$suffix" = "x" ] ; then + debugbuild=release +else + # change --something to something + debugbuild=`echo $suffix | sed "s/-//g"` +fi + +# Variable used in hs_err hook on build failures +top_dir_abs_path=$(pwd)/%{top_level_dir_name} + +mkdir -p %{buildoutputdir -- $suffix} +pushd %{buildoutputdir -- $suffix} + +bash ../configure \ +%ifnarch %{jit_arches} + --with-jvm-variants=zero \ +%endif +%ifarch %{ppc64le} + --with-jobs=1 \ +%endif + --with-version-build=%{buildver} \ + --with-version-pre="ea" \ + --with-version-opt="" \ + --with-boot-jdk=/usr/lib/jvm/java-11-openjdk \ + --with-debug-level=$debugbuild \ + --with-native-debug-symbols=internal \ + --enable-unlimited-crypto \ + --enable-system-nss \ + --with-zlib=system \ + --with-libjpeg=system \ + --with-giflib=system \ + --with-libpng=system \ + --with-lcms=system \ + --with-stdc++lib=dynamic \ + --with-extra-cxxflags="$EXTRA_CPP_FLAGS" \ + --with-extra-cflags="$EXTRA_CFLAGS" \ + --with-extra-ldflags="%{ourldflags}" \ + --with-num-cores="$NUM_PROC" \ + --disable-javac-server \ + --disable-warnings-as-errors \ + --with-boot-jdk-jvmargs=-XX:-UsePerfData + +make \ + JAVAC_FLAGS=-g \ + LOG=trace \ + WARNINGS_ARE_ERRORS="-Wno-error" \ + CFLAGS_WARNINGS_ARE_ERRORS="-Wno-error" \ + %{targets} || ( pwd; find $top_dir_abs_path -name "hs_err_pid*.log" | xargs cat && false ) + +make docs-zip + +# the build (erroneously) removes read permissions from some jars +# this is a regression in OpenJDK 7 (our compiler): +# http://icedtea.classpath.org/bugzilla/show_bug.cgi?id=1437 +find images/%{jdkimage} -iname '*.jar' -exec chmod ugo+r {} \; + +# Build screws up permissions on binaries +# https://bugs.openjdk.java.net/browse/JDK-8173610 +find images/%{jdkimage} -iname '*.so' -exec chmod +x {} \; +find images/%{jdkimage}/bin/ -exec chmod +x {} \; + +popd >& /dev/null + +# Install nss.cfg right away as we will be using the JRE above +export JAVA_HOME=$(pwd)/%{buildoutputdir -- $suffix}/images/%{jdkimage} + +# Install nss.cfg right away as we will be using the JRE above +install -m 644 nss.cfg $JAVA_HOME/conf/security/ + +# Use system-wide tzdata +rm $JAVA_HOME/lib/tzdb.dat +ln -s %{_datadir}/javazi-1.8/tzdb.dat $JAVA_HOME/lib/tzdb.dat + +# build cycles +done + +%check + +# We test debug first as it will give better diagnostics on a crash +for suffix in %{rev_build_loop} ; do + +export JAVA_HOME=$(pwd)/%{buildoutputdir -- $suffix}/images/%{jdkimage} + +#check sheandoah is enabled +%if %{use_shenandoah_hotspot} +$JAVA_HOME//bin/java -XX:+UseShenandoahGC -version +%endif + +# Check unlimited policy has been used +$JAVA_HOME/bin/javac -d . %{SOURCE13} +$JAVA_HOME/bin/java --add-opens java.base/javax.crypto=ALL-UNNAMED TestCryptoLevel + +# Check ECC is working +$JAVA_HOME/bin/javac -d . %{SOURCE14} +$JAVA_HOME/bin/java $(echo $(basename %{SOURCE14})|sed "s|\.java||") + +# Check debug symbols are present and can identify code +find "$JAVA_HOME" -iname '*.so' -print0 | while read -d $'\0' lib +do + if [ -f "$lib" ] ; then + echo "Testing $lib for debug symbols" + # All these tests rely on RPM failing the build if the exit code of any set + # of piped commands is non-zero. + + # Test for .debug_* sections in the shared object. This is the main test + # Stripped objects will not contain these + eu-readelf -S "$lib" | grep "] .debug_" + test $(eu-readelf -S "$lib" | grep -E "\]\ .debug_(info|abbrev)" | wc --lines) == 2 + + # Test FILE symbols. These will most likely be removed by anything that + # manipulates symbol tables because it's generally useless. So a nice test + # that nothing has messed with symbols + old_IFS="$IFS" + IFS=$'\n' + for line in $(eu-readelf -s "$lib" | grep "00000000 0 FILE LOCAL DEFAULT") + do + # We expect to see .cpp files, except for architectures like aarch64 and + # s390 where we expect .o and .oS files + echo "$line" | grep -E "ABS ((.*/)?[-_a-zA-Z0-9]+\.(c|cc|cpp|cxx|o|oS))?$" + done + IFS="$old_IFS" + + # If this is the JVM, look for javaCalls.(cpp|o) in FILEs, for extra sanity checking + if [ "`basename $lib`" = "libjvm.so" ]; then + eu-readelf -s "$lib" | \ + grep -E "00000000 0 FILE LOCAL DEFAULT ABS javaCalls.(cpp|o)$" + fi + + # Test that there are no .gnu_debuglink sections pointing to another + # debuginfo file. There shouldn't be any debuginfo files, so the link makes + # no sense either + eu-readelf -S "$lib" | grep 'gnu' + if eu-readelf -S "$lib" | grep '] .gnu_debuglink' | grep PROGBITS; then + echo "bad .gnu_debuglink section." + eu-readelf -x .gnu_debuglink "$lib" + false + fi + fi +done + +# Make sure gdb can do a backtrace based on line numbers on libjvm.so +# javaCalls.cpp:58 should map to: +# http://hg.openjdk.java.net/jdk8u/jdk8u/hotspot/file/ff3b27e6bcc2/src/share/vm/runtime/javaCalls.cpp#l58 +# Using line number 1 might cause build problems. See: +# https://bugzilla.redhat.com/show_bug.cgi?id=1539664 +# https://bugzilla.redhat.com/show_bug.cgi?id=1538767 +gdb -q "$JAVA_HOME/bin/java" < +-- see https://bugzilla.redhat.com/show_bug.cgi?id=1038092 for whole issue +-- see https://bugzilla.redhat.com/show_bug.cgi?id=1290388 for pretrans over pre +-- if copy-jdk-configs is in transaction, it installs in pretrans to temp +-- if copy_jdk_configs is in temp, then it means that copy-jdk-configs is in transaction and so is +-- preferred over one in %%{_libexecdir}. If it is not in transaction, then depends +-- whether copy-jdk-configs is installed or not. If so, then configs are copied +-- (copy_jdk_configs from %%{_libexecdir} used) or not copied at all +local posix = require "posix" +local debug = false + +SOURCE1 = "%{rpm_state_dir}/copy_jdk_configs.lua" +SOURCE2 = "%{_libexecdir}/copy_jdk_configs.lua" + +local stat1 = posix.stat(SOURCE1, "type"); +local stat2 = posix.stat(SOURCE2, "type"); + + if (stat1 ~= nil) then + if (debug) then + print(SOURCE1 .." exists - copy-jdk-configs in transaction, using this one.") + end; + package.path = package.path .. ";" .. SOURCE1 +else + if (stat2 ~= nil) then + if (debug) then + print(SOURCE2 .." exists - copy-jdk-configs already installed and NOT in transaction. Using.") + end; + package.path = package.path .. ";" .. SOURCE2 + else + if (debug) then + print(SOURCE1 .." does NOT exists") + print(SOURCE2 .." does NOT exists") + print("No config files will be copied") + end + return + end +end +-- run content of included file with fake args +arg = {"--currentjvm", "%{uniquesuffix %{nil}}", "--jvmdir", "%{_jvmdir %{nil}}", "--origname", "%{name}", "--origjavaver", "%{javaver}", "--arch", "%{_arch}", "--temp", "%{rpm_state_dir}/%{name}.%{_arch}"} +require "copy_jdk_configs.lua" + +%post +%{post_script %{nil}} + +%post headless +%{post_headless %{nil}} + +%postun +%{postun_script %{nil}} + +%postun headless +%{postun_headless %{nil}} + +%posttrans +%{posttrans_script %{nil}} + +%post devel +%{post_devel %{nil}} + +%postun devel +%{postun_devel %{nil}} + +%posttrans devel +%{posttrans_devel %{nil}} + +%post javadoc +%{post_javadoc %{nil}} + +%postun javadoc +%{postun_javadoc %{nil}} + +%post javadoc-zip +%{post_javadoc_zip %{nil}} + +%postun javadoc-zip +%{postun_javadoc_zip %{nil}} +%endif + +%if %{include_debug_build} +%post slowdebug +%{post_script -- %{debug_suffix_unquoted}} + +%post headless-slowdebug +%{post_headless -- %{debug_suffix_unquoted}} + +%postun slowdebug +%{postun_script -- %{debug_suffix_unquoted}} + +%postun headless-slowdebug +%{postun_headless -- %{debug_suffix_unquoted}} + +%posttrans slowdebug +%{posttrans_script -- %{debug_suffix_unquoted}} + +%post devel-slowdebug +%{post_devel -- %{debug_suffix_unquoted}} + +%postun devel-slowdebug +%{postun_devel -- %{debug_suffix_unquoted}} + +%posttrans devel-slowdebug +%{posttrans_devel -- %{debug_suffix_unquoted}} + +%post javadoc-slowdebug +%{post_javadoc -- %{debug_suffix_unquoted}} + +%postun javadoc-slowdebug +%{postun_javadoc -- %{debug_suffix_unquoted}} + +%post javadoc-zip-slowdebug +%{post_javadoc_zip -- %{debug_suffix_unquoted}} + +%postun javadoc-zip-slowdebug +%{postun_javadoc_zip -- %{debug_suffix_unquoted}} +%endif + +%if %{include_normal_build} +%files +# main package builds always +%{files_jre %{nil}} +%else +%files +# placeholder +%endif + + +%if %{include_normal_build} +%files headless +# important note, see https://bugzilla.redhat.com/show_bug.cgi?id=1038092 for whole issue +# all config/noreplace files (and more) have to be declared in pretrans. See pretrans +%{files_jre_headless %{nil}} + +%files devel +%{files_devel %{nil}} + +%files jmods +%{files_jmods %{nil}} + +%files demo +%{files_demo %{nil}} + +%files src +%{files_src %{nil}} + +%files javadoc +%{files_javadoc %{nil}} + +# this puts huge file to /usr/share +# unluckily ti is really a documentation file +# and unluckily it really is architecture-dependent, as eg. aot and grail are now x86_64 only +# same for debug variant +%files javadoc-zip +%{files_javadoc_zip %{nil}} +%endif + +%if %{include_debug_build} +%files slowdebug +%{files_jre -- %{debug_suffix_unquoted}} + +%files headless-slowdebug +%{files_jre_headless -- %{debug_suffix_unquoted}} + +%files devel-slowdebug +%{files_devel -- %{debug_suffix_unquoted}} + +%files jmods-slowdebug +%{files_jmods -- %{debug_suffix_unquoted}} + +%files demo-slowdebug +%{files_demo -- %{debug_suffix_unquoted}} + +%files src-slowdebug +%{files_src -- %{debug_suffix_unquoted}} + +%files javadoc-slowdebug +%{files_javadoc -- %{debug_suffix_unquoted}} + +%files javadoc-zip-slowdebug +%{files_javadoc_zip -- %{debug_suffix_unquoted}} +%endif + + +%changelog +* Fri April 3 2020 jvmboy - 1:11.0.ea.28-1 +- Initial build from OpenJDK 11.0.6 diff --git a/openjdk-11.0.6-ga.tar.xz b/openjdk-11.0.6-ga.tar.xz new file mode 100644 index 0000000..ff0ba70 Binary files /dev/null and b/openjdk-11.0.6-ga.tar.xz differ diff --git a/prohibition-of-irreducible-loop-in-mergers.patch b/prohibition-of-irreducible-loop-in-mergers.patch new file mode 100644 index 0000000..54906dc --- /dev/null +++ b/prohibition-of-irreducible-loop-in-mergers.patch @@ -0,0 +1,27 @@ +From fd0b27781fdc8a8d4089aeca7acfcaf7cfb1a82c Mon Sep 17 00:00:00 2001 +Date: Thu, 19 Mar 2020 15:07:23 +0800 +Subject: [PATCH] prohibition of irreducible loop in mergers + +Summary: C2Compiler: irreducible loop should not enter merge_many_backedges +LLT: NA +Bug url: NA +--- + src/hotspot/share/opto/loopnode.cpp | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp +index da81c3a5d..5c01a44e4 100644 +--- a/src/hotspot/share/opto/loopnode.cpp ++++ b/src/hotspot/share/opto/loopnode.cpp +@@ -2089,7 +2089,7 @@ bool IdealLoopTree::beautify_loops( PhaseIdealLoop *phase ) { + // If I am a shared header (multiple backedges), peel off the many + // backedges into a private merge point and use the merge point as + // the one true backedge. +- if( _head->req() > 3 ) { ++ if( _head->req() > 3 && !_irreducible ) { + // Merge the many backedges into a single backedge but leave + // the hottest backedge as separate edge for the following peel. + merge_many_backedges( phase ); +-- +2.12.3 +