From e28327b74af8fdab2df76733eff3ba560dcbc2e6 Mon Sep 17 00:00:00 2001 Date: Wed, 8 Dec 2021 16:07:54 +0800 Subject: [PATCH] G1 GC NUMA feature preferentially selects the nearest NUMA node for memory allocation --- src/hotspot/os/aix/os_aix.cpp | 4 +++ src/hotspot/os/bsd/os_bsd.cpp | 4 ++- src/hotspot/os/linux/os_linux.cpp | 4 +++ src/hotspot/os/windows/os_windows.cpp | 4 +++ src/hotspot/share/gc/g1/g1NUMA.cpp | 31 ++++++++++++++++++- src/hotspot/share/gc/g1/g1NUMA.hpp | 9 ++++++ .../share/gc/g1/heapRegionSet.inline.hpp | 24 +++++++++++++- src/hotspot/share/runtime/os.hpp | 1 + 8 files changed, 78 insertions(+), 3 deletions(-) diff --git a/src/hotspot/os/aix/os_aix.cpp b/src/hotspot/os/aix/os_aix.cpp index 1d2ce086b..5b5c6c449 100644 --- a/src/hotspot/os/aix/os_aix.cpp +++ b/src/hotspot/os/aix/os_aix.cpp @@ -279,6 +279,10 @@ bool os::have_special_privileges() { return privileges; } +uint os::numa_distance(uint node_index, uint node_index_other) { + return 0; +} + // Helper function, emulates disclaim64 using multiple 32bit disclaims // because we cannot use disclaim64() on AS/400 and old AIX releases. static bool my_disclaim64(char* addr, size_t size) { diff --git a/src/hotspot/os/bsd/os_bsd.cpp b/src/hotspot/os/bsd/os_bsd.cpp index d71d44d1d..ba3d1568c 100644 --- a/src/hotspot/os/bsd/os_bsd.cpp +++ b/src/hotspot/os/bsd/os_bsd.cpp @@ -191,7 +191,9 @@ bool os::have_special_privileges() { return privileges; } - +uint os::numa_distance(uint node_index, uint node_index_other) { + return 0; +} // Cpu architecture string #if defined(ZERO) diff --git a/src/hotspot/os/linux/os_linux.cpp b/src/hotspot/os/linux/os_linux.cpp index b4c89b518..fc3800edd 100644 --- a/src/hotspot/os/linux/os_linux.cpp +++ b/src/hotspot/os/linux/os_linux.cpp @@ -2862,6 +2862,10 @@ int os::numa_get_group_id_for_address(const void* address) { return id; } +uint os::numa_distance(uint node_index, uint node_index_other) { + return Linux::numa_distance(node_index, node_index_other); +} + int os::Linux::get_existing_num_nodes() { int node; int highest_node_number = Linux::numa_max_node(); diff --git a/src/hotspot/os/windows/os_windows.cpp b/src/hotspot/os/windows/os_windows.cpp index 4f9d54029..fe52ffb22 100644 --- a/src/hotspot/os/windows/os_windows.cpp +++ b/src/hotspot/os/windows/os_windows.cpp @@ -278,6 +278,10 @@ void os::run_periodic_checks() { return; } +uint os::numa_distance(uint node_index, uint node_index_other) { + return 0; +} + // previous UnhandledExceptionFilter, if there is one static LPTOP_LEVEL_EXCEPTION_FILTER prev_uef_handler = NULL; diff --git a/src/hotspot/share/gc/g1/g1NUMA.cpp b/src/hotspot/share/gc/g1/g1NUMA.cpp index 95d9d8c15..2c2431a39 100644 --- a/src/hotspot/share/gc/g1/g1NUMA.cpp +++ b/src/hotspot/share/gc/g1/g1NUMA.cpp @@ -70,9 +70,20 @@ uint G1NUMA::index_of_node_id(int node_id) const { return node_index; } +// Returns numa distance +const uint G1NUMA::calc_numa_node_distance(uint node_index, uint other_node_index) const { + if (node_index >= _num_active_node_ids || other_node_index >= _num_active_node_ids) { + return UINT_MAX; + } + return _numa_node_distance[node_index][other_node_index]; +} +bool G1NUMA::use_nearest_node() const { + return _use_nearest; +} + G1NUMA::G1NUMA() : _node_id_to_index_map(NULL), _len_node_id_to_index_map(0), - _node_ids(NULL), _num_active_node_ids(0), + _node_ids(NULL), _num_active_node_ids(0), _use_nearest(false), _region_size(0), _page_size(0), _stats(NULL) { } @@ -104,10 +115,24 @@ void G1NUMA::initialize(bool use_numa) { for (uint i = 0; i < _num_active_node_ids; i++) { max_node_id = MAX2(max_node_id, _node_ids[i]); } + if (_num_active_node_ids > 2) { + _use_nearest = true; + } // Create a mapping between node_id and index. _len_node_id_to_index_map = max_node_id + 1; _node_id_to_index_map = NEW_C_HEAP_ARRAY(uint, _len_node_id_to_index_map, mtGC); + _numa_node_distance = NEW_C_HEAP_ARRAY(uint*, _num_active_node_ids * _num_active_node_ids, mtGC); + // Set node disctance + for (uint node_i = 0; node_i < _num_active_node_ids; node_i++) { + _numa_node_distance[node_i] = NEW_C_HEAP_ARRAY(uint, _num_active_node_ids*_num_active_node_ids, mtGC); + } + for (uint node_i = 0; node_i < _num_active_node_ids; node_i++) { + for (uint node_j = 0; node_j < _num_active_node_ids; node_j++) { + uint distance = os::numa_distance(node_i, node_j); + _numa_node_distance[node_i][node_j] = distance; + } + } // Set all indices with unknown node id. for (int i = 0; i < _len_node_id_to_index_map; i++) { @@ -126,6 +151,10 @@ G1NUMA::~G1NUMA() { delete _stats; FREE_C_HEAP_ARRAY(int, _node_id_to_index_map); FREE_C_HEAP_ARRAY(int, _node_ids); + for (uint node_i = 0; node_i < _num_active_node_ids; node_i++) { + FREE_C_HEAP_ARRAY(uint, _numa_node_distance[node_i]); + } + FREE_C_HEAP_ARRAY(uint*, _numa_node_distance); } void G1NUMA::set_region_info(size_t region_size, size_t page_size) { diff --git a/src/hotspot/share/gc/g1/g1NUMA.hpp b/src/hotspot/share/gc/g1/g1NUMA.hpp index 83117b8ec..24ae9712d 100644 --- a/src/hotspot/share/gc/g1/g1NUMA.hpp +++ b/src/hotspot/share/gc/g1/g1NUMA.hpp @@ -45,6 +45,10 @@ class G1NUMA: public CHeapObj { int* _node_ids; // Total number of node ids. uint _num_active_node_ids; + // numa node distance + uint** _numa_node_distance; + // Use nearest numa node + bool _use_nearest; // HeapRegion size size_t _region_size; @@ -84,6 +88,8 @@ public: // Returns active memory node count. uint num_active_nodes() const; + bool use_nearest_node() const; + bool is_enabled() const; int numa_id(int index) const; @@ -91,6 +97,9 @@ public: // Returns memory node ids const int* node_ids() const; + // Returns numa distance + const uint calc_numa_node_distance(uint node_index, uint other_node_index) const; + // Returns node index of current calling thread. uint index_of_current_thread() const; diff --git a/src/hotspot/share/gc/g1/heapRegionSet.inline.hpp b/src/hotspot/share/gc/g1/heapRegionSet.inline.hpp index 1f509ff3e..67b3ade50 100644 --- a/src/hotspot/share/gc/g1/heapRegionSet.inline.hpp +++ b/src/hotspot/share/gc/g1/heapRegionSet.inline.hpp @@ -183,26 +183,48 @@ inline HeapRegion* FreeRegionList::remove_region_with_node_index(bool from_head, // Find the region to use, searching from _head or _tail as requested. size_t cur_depth = 0; + bool exist_region = false; + uint numa_distance_min = UINT_MAX; + HeapRegion* remote_node_region = NULL; + if (from_head) { for (cur = _head; cur != NULL && cur_depth < max_search_depth; cur = cur->next(), ++cur_depth) { if (requested_node_index == cur->node_index()) { + exist_region = true; break; } + if (G1NUMA::numa()->use_nearest_node()) { + uint distance = G1NUMA::numa()->calc_numa_node_distance(requested_node_index, cur->node_index()); + if (distance < numa_distance_min) { + remote_node_region = cur; + numa_distance_min = distance; + } + } } } else { for (cur = _tail; cur != NULL && cur_depth < max_search_depth; cur = cur->prev(), ++cur_depth) { if (requested_node_index == cur->node_index()) { + exist_region = true; break; } + if (G1NUMA::numa()->use_nearest_node()) { + uint distance = G1NUMA::numa()->calc_numa_node_distance(requested_node_index, cur->node_index()); + if (distance < numa_distance_min) { + remote_node_region = cur; + numa_distance_min = distance; + } + } } } // Didn't find a region to use. - if (cur == NULL || cur_depth >= max_search_depth) { + if (G1NUMA::numa()->use_nearest_node() && !exist_region && remote_node_region != NULL) { + cur = remote_node_region; + } else if (cur == NULL || cur_depth >= max_search_depth) { return NULL; } diff --git a/src/hotspot/share/runtime/os.hpp b/src/hotspot/share/runtime/os.hpp index 7eaaa9db9..34dcebd9e 100644 --- a/src/hotspot/share/runtime/os.hpp +++ b/src/hotspot/share/runtime/os.hpp @@ -418,6 +418,7 @@ class os: AllStatic { static bool numa_topology_changed(); static int numa_get_group_id(); static int numa_get_group_id_for_address(const void* address); + static uint numa_distance(uint node_index, uint node_index_other); // Page manipulation struct page_info { -- 2.23.0