2020-09-11 18:20:33 +08:00
|
|
|
diff --git a/src/hotspot/share/opto/classes.hpp b/src/hotspot/share/opto/classes.hpp
|
2020-10-22 22:22:56 +08:00
|
|
|
index 5ee23f7..b847caf 100644
|
2020-09-11 18:20:33 +08:00
|
|
|
--- a/src/hotspot/share/opto/classes.hpp
|
|
|
|
|
+++ b/src/hotspot/share/opto/classes.hpp
|
2020-10-22 22:22:56 +08:00
|
|
|
@@ -226,6 +226,8 @@ macro(NegF)
|
2020-09-11 18:20:33 +08:00
|
|
|
macro(NeverBranch)
|
|
|
|
|
macro(OnSpinWait)
|
|
|
|
|
macro(Opaque1)
|
|
|
|
|
+macro(OpaqueLoopInit)
|
|
|
|
|
+macro(OpaqueLoopStride)
|
|
|
|
|
macro(Opaque2)
|
|
|
|
|
macro(Opaque3)
|
|
|
|
|
macro(Opaque4)
|
|
|
|
|
diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp
|
2020-10-22 22:22:56 +08:00
|
|
|
index e5559a2..efe4cff 100644
|
2020-09-11 18:20:33 +08:00
|
|
|
--- a/src/hotspot/share/opto/compile.cpp
|
|
|
|
|
+++ b/src/hotspot/share/opto/compile.cpp
|
2020-10-22 22:22:56 +08:00
|
|
|
@@ -1987,7 +1987,17 @@ void Compile::remove_opaque4_nodes(PhaseIterGVN &igvn) {
|
2020-09-11 18:20:33 +08:00
|
|
|
for (int i = opaque4_count(); i > 0; i--) {
|
|
|
|
|
Node* opaq = opaque4_node(i-1);
|
|
|
|
|
assert(opaq->Opcode() == Op_Opaque4, "Opaque4 only");
|
|
|
|
|
+ // With Opaque4 nodes, the expectation is that the test of input 1
|
|
|
|
|
+ // is always equal to the constant value of input 2. So we can
|
|
|
|
|
+ // remove the Opaque4 and replace it by input 2. In debug builds,
|
|
|
|
|
+ // leave the non constant test in instead to sanity check that it
|
|
|
|
|
+ // never fails (if it does, that subgraph was constructed so, at
|
|
|
|
|
+ // runtime, a Halt node is executed).
|
|
|
|
|
+ #ifdef ASSERT
|
|
|
|
|
+ igvn.replace_node(opaq, opaq->in(1));
|
|
|
|
|
+ #else
|
|
|
|
|
igvn.replace_node(opaq, opaq->in(2));
|
|
|
|
|
+ #endif
|
|
|
|
|
}
|
|
|
|
|
assert(opaque4_count() == 0, "should be empty");
|
|
|
|
|
}
|
|
|
|
|
diff --git a/src/hotspot/share/opto/loopPredicate.cpp b/src/hotspot/share/opto/loopPredicate.cpp
|
2020-10-22 22:22:56 +08:00
|
|
|
index 6e85398..a6d2257 100644
|
2020-09-11 18:20:33 +08:00
|
|
|
--- a/src/hotspot/share/opto/loopPredicate.cpp
|
|
|
|
|
+++ b/src/hotspot/share/opto/loopPredicate.cpp
|
2020-10-22 22:22:56 +08:00
|
|
|
@@ -1243,8 +1243,9 @@ ProjNode* PhaseIdealLoop::insert_initial_skeleton_predicate(IfNode* iff, IdealLo
|
2020-09-11 18:20:33 +08:00
|
|
|
Node* init, Node* limit, jint stride,
|
|
|
|
|
Node* rng, bool &overflow,
|
|
|
|
|
Deoptimization::DeoptReason reason) {
|
|
|
|
|
+ // First predicate for the initial value on first loop iteration
|
|
|
|
|
assert(proj->_con && predicate_proj->_con, "not a range check?");
|
|
|
|
|
- Node* opaque_init = new Opaque1Node(C, init);
|
|
|
|
|
+ Node* opaque_init = new OpaqueLoopInitNode(C, init);
|
|
|
|
|
register_new_node(opaque_init, upper_bound_proj);
|
|
|
|
|
BoolNode* bol = rc_predicate(loop, upper_bound_proj, scale, offset, opaque_init, limit, stride, rng, (stride > 0) != (scale > 0), overflow);
|
|
|
|
|
Node* opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1)); // This will go away once loop opts are over
|
2020-10-22 22:22:56 +08:00
|
|
|
@@ -1252,6 +1253,22 @@ ProjNode* PhaseIdealLoop::insert_initial_skeleton_predicate(IfNode* iff, IdealLo
|
2020-09-11 18:20:33 +08:00
|
|
|
ProjNode* new_proj = create_new_if_for_predicate(predicate_proj, NULL, reason, overflow ? Op_If : iff->Opcode());
|
|
|
|
|
_igvn.replace_input_of(new_proj->in(0), 1, opaque_bol);
|
|
|
|
|
assert(opaque_init->outcnt() > 0, "should be used");
|
|
|
|
|
+ // Second predicate for init + (current stride - initial stride)
|
|
|
|
|
+ // This is identical to the previous predicate initially but as
|
|
|
|
|
+ // unrolling proceeds current stride is updated.
|
|
|
|
|
+ Node* init_stride = loop->_head->as_CountedLoop()->stride();
|
|
|
|
|
+ Node* opaque_stride = new OpaqueLoopStrideNode(C, init_stride);
|
|
|
|
|
+ register_new_node(opaque_stride, new_proj);
|
|
|
|
|
+ Node* max_value = new SubINode(opaque_stride, init_stride);
|
|
|
|
|
+ register_new_node(max_value, new_proj);
|
|
|
|
|
+ max_value = new AddINode(opaque_init, max_value);
|
|
|
|
|
+ register_new_node(max_value, new_proj);
|
|
|
|
|
+ bol = rc_predicate(loop, new_proj, scale, offset, max_value, limit, stride, rng, (stride > 0) != (scale > 0), overflow);
|
|
|
|
|
+ opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1));
|
|
|
|
|
+ register_new_node(opaque_bol, new_proj);
|
|
|
|
|
+ new_proj = create_new_if_for_predicate(predicate_proj, NULL, reason, overflow ? Op_If : iff->Opcode());
|
|
|
|
|
+ _igvn.replace_input_of(new_proj->in(0), 1, opaque_bol);
|
|
|
|
|
+ assert(max_value->outcnt() > 0, "should be used");
|
|
|
|
|
return new_proj;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
diff --git a/src/hotspot/share/opto/loopTransform.cpp b/src/hotspot/share/opto/loopTransform.cpp
|
2020-10-22 22:22:56 +08:00
|
|
|
index 5e6faaa..89628bb 100644
|
2020-09-11 18:20:33 +08:00
|
|
|
--- a/src/hotspot/share/opto/loopTransform.cpp
|
|
|
|
|
+++ b/src/hotspot/share/opto/loopTransform.cpp
|
|
|
|
|
@@ -1080,7 +1080,7 @@ void PhaseIdealLoop::ensure_zero_trip_guard_proj(Node* node, bool is_main_loop)
|
|
|
|
|
// CastII/ConvI2L nodes cause some data paths to die. For consistency,
|
|
|
|
|
// the control paths must die too but the range checks were removed by
|
|
|
|
|
// predication. The range checks that we add here guarantee that they do.
|
|
|
|
|
-void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop_helper(Node* predicate, Node* start, Node* end,
|
|
|
|
|
+void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop_helper(Node* predicate, Node* init, Node* stride,
|
|
|
|
|
IdealLoopTree* outer_loop, LoopNode* outer_main_head,
|
|
|
|
|
uint dd_main_head, const uint idx_before_pre_post,
|
|
|
|
|
const uint idx_after_post_before_pre, Node* zero_trip_guard_proj_main,
|
|
|
|
|
@@ -1098,6 +1098,10 @@ void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop_helper(Node* predicat
|
|
|
|
|
predicate = iff->in(0);
|
|
|
|
|
Node* current_proj = outer_main_head->in(LoopNode::EntryControl);
|
|
|
|
|
Node* prev_proj = current_proj;
|
|
|
|
|
+ Node* opaque_init = new OpaqueLoopInitNode(C, init);
|
|
|
|
|
+ register_new_node(opaque_init, outer_main_head->in(LoopNode::EntryControl));
|
|
|
|
|
+ Node* opaque_stride = new OpaqueLoopStrideNode(C, stride);
|
|
|
|
|
+ register_new_node(opaque_stride, outer_main_head->in(LoopNode::EntryControl));
|
|
|
|
|
while (predicate != NULL && predicate->is_Proj() && predicate->in(0)->is_If()) {
|
|
|
|
|
iff = predicate->in(0)->as_If();
|
|
|
|
|
uncommon_proj = iff->proj_out(1 - predicate->as_Proj()->_con);
|
|
|
|
|
@@ -1108,11 +1112,10 @@ void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop_helper(Node* predicat
|
|
|
|
|
// Clone the skeleton predicate twice and initialize one with the initial
|
|
|
|
|
// value of the loop induction variable. Leave the other predicate
|
|
|
|
|
// to be initialized when increasing the stride during loop unrolling.
|
|
|
|
|
- prev_proj = clone_skeleton_predicate(iff, start, predicate, uncommon_proj, current_proj, outer_loop, prev_proj);
|
|
|
|
|
- assert(skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()) == (start->Opcode() == Op_Opaque1), "");
|
|
|
|
|
- prev_proj = clone_skeleton_predicate(iff, end, predicate, uncommon_proj, current_proj, outer_loop, prev_proj);
|
|
|
|
|
- assert(skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()) == (end->Opcode() == Op_Opaque1), "");
|
|
|
|
|
-
|
|
|
|
|
+ prev_proj = clone_skeleton_predicate(iff, opaque_init, NULL, predicate, uncommon_proj, current_proj, outer_loop, prev_proj);
|
|
|
|
|
+ assert(skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()), "");
|
|
|
|
|
+ prev_proj = clone_skeleton_predicate(iff, init, stride, predicate, uncommon_proj, current_proj, outer_loop, prev_proj);
|
|
|
|
|
+ assert(!skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()), "");
|
|
|
|
|
// Rewire any control inputs from the cloned skeleton predicates down to the main and post loop for data nodes that are part of the
|
|
|
|
|
// main loop (and were cloned to the pre and post loop).
|
|
|
|
|
for (DUIterator i = predicate->outs(); predicate->has_out(i); i++) {
|
|
|
|
|
@@ -1177,14 +1180,14 @@ bool PhaseIdealLoop::skeleton_predicate_has_opaque(IfNode* iff) {
|
|
|
|
|
}
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
- if (op == Op_Opaque1) {
|
|
|
|
|
+ if (n->is_Opaque1()) {
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
-Node* PhaseIdealLoop::clone_skeleton_predicate(Node* iff, Node* value, Node* predicate, Node* uncommon_proj,
|
|
|
|
|
+Node* PhaseIdealLoop::clone_skeleton_predicate(Node* iff, Node* new_init, Node* new_stride, Node* predicate, Node* uncommon_proj,
|
|
|
|
|
Node* current_proj, IdealLoopTree* outer_loop, Node* prev_proj) {
|
|
|
|
|
Node_Stack to_clone(2);
|
|
|
|
|
to_clone.push(iff->in(1), 1);
|
|
|
|
|
@@ -1204,12 +1207,19 @@ Node* PhaseIdealLoop::clone_skeleton_predicate(Node* iff, Node* value, Node* pre
|
|
|
|
|
to_clone.push(m, 1);
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
- if (op == Op_Opaque1) {
|
|
|
|
|
+ if (m->is_Opaque1()) {
|
|
|
|
|
if (n->_idx < current) {
|
|
|
|
|
n = n->clone();
|
|
|
|
|
+ register_new_node(n, current_proj);
|
|
|
|
|
+ }
|
|
|
|
|
+ if (op == Op_OpaqueLoopInit) {
|
|
|
|
|
+ n->set_req(i, new_init);
|
|
|
|
|
+ } else {
|
|
|
|
|
+ assert(op == Op_OpaqueLoopStride, "unexpected opaque node");
|
|
|
|
|
+ if (new_stride != NULL) {
|
|
|
|
|
+ n->set_req(i, new_stride);
|
|
|
|
|
+ }
|
|
|
|
|
}
|
|
|
|
|
- n->set_req(i, value);
|
|
|
|
|
- register_new_node(n, current_proj);
|
|
|
|
|
to_clone.set_node(n);
|
|
|
|
|
}
|
|
|
|
|
for (;;) {
|
|
|
|
|
@@ -1259,7 +1269,7 @@ Node* PhaseIdealLoop::clone_skeleton_predicate(Node* iff, Node* value, Node* pre
|
|
|
|
|
return proj;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
-void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop(CountedLoopNode* pre_head, Node* start, Node* end,
|
|
|
|
|
+void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop(CountedLoopNode* pre_head, Node* init, Node* stride,
|
|
|
|
|
IdealLoopTree* outer_loop, LoopNode* outer_main_head,
|
|
|
|
|
uint dd_main_head, const uint idx_before_pre_post,
|
|
|
|
|
const uint idx_after_post_before_pre, Node* zero_trip_guard_proj_main,
|
|
|
|
|
@@ -1279,10 +1289,10 @@ void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop(CountedLoopNode* pre_
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
|
|
|
|
|
- copy_skeleton_predicates_to_main_loop_helper(predicate, start, end, outer_loop, outer_main_head, dd_main_head,
|
|
|
|
|
+ copy_skeleton_predicates_to_main_loop_helper(predicate, init, stride, outer_loop, outer_main_head, dd_main_head,
|
|
|
|
|
idx_before_pre_post, idx_after_post_before_pre, zero_trip_guard_proj_main,
|
|
|
|
|
zero_trip_guard_proj_post, old_new);
|
|
|
|
|
- copy_skeleton_predicates_to_main_loop_helper(profile_predicate, start, end, outer_loop, outer_main_head, dd_main_head,
|
|
|
|
|
+ copy_skeleton_predicates_to_main_loop_helper(profile_predicate, init, stride, outer_loop, outer_main_head, dd_main_head,
|
|
|
|
|
idx_before_pre_post, idx_after_post_before_pre, zero_trip_guard_proj_main,
|
|
|
|
|
zero_trip_guard_proj_post, old_new);
|
|
|
|
|
}
|
|
|
|
|
@@ -1433,10 +1443,8 @@ void PhaseIdealLoop::insert_pre_post_loops( IdealLoopTree *loop, Node_List &old_
|
|
|
|
|
// CastII for the main loop:
|
|
|
|
|
Node* castii = cast_incr_before_loop( pre_incr, min_taken, main_head );
|
|
|
|
|
assert(castii != NULL, "no castII inserted");
|
|
|
|
|
- Node* opaque_castii = new Opaque1Node(C, castii);
|
|
|
|
|
- register_new_node(opaque_castii, outer_main_head->in(LoopNode::EntryControl));
|
|
|
|
|
assert(post_head->in(1)->is_IfProj(), "must be zero-trip guard If node projection of the post loop");
|
|
|
|
|
- copy_skeleton_predicates_to_main_loop(pre_head, castii, opaque_castii, outer_loop, outer_main_head, dd_main_head,
|
|
|
|
|
+ copy_skeleton_predicates_to_main_loop(pre_head, castii, stride, outer_loop, outer_main_head, dd_main_head,
|
|
|
|
|
idx_before_pre_post, idx_after_post_before_pre, min_taken, post_head->in(1), old_new);
|
|
|
|
|
|
|
|
|
|
// Step B4: Shorten the pre-loop to run only 1 iteration (for now).
|
|
|
|
|
@@ -1722,6 +1730,11 @@ void PhaseIdealLoop::update_main_loop_skeleton_predicates(Node* ctrl, CountedLoo
|
|
|
|
|
Node* prev_proj = ctrl;
|
|
|
|
|
LoopNode* outer_loop_head = loop_head->skip_strip_mined();
|
|
|
|
|
IdealLoopTree* outer_loop = get_loop(outer_loop_head);
|
|
|
|
|
+ // Compute the value of the loop induction variable at the end of the
|
|
|
|
|
+ // first iteration of the unrolled loop: init + new_stride_con - init_inc
|
|
|
|
|
+ int new_stride_con = stride_con * 2;
|
|
|
|
|
+ Node* max_value = _igvn.intcon(new_stride_con);
|
|
|
|
|
+ set_ctrl(max_value, C->root());
|
|
|
|
|
while (entry != NULL && entry->is_Proj() && entry->in(0)->is_If()) {
|
|
|
|
|
IfNode* iff = entry->in(0)->as_If();
|
|
|
|
|
ProjNode* proj = iff->proj_out(1 - entry->as_Proj()->_con);
|
|
|
|
|
@@ -1737,18 +1750,8 @@ void PhaseIdealLoop::update_main_loop_skeleton_predicates(Node* ctrl, CountedLoo
|
|
|
|
|
// tell. Kill it in any case.
|
|
|
|
|
_igvn.replace_input_of(iff, 1, iff->in(1)->in(2));
|
|
|
|
|
} else {
|
|
|
|
|
- // Add back the predicate for the value at the beginning of the first entry
|
|
|
|
|
- prev_proj = clone_skeleton_predicate(iff, init, entry, proj, ctrl, outer_loop, prev_proj);
|
|
|
|
|
- assert(!skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()), "unexpected");
|
|
|
|
|
- // Compute the value of the loop induction variable at the end of the
|
|
|
|
|
- // first iteration of the unrolled loop: init + new_stride_con - init_inc
|
|
|
|
|
- int init_inc = stride_con/loop_head->unrolled_count();
|
|
|
|
|
- assert(init_inc != 0, "invalid loop increment");
|
|
|
|
|
- int new_stride_con = stride_con * 2;
|
|
|
|
|
- Node* max_value = _igvn.intcon(new_stride_con - init_inc);
|
|
|
|
|
- max_value = new AddINode(init, max_value);
|
|
|
|
|
- register_new_node(max_value, get_ctrl(iff->in(1)));
|
|
|
|
|
- prev_proj = clone_skeleton_predicate(iff, max_value, entry, proj, ctrl, outer_loop, prev_proj);
|
|
|
|
|
+ //Add back predicates updated for the new stride.
|
|
|
|
|
+ prev_proj = clone_skeleton_predicate(iff, init, max_value, entry, proj, ctrl, outer_loop, prev_proj);
|
|
|
|
|
assert(!skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()), "unexpected");
|
|
|
|
|
}
|
|
|
|
|
}
|
2020-10-22 22:22:56 +08:00
|
|
|
@@ -2547,22 +2550,22 @@ int PhaseIdealLoop::do_range_check( IdealLoopTree *loop, Node_List &old_new ) {
|
|
|
|
|
// The underflow and overflow limits: 0 <= scale*I+offset < limit
|
|
|
|
|
add_constraint(stride_con, lscale_con, offset, zero, limit, pre_ctrl, &pre_limit, &main_limit);
|
2020-09-11 18:20:33 +08:00
|
|
|
Node* init = cl->init_trip();
|
|
|
|
|
- Node* opaque_init = new Opaque1Node(C, init);
|
|
|
|
|
+ Node* opaque_init = new OpaqueLoopInitNode(C, init);
|
|
|
|
|
register_new_node(opaque_init, predicate_proj);
|
2020-10-22 22:22:56 +08:00
|
|
|
+ // predicate on first value of first iteration
|
2020-10-26 12:13:57 +08:00
|
|
|
+ predicate_proj = add_range_check_predicate(loop, cl, predicate_proj, scale_con, int_offset, int_limit, stride_con, init);
|
2020-09-11 18:20:33 +08:00
|
|
|
+ assert(!skeleton_predicate_has_opaque(predicate_proj->in(0)->as_If()), "unexpected");
|
|
|
|
|
// template predicate so it can be updated on next unrolling
|
2020-10-22 22:22:56 +08:00
|
|
|
predicate_proj = add_range_check_predicate(loop, cl, predicate_proj, scale_con, int_offset, int_limit, stride_con, opaque_init);
|
2020-09-11 18:20:33 +08:00
|
|
|
assert(skeleton_predicate_has_opaque(predicate_proj->in(0)->as_If()), "unexpected");
|
|
|
|
|
- // predicate on first value of first iteration
|
2020-10-22 22:22:56 +08:00
|
|
|
- predicate_proj = add_range_check_predicate(loop, cl, predicate_proj, scale_con, int_offset, int_limit, stride_con, init);
|
2020-09-11 18:20:33 +08:00
|
|
|
- assert(!skeleton_predicate_has_opaque(predicate_proj->in(0)->as_If()), "unexpected");
|
|
|
|
|
- int init_inc = stride_con/cl->unrolled_count();
|
|
|
|
|
- assert(init_inc != 0, "invalid loop increment");
|
|
|
|
|
- Node* max_value = _igvn.intcon(stride_con - init_inc);
|
|
|
|
|
- max_value = new AddINode(init, max_value);
|
2020-10-22 22:22:56 +08:00
|
|
|
+ Node* opaque_stride = new OpaqueLoopStrideNode(C, cl->stride());
|
2020-09-11 18:20:33 +08:00
|
|
|
+ register_new_node(opaque_stride, predicate_proj);
|
|
|
|
|
+ Node* max_value = new SubINode(opaque_stride, cl->stride());
|
|
|
|
|
+ register_new_node(max_value, predicate_proj);
|
2020-10-22 22:22:56 +08:00
|
|
|
+ max_value = new AddINode(opaque_init, max_value);
|
2020-09-11 18:20:33 +08:00
|
|
|
register_new_node(max_value, predicate_proj);
|
|
|
|
|
- // predicate on last value of first iteration (in case unrolling has already happened)
|
2020-10-22 22:22:56 +08:00
|
|
|
predicate_proj = add_range_check_predicate(loop, cl, predicate_proj, scale_con, int_offset, int_limit, stride_con, max_value);
|
2020-09-11 18:20:33 +08:00
|
|
|
- assert(!skeleton_predicate_has_opaque(predicate_proj->in(0)->as_If()), "unexpected");
|
|
|
|
|
+ assert(skeleton_predicate_has_opaque(predicate_proj->in(0)->as_If()), "unexpected");
|
|
|
|
|
} else {
|
|
|
|
|
if (PrintOpto) {
|
|
|
|
|
tty->print_cr("missed RCE opportunity");
|
|
|
|
|
diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp
|
2020-10-22 22:22:56 +08:00
|
|
|
index 53b9692..7c54113 100644
|
2020-09-11 18:20:33 +08:00
|
|
|
--- a/src/hotspot/share/opto/loopnode.hpp
|
|
|
|
|
+++ b/src/hotspot/share/opto/loopnode.hpp
|
2020-10-22 22:22:56 +08:00
|
|
|
@@ -748,13 +748,13 @@ private:
|
2020-09-11 18:20:33 +08:00
|
|
|
#ifdef ASSERT
|
|
|
|
|
void ensure_zero_trip_guard_proj(Node* node, bool is_main_loop);
|
|
|
|
|
#endif
|
|
|
|
|
- void copy_skeleton_predicates_to_main_loop_helper(Node* predicate, Node* start, Node* end, IdealLoopTree* outer_loop, LoopNode* outer_main_head,
|
|
|
|
|
+ void copy_skeleton_predicates_to_main_loop_helper(Node* predicate, Node* init, Node* stride, IdealLoopTree* outer_loop, LoopNode* outer_main_head,
|
|
|
|
|
uint dd_main_head, const uint idx_before_pre_post, const uint idx_after_post_before_pre,
|
|
|
|
|
Node* zero_trip_guard_proj_main, Node* zero_trip_guard_proj_post, const Node_List &old_new);
|
|
|
|
|
- void copy_skeleton_predicates_to_main_loop(CountedLoopNode* pre_head, Node* start, Node* end, IdealLoopTree* outer_loop, LoopNode* outer_main_head,
|
|
|
|
|
+ void copy_skeleton_predicates_to_main_loop(CountedLoopNode* pre_head, Node* init, Node* stride, IdealLoopTree* outer_loop, LoopNode* outer_main_head,
|
|
|
|
|
uint dd_main_head, const uint idx_before_pre_post, const uint idx_after_post_before_pre,
|
|
|
|
|
Node* zero_trip_guard_proj_main, Node* zero_trip_guard_proj_post, const Node_List &old_new);
|
|
|
|
|
- Node* clone_skeleton_predicate(Node* iff, Node* value, Node* predicate, Node* uncommon_proj,
|
|
|
|
|
+ Node* clone_skeleton_predicate(Node* iff, Node* new_init, Node* new_stride, Node* predicate, Node* uncommon_proj,
|
|
|
|
|
Node* current_proj, IdealLoopTree* outer_loop, Node* prev_proj);
|
|
|
|
|
bool skeleton_predicate_has_opaque(IfNode* iff);
|
|
|
|
|
void update_main_loop_skeleton_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init, int stride_con);
|
|
|
|
|
diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp
|
|
|
|
|
index 6b85529..adbee17 100644
|
|
|
|
|
--- a/src/hotspot/share/opto/loopopts.cpp
|
|
|
|
|
+++ b/src/hotspot/share/opto/loopopts.cpp
|
|
|
|
|
@@ -890,30 +890,42 @@ void PhaseIdealLoop::try_move_store_after_loop(Node* n) {
|
|
|
|
|
Node *PhaseIdealLoop::split_if_with_blocks_pre( Node *n ) {
|
|
|
|
|
// Cloning these guys is unlikely to win
|
|
|
|
|
int n_op = n->Opcode();
|
|
|
|
|
- if( n_op == Op_MergeMem ) return n;
|
|
|
|
|
- if( n->is_Proj() ) return n;
|
|
|
|
|
+ if (n_op == Op_MergeMem) {
|
|
|
|
|
+ return n;
|
|
|
|
|
+ }
|
|
|
|
|
+ if (n->is_Proj()) {
|
|
|
|
|
+ return n;
|
|
|
|
|
+ }
|
|
|
|
|
// Do not clone-up CmpFXXX variations, as these are always
|
|
|
|
|
// followed by a CmpI
|
|
|
|
|
- if( n->is_Cmp() ) return n;
|
|
|
|
|
+ if (n->is_Cmp()) {
|
|
|
|
|
+ return n;
|
|
|
|
|
+ }
|
|
|
|
|
// Attempt to use a conditional move instead of a phi/branch
|
|
|
|
|
- if( ConditionalMoveLimit > 0 && n_op == Op_Region ) {
|
|
|
|
|
+ if (ConditionalMoveLimit > 0 && n_op == Op_Region) {
|
|
|
|
|
Node *cmov = conditional_move( n );
|
|
|
|
|
- if( cmov ) return cmov;
|
|
|
|
|
+ if (cmov) {
|
|
|
|
|
+ return cmov;
|
|
|
|
|
+ }
|
|
|
|
|
}
|
|
|
|
|
- if( n->is_CFG() || n->is_LoadStore() )
|
|
|
|
|
+ if (n->is_CFG() || n->is_LoadStore()) {
|
|
|
|
|
return n;
|
|
|
|
|
- if( n_op == Op_Opaque1 || // Opaque nodes cannot be mod'd
|
|
|
|
|
- n_op == Op_Opaque2 ) {
|
|
|
|
|
- if( !C->major_progress() ) // If chance of no more loop opts...
|
|
|
|
|
+ }
|
|
|
|
|
+ if (n->is_Opaque1() || // Opaque nodes cannot be mod'd
|
|
|
|
|
+ n_op == Op_Opaque2) {
|
|
|
|
|
+ if (!C->major_progress()) { // If chance of no more loop opts...
|
|
|
|
|
_igvn._worklist.push(n); // maybe we'll remove them
|
|
|
|
|
+ }
|
|
|
|
|
return n;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
- if( n->is_Con() ) return n; // No cloning for Con nodes
|
|
|
|
|
-
|
|
|
|
|
+ if (n->is_Con()) {
|
|
|
|
|
+ return n; // No cloning for Con nodes
|
|
|
|
|
+ }
|
|
|
|
|
Node *n_ctrl = get_ctrl(n);
|
|
|
|
|
- if( !n_ctrl ) return n; // Dead node
|
|
|
|
|
-
|
|
|
|
|
+ if (!n_ctrl) {
|
|
|
|
|
+ return n; // Dead node
|
|
|
|
|
+ }
|
|
|
|
|
Node* res = try_move_store_before_loop(n, n_ctrl);
|
|
|
|
|
if (res != NULL) {
|
|
|
|
|
return n;
|
|
|
|
|
diff --git a/src/hotspot/share/opto/macro.cpp b/src/hotspot/share/opto/macro.cpp
|
2020-10-22 22:22:56 +08:00
|
|
|
index cf8eab1..8331d25 100644
|
2020-09-11 18:20:33 +08:00
|
|
|
--- a/src/hotspot/share/opto/macro.cpp
|
|
|
|
|
+++ b/src/hotspot/share/opto/macro.cpp
|
2020-10-22 22:22:56 +08:00
|
|
|
@@ -2617,9 +2617,10 @@ void PhaseMacroExpand::eliminate_macro_nodes() {
|
2020-09-11 18:20:33 +08:00
|
|
|
break;
|
|
|
|
|
case Node::Class_OuterStripMinedLoop:
|
|
|
|
|
break;
|
|
|
|
|
+ case Node::Class_Opaque1:
|
|
|
|
|
+ break;
|
|
|
|
|
default:
|
|
|
|
|
assert(n->Opcode() == Op_LoopLimit ||
|
|
|
|
|
- n->Opcode() == Op_Opaque1 ||
|
|
|
|
|
n->Opcode() == Op_Opaque2 ||
|
|
|
|
|
n->Opcode() == Op_Opaque3 ||
|
|
|
|
|
BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(n),
|
2020-10-22 22:22:56 +08:00
|
|
|
@@ -2661,7 +2662,7 @@ bool PhaseMacroExpand::expand_macro_nodes() {
|
2020-09-11 18:20:33 +08:00
|
|
|
C->remove_macro_node(n);
|
|
|
|
|
_igvn._worklist.push(n);
|
|
|
|
|
success = true;
|
|
|
|
|
- } else if (n->Opcode() == Op_Opaque1 || n->Opcode() == Op_Opaque2) {
|
|
|
|
|
+ } else if (n->is_Opaque1() || n->Opcode() == Op_Opaque2) {
|
|
|
|
|
_igvn.replace_node(n, n->in(1));
|
|
|
|
|
success = true;
|
|
|
|
|
#if INCLUDE_RTM_OPT
|
|
|
|
|
diff --git a/src/hotspot/share/opto/node.hpp b/src/hotspot/share/opto/node.hpp
|
2020-10-22 22:22:56 +08:00
|
|
|
index 2a42e99..0eac634 100644
|
2020-09-11 18:20:33 +08:00
|
|
|
--- a/src/hotspot/share/opto/node.hpp
|
|
|
|
|
+++ b/src/hotspot/share/opto/node.hpp
|
|
|
|
|
@@ -116,6 +116,7 @@ class MulNode;
|
|
|
|
|
class MultiNode;
|
|
|
|
|
class MultiBranchNode;
|
|
|
|
|
class NeverBranchNode;
|
|
|
|
|
+class Opaque1Node;
|
|
|
|
|
class OuterStripMinedLoopNode;
|
|
|
|
|
class OuterStripMinedLoopEndNode;
|
|
|
|
|
class Node;
|
2020-10-22 22:22:56 +08:00
|
|
|
@@ -611,10 +612,10 @@ public:
|
2020-09-11 18:20:33 +08:00
|
|
|
// This enum is used only for C2 ideal and mach nodes with is_<node>() methods
|
|
|
|
|
// so that it's values fits into 16 bits.
|
|
|
|
|
enum NodeClasses {
|
|
|
|
|
- Bit_Node = 0x0000,
|
|
|
|
|
- Class_Node = 0x0000,
|
|
|
|
|
- ClassMask_Node = 0xFFFF,
|
|
|
|
|
-
|
|
|
|
|
+ Bit_Node = 0x00000000,
|
|
|
|
|
+ Class_Node = 0x00000000,
|
|
|
|
|
+ ClassMask_Node = 0xFFFFFFFF,
|
|
|
|
|
+
|
|
|
|
|
DEFINE_CLASS_ID(Multi, Node, 0)
|
|
|
|
|
DEFINE_CLASS_ID(SafePoint, Multi, 0)
|
|
|
|
|
DEFINE_CLASS_ID(Call, SafePoint, 0)
|
2020-10-22 22:22:56 +08:00
|
|
|
@@ -720,6 +721,7 @@ public:
|
2020-09-11 18:20:33 +08:00
|
|
|
DEFINE_CLASS_ID(Vector, Node, 13)
|
|
|
|
|
DEFINE_CLASS_ID(ClearArray, Node, 14)
|
|
|
|
|
DEFINE_CLASS_ID(Halt, Node, 15)
|
|
|
|
|
+ DEFINE_CLASS_ID(Opaque1, Node, 16)
|
|
|
|
|
|
|
|
|
|
_max_classes = ClassMask_Halt
|
|
|
|
|
};
|
2020-10-22 22:22:56 +08:00
|
|
|
@@ -746,12 +748,12 @@ public:
|
2020-09-11 18:20:33 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
private:
|
|
|
|
|
- jushort _class_id;
|
|
|
|
|
+ juint _class_id;
|
|
|
|
|
jushort _flags;
|
|
|
|
|
|
|
|
|
|
protected:
|
|
|
|
|
// These methods should be called from constructors only.
|
|
|
|
|
- void init_class_id(jushort c) {
|
|
|
|
|
+ void init_class_id(juint c) {
|
|
|
|
|
_class_id = c; // cast out const
|
|
|
|
|
}
|
|
|
|
|
void init_flags(jushort fl) {
|
2020-10-22 22:22:56 +08:00
|
|
|
@@ -764,7 +766,7 @@ protected:
|
2020-09-11 18:20:33 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public:
|
|
|
|
|
- const jushort class_id() const { return _class_id; }
|
|
|
|
|
+ const juint class_id() const { return _class_id; }
|
|
|
|
|
|
|
|
|
|
const jushort flags() const { return _flags; }
|
|
|
|
|
|
2020-10-22 22:22:56 +08:00
|
|
|
@@ -865,6 +867,7 @@ public:
|
2020-09-11 18:20:33 +08:00
|
|
|
DEFINE_CLASS_QUERY(Mul)
|
|
|
|
|
DEFINE_CLASS_QUERY(Multi)
|
|
|
|
|
DEFINE_CLASS_QUERY(MultiBranch)
|
|
|
|
|
+ DEFINE_CLASS_QUERY(Opaque1)
|
|
|
|
|
DEFINE_CLASS_QUERY(OuterStripMinedLoop)
|
|
|
|
|
DEFINE_CLASS_QUERY(OuterStripMinedLoopEnd)
|
|
|
|
|
DEFINE_CLASS_QUERY(Parm)
|
|
|
|
|
diff --git a/src/hotspot/share/opto/opaquenode.hpp b/src/hotspot/share/opto/opaquenode.hpp
|
|
|
|
|
index f97de4a..4c00528 100644
|
|
|
|
|
--- a/src/hotspot/share/opto/opaquenode.hpp
|
|
|
|
|
+++ b/src/hotspot/share/opto/opaquenode.hpp
|
|
|
|
|
@@ -38,6 +38,7 @@ class Opaque1Node : public Node {
|
|
|
|
|
Opaque1Node(Compile* C, Node *n) : Node(NULL, n) {
|
|
|
|
|
// Put it on the Macro nodes list to removed during macro nodes expansion.
|
|
|
|
|
init_flags(Flag_is_macro);
|
|
|
|
|
+ init_class_id(Class_Opaque1);
|
|
|
|
|
C->add_macro_node(this);
|
|
|
|
|
}
|
|
|
|
|
// Special version for the pre-loop to hold the original loop limit
|
|
|
|
|
@@ -45,6 +46,7 @@ class Opaque1Node : public Node {
|
|
|
|
|
Opaque1Node(Compile* C, Node *n, Node* orig_limit) : Node(NULL, n, orig_limit) {
|
|
|
|
|
// Put it on the Macro nodes list to removed during macro nodes expansion.
|
|
|
|
|
init_flags(Flag_is_macro);
|
|
|
|
|
+ init_class_id(Class_Opaque1);
|
|
|
|
|
C->add_macro_node(this);
|
|
|
|
|
}
|
|
|
|
|
Node* original_loop_limit() { return req()==3 ? in(2) : NULL; }
|
|
|
|
|
@@ -52,6 +54,20 @@ class Opaque1Node : public Node {
|
|
|
|
|
virtual const Type *bottom_type() const { return TypeInt::INT; }
|
|
|
|
|
virtual Node* Identity(PhaseGVN* phase);
|
|
|
|
|
};
|
|
|
|
|
+// Opaque nodes specific to range check elimination handling
|
|
|
|
|
+class OpaqueLoopInitNode : public Opaque1Node {
|
|
|
|
|
+ public:
|
|
|
|
|
+ OpaqueLoopInitNode(Compile* C, Node *n) : Opaque1Node(C, n) {
|
|
|
|
|
+ }
|
|
|
|
|
+ virtual int Opcode() const;
|
|
|
|
|
+};
|
|
|
|
|
+
|
|
|
|
|
+class OpaqueLoopStrideNode : public Opaque1Node {
|
|
|
|
|
+ public:
|
|
|
|
|
+ OpaqueLoopStrideNode(Compile* C, Node *n) : Opaque1Node(C, n) {
|
|
|
|
|
+ }
|
|
|
|
|
+ virtual int Opcode() const;
|
|
|
|
|
+};
|
|
|
|
|
|
|
|
|
|
//------------------------------Opaque2Node------------------------------------
|
|
|
|
|
// A node to prevent unwanted optimizations. Allows constant folding. Stops
|
|
|
|
|
diff --git a/src/hotspot/share/runtime/vmStructs.cpp b/src/hotspot/share/runtime/vmStructs.cpp
|
2020-10-22 22:22:56 +08:00
|
|
|
index ad7fb42..85c4dfe 100644
|
2020-09-11 18:20:33 +08:00
|
|
|
--- a/src/hotspot/share/runtime/vmStructs.cpp
|
|
|
|
|
+++ b/src/hotspot/share/runtime/vmStructs.cpp
|
2020-10-22 22:22:56 +08:00
|
|
|
@@ -951,7 +951,7 @@ typedef PaddedEnd<ObjectMonitor> PaddedObjectMonitor;
|
2020-09-11 18:20:33 +08:00
|
|
|
c2_nonstatic_field(Node, _outcnt, node_idx_t) \
|
|
|
|
|
c2_nonstatic_field(Node, _outmax, node_idx_t) \
|
|
|
|
|
c2_nonstatic_field(Node, _idx, const node_idx_t) \
|
|
|
|
|
- c2_nonstatic_field(Node, _class_id, jushort) \
|
|
|
|
|
+ c2_nonstatic_field(Node, _class_id, juint) \
|
|
|
|
|
c2_nonstatic_field(Node, _flags, jushort) \
|
|
|
|
|
\
|
|
|
|
|
c2_nonstatic_field(Compile, _root, RootNode*) \
|
|
|
|
|
diff --git a/test/hotspot/jtreg/compiler/loopopts/TestRCEAfterUnrolling.java b/test/hotspot/jtreg/compiler/loopopts/TestRCEAfterUnrolling.java
|
|
|
|
|
new file mode 100644
|
|
|
|
|
index 0000000..06bca79
|
|
|
|
|
--- /dev/null
|
|
|
|
|
+++ b/test/hotspot/jtreg/compiler/loopopts/TestRCEAfterUnrolling.java
|
|
|
|
|
@@ -0,0 +1,78 @@
|
|
|
|
|
+/*
|
|
|
|
|
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
|
|
|
|
|
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
|
|
|
|
|
+ *
|
|
|
|
|
+ * This code is free software; you can redistribute it and/or modify it
|
|
|
|
|
+ * under the terms of the GNU General Public License version 2 only, as
|
|
|
|
|
+ * published by the Free Software Foundation.
|
|
|
|
|
+ *
|
|
|
|
|
+ * This code is distributed in the hope that it will be useful, but WITHOUT
|
|
|
|
|
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
|
|
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
|
|
|
+ * version 2 for more details (a copy is included in the LICENSE file that
|
|
|
|
|
+ * accompanied this code).
|
|
|
|
|
+ *
|
|
|
|
|
+ * You should have received a copy of the GNU General Public License version
|
|
|
|
|
+ * 2 along with this work; if not, write to the Free Software Foundation,
|
|
|
|
|
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
|
|
|
+ *
|
|
|
|
|
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
|
|
|
|
|
+ * or visit www.oracle.com if you need additional information or have any
|
|
|
|
|
+ * questions.
|
|
|
|
|
+ */
|
|
|
|
|
+
|
|
|
|
|
+/*
|
|
|
|
|
+ * @test
|
|
|
|
|
+ * @bug 8229495
|
|
|
|
|
+ * @summary SIGILL in C2 generated OSR compilation.
|
|
|
|
|
+ *
|
|
|
|
|
+ * @run main/othervm -Xcomp -XX:-TieredCompilation -XX:CompileOnly=TestRCEAfterUnrolling::test TestRCEAfterUnrolling
|
|
|
|
|
+ *
|
|
|
|
|
+ */
|
|
|
|
|
+
|
|
|
|
|
+public class TestRCEAfterUnrolling {
|
|
|
|
|
+
|
|
|
|
|
+ public static int iFld = 0;
|
|
|
|
|
+ public static short sFld = 1;
|
|
|
|
|
+
|
|
|
|
|
+ public static void main(String[] strArr) {
|
|
|
|
|
+ test();
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ public static int test() {
|
|
|
|
|
+ int x = 11;
|
|
|
|
|
+ int y = 0;
|
|
|
|
|
+ int j = 0;
|
|
|
|
|
+ int iArr[] = new int[400];
|
|
|
|
|
+
|
|
|
|
|
+ init(iArr);
|
|
|
|
|
+
|
|
|
|
|
+ for (int i = 0; i < 2; i++) {
|
|
|
|
|
+ doNothing();
|
|
|
|
|
+ for (j = 10; j > 1; j -= 2) {
|
|
|
|
|
+ sFld += (short)j;
|
|
|
|
|
+ iArr = iArr;
|
|
|
|
|
+ y += (j * 3);
|
|
|
|
|
+ x = (iArr[j - 1]/ x);
|
|
|
|
|
+ x = sFld;
|
|
|
|
|
+ }
|
|
|
|
|
+ int k = 1;
|
|
|
|
|
+ while (++k < 8) {
|
|
|
|
|
+ iFld += x;
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ return Float.floatToIntBits(654) + x + j + y;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ // Inlined
|
|
|
|
|
+ public static void doNothing() {
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ // Inlined
|
|
|
|
|
+ public static void init(int[] a) {
|
|
|
|
|
+ for (int j = 0; j < a.length; j++) {
|
|
|
|
|
+ a[j] = 0;
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+}
|
|
|
|
|
+
|