Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport] 8255000: C2: Unify IGVN processing when loop opts are over #916

Merged
merged 3 commits into from
Jan 9, 2025
Merged

[Backport] 8255000: C2: Unify IGVN processing when loop opts are over #916

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2918a77
[JIT] Backport 8255000: C2: Unify IGVN processing when loop opts are …
kuaiwei Dec 30, 2024
acbb8dc
[JIT] Backport 8255400: Shenandoah: C2 failures after JDK-8255000
kuaiwei Dec 30, 2024
46a012b
[JIT] Backport 8255120: C2: assert(outer->outcnt() >= phis + 2 && out…
kuaiwei Dec 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 17 additions & 11 deletions src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) {
ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
if ((state->iu_barriers_count() +
state->load_reference_barriers_count()) > 0) {
assert(C->post_loop_opts_phase(), "no loop opts allowed");
C->reset_post_loop_opts_phase(); // ... but we know what we are doing
bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion;
C->clear_major_progress();
PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand);
Expand All @@ -58,12 +60,10 @@ bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) {
return false;
}
C->clear_major_progress();
if (C->range_check_cast_count() > 0) {
// No more loop optimizations. Remove all range check dependent CastIINodes.
C->remove_range_check_casts(igvn);
igvn.optimize();
}

C->process_for_post_loop_opts_igvn(igvn);
}
C->set_post_loop_opts_phase(); // now for real!
}
return true;
}
Expand Down Expand Up @@ -2470,12 +2470,12 @@ void MemoryGraphFixer::collect_memory_nodes() {
Node* m = _memory_nodes[c->in(k)->_idx];
assert(m != NULL, "expect memory state");
if (u->in(k) != m) {
phi = NULL;
phi = NodeSentinel;
}
}
}
}
if (phi == NULL) {
if (phi == NodeSentinel) {
phi = new PhiNode(c, Type::MEMORY, _phase->C->get_adr_type(_alias));
for (uint k = 1; k < c->req(); k++) {
Node* m = _memory_nodes[c->in(k)->_idx];
Expand All @@ -2484,8 +2484,11 @@ void MemoryGraphFixer::collect_memory_nodes() {
}
}
}
assert(phi != NULL, "");
regions.map(c->_idx, phi);
if (phi != NULL) {
regions.map(c->_idx, phi);
} else {
assert(c->unique_ctrl_out()->Opcode() == Op_Halt, "expected memory state");
}
}
Node* current_region = regions[c->_idx];
if (current_region != prev_region) {
Expand All @@ -2496,7 +2499,7 @@ void MemoryGraphFixer::collect_memory_nodes() {
}
} else if (prev_mem == NULL || prev_mem->is_Phi() || ctrl_or_self(prev_mem) != c) {
Node* m = _memory_nodes[_phase->idom(c)->_idx];
assert(m != NULL, "expect memory state");
assert(m != NULL || c->Opcode() == Op_Halt, "expect memory state");
if (m != prev_mem) {
_memory_nodes.map(c->_idx, m);
progress = true;
Expand All @@ -2520,7 +2523,8 @@ void MemoryGraphFixer::collect_memory_nodes() {
Node* c = rpo_list.at(i);
if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) {
Node* n = regions[c->_idx];
if (n->is_Phi() && n->_idx >= last && n->in(0) == c) {
assert(n != NULL || c->unique_ctrl_out()->Opcode() == Op_Halt, "expected memory state");
if (n != NULL && n->is_Phi() && n->_idx >= last && n->in(0) == c) {
_phase->register_new_node(n, c);
}
}
Expand All @@ -2529,10 +2533,12 @@ void MemoryGraphFixer::collect_memory_nodes() {
Node* c = rpo_list.at(i);
if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) {
Node* n = regions[c->_idx];
assert(n != NULL || c->unique_ctrl_out()->Opcode() == Op_Halt, "expected memory state");
for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
Node* u = c->fast_out(i);
if (u->is_Phi() && u->bottom_type() == Type::MEMORY &&
u != n) {
assert(c->unique_ctrl_out()->Opcode() != Op_Halt, "expected memory state");
if (u->adr_type() == TypePtr::BOTTOM) {
fix_memory_uses(u, n, n, c);
} else if (_phase->C->get_alias_index(u->adr_type()) == _alias) {
Expand Down
69 changes: 44 additions & 25 deletions src/hotspot/share/opto/castnode.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,37 +231,56 @@ Node *CastIINode::Ideal(PhaseGVN *phase, bool can_reshape) {
// Do not narrow the type of range check dependent CastIINodes to
// avoid corruption of the graph if a CastII is replaced by TOP but
// the corresponding range check is not removed.
if (can_reshape && !_range_check_dependency && !phase->C->major_progress()) {
const TypeInt* this_type = this->type()->is_int();
const TypeInt* in_type = phase->type(in(1))->isa_int();
if (in_type != NULL && this_type != NULL &&
(in_type->_lo != this_type->_lo ||
in_type->_hi != this_type->_hi)) {
jint lo1 = this_type->_lo;
jint hi1 = this_type->_hi;
int w1 = this_type->_widen;

if (lo1 >= 0) {
// Keep a range assertion of >=0.
lo1 = 0; hi1 = max_jint;
} else if (hi1 < 0) {
// Keep a range assertion of <0.
lo1 = min_jint; hi1 = -1;
} else {
lo1 = min_jint; hi1 = max_jint;
}
const TypeInt* wtype = TypeInt::make(MAX2(in_type->_lo, lo1),
MIN2(in_type->_hi, hi1),
MAX2((int)in_type->_widen, w1));
if (wtype != type()) {
set_type(wtype);
return this;
if (can_reshape && !_range_check_dependency) {
if (phase->C->post_loop_opts_phase()) {
const TypeInt* this_type = this->type()->is_int();
const TypeInt* in_type = phase->type(in(1))->isa_int();
if (in_type != NULL && this_type != NULL &&
(in_type->_lo != this_type->_lo ||
in_type->_hi != this_type->_hi)) {
jint lo1 = this_type->_lo;
jint hi1 = this_type->_hi;
int w1 = this_type->_widen;

if (lo1 >= 0) {
// Keep a range assertion of >=0.
lo1 = 0; hi1 = max_jint;
} else if (hi1 < 0) {
// Keep a range assertion of <0.
lo1 = min_jint; hi1 = -1;
} else {
lo1 = min_jint; hi1 = max_jint;
}
const TypeInt* wtype = TypeInt::make(MAX2(in_type->_lo, lo1),
MIN2(in_type->_hi, hi1),
MAX2((int)in_type->_widen, w1));
if (wtype != type()) {
set_type(wtype);
return this;
}
}
} else {
phase->C->record_for_post_loop_opts_igvn(this);
}
}
return NULL;
}

Node* CastIINode::Identity(PhaseGVN* phase) {
Node* progress = ConstraintCastNode::Identity(phase);
if (progress != this) {
return progress;
}
if (_range_check_dependency) {
if (phase->C->post_loop_opts_phase()) {
return this->in(1);
} else {
phase->C->record_for_post_loop_opts_igvn(this);
}
}
return this;
}

uint CastIINode::cmp(const Node &n) const {
return ConstraintCastNode::cmp(n) && ((CastIINode&)n)._range_check_dependency == _range_check_dependency;
}
Expand Down
1 change: 1 addition & 0 deletions src/hotspot/share/opto/castnode.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ class CastIINode: public ConstraintCastNode {
}
virtual int Opcode() const;
virtual uint ideal_reg() const { return Op_RegI; }
virtual Node* Identity(PhaseGVN* phase);
virtual const Type* Value(PhaseGVN* phase) const;
virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
const bool has_range_check() {
Expand Down
111 changes: 39 additions & 72 deletions src/hotspot/share/opto/compile.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,15 @@ void Compile::remove_useless_late_inlines(GrowableArray<CallGenerator*>* inlines
inlines->trunc_to(inlines->length()-shift);
}

void Compile::remove_useless_nodes(GrowableArray<Node*>& node_list, Unique_Node_List& useful) {
for (int i = node_list.length() - 1; i >= 0; i--) {
Node* node = node_list.at(i);
if (!useful.member(node)) {
node_list.delete_at(i); // replaces i-th with last element which is known to be useful (already processed)
}
}
}

// Disconnect all useless nodes by disconnecting those at the boundary.
void Compile::remove_useless_nodes(Unique_Node_List &useful) {
uint next = 0;
Expand Down Expand Up @@ -401,27 +410,14 @@ void Compile::remove_useless_nodes(Unique_Node_List &useful) {
remove_macro_node(n);
}
}
// Remove useless CastII nodes with range check dependency
for (int i = range_check_cast_count() - 1; i >= 0; i--) {
Node* cast = range_check_cast_node(i);
if (!useful.member(cast)) {
remove_range_check_cast(cast);
}
}
// Remove useless expensive nodes
for (int i = C->expensive_count()-1; i >= 0; i--) {
Node* n = C->expensive_node(i);
if (!useful.member(n)) {
remove_expensive_node(n);
}
}
// Remove useless Opaque4 nodes
for (int i = opaque4_count() - 1; i >= 0; i--) {
Node* opaq = opaque4_node(i);
if (!useful.member(opaq)) {
remove_opaque4_node(opaq);
}
}
remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass
remove_useless_coarsened_locks(useful); // remove useless coarsened locks nodes
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
bs->eliminate_useless_gc_barriers(useful);
Expand Down Expand Up @@ -677,6 +673,7 @@ Compile::Compile( ciEnv* ci_env, C2Compiler* compiler, ciMethod* target, int osr
_inner_loops(0),
_scratch_const_size(-1),
_in_scratch_emit_size(false),
_for_post_loop_igvn(comp_arena(), 8, 0, NULL),
_coarsened_locks (comp_arena(), 8, 0, NULL),
_dead_node_list(comp_arena()),
_dead_node_count(0),
Expand Down Expand Up @@ -705,6 +702,7 @@ Compile::Compile( ciEnv* ci_env, C2Compiler* compiler, ciMethod* target, int osr
_print_inlining_output(NULL),
_interpreter_frame_size(0),
_max_node_limit(MaxNodeLimit),
_post_loop_opts_phase(false),
_has_reserved_stack_access(target->has_reserved_stack_access()) {
C = this;
CompileWrapper cw(this);
Expand Down Expand Up @@ -1007,6 +1005,7 @@ Compile::Compile( ciEnv* ci_env,
_Compile_types(mtCompiler),
_dead_node_list(comp_arena()),
_dead_node_count(0),
_for_post_loop_igvn(comp_arena(), 8, 0, NULL),
_congraph(NULL),
_replay_inline_data(NULL),
_number_of_mh_late_inlines(0),
Expand All @@ -1019,6 +1018,7 @@ Compile::Compile( ciEnv* ci_env,
_allowed_reasons(0),
_interpreter_frame_size(0),
_max_node_limit(MaxNodeLimit),
_post_loop_opts_phase(false),
_has_reserved_stack_access(false) {
C = this;

Expand Down Expand Up @@ -1203,8 +1203,6 @@ void Compile::Init(int aliaslevel) {
_macro_nodes = new(comp_arena()) GrowableArray<Node*>(comp_arena(), 8, 0, NULL);
_predicate_opaqs = new(comp_arena()) GrowableArray<Node*>(comp_arena(), 8, 0, NULL);
_expensive_nodes = new(comp_arena()) GrowableArray<Node*>(comp_arena(), 8, 0, NULL);
_range_check_casts = new(comp_arena()) GrowableArray<Node*>(comp_arena(), 8, 0, NULL);
_opaque4_nodes = new(comp_arena()) GrowableArray<Node*>(comp_arena(), 8, 0, NULL);
register_library_intrinsics();
#ifdef ASSERT
_type_verify_symmetry = true;
Expand Down Expand Up @@ -1977,46 +1975,35 @@ void Compile::cleanup_loop_predicates(PhaseIterGVN &igvn) {
assert(predicate_count()==0, "should be clean!");
}

void Compile::add_range_check_cast(Node* n) {
assert(n->isa_CastII()->has_range_check(), "CastII should have range check dependency");
assert(!_range_check_casts->contains(n), "duplicate entry in range check casts");
_range_check_casts->append(n);
}

// Remove all range check dependent CastIINodes.
void Compile::remove_range_check_casts(PhaseIterGVN &igvn) {
for (int i = range_check_cast_count(); i > 0; i--) {
Node* cast = range_check_cast_node(i-1);
assert(cast->isa_CastII()->has_range_check(), "CastII should have range check dependency");
igvn.replace_node(cast, cast->in(1));
void Compile::record_for_post_loop_opts_igvn(Node* n) {
if (!n->for_post_loop_opts_igvn()) {
assert(!_for_post_loop_igvn.contains(n), "duplicate");
n->add_flag(Node::Flag_for_post_loop_opts_igvn);
_for_post_loop_igvn.append(n);
}
assert(range_check_cast_count() == 0, "should be empty");
}

void Compile::add_opaque4_node(Node* n) {
assert(n->Opcode() == Op_Opaque4, "Opaque4 only");
assert(!_opaque4_nodes->contains(n), "duplicate entry in Opaque4 list");
_opaque4_nodes->append(n);
void Compile::remove_from_post_loop_opts_igvn(Node* n) {
n->remove_flag(Node::Flag_for_post_loop_opts_igvn);
_for_post_loop_igvn.remove(n);
}

// Remove all Opaque4 nodes.
void Compile::remove_opaque4_nodes(PhaseIterGVN &igvn) {
for (int i = opaque4_count(); i > 0; i--) {
Node* opaq = opaque4_node(i-1);
assert(opaq->Opcode() == Op_Opaque4, "Opaque4 only");
// With Opaque4 nodes, the expectation is that the test of input 1
// is always equal to the constant value of input 2. So we can
// remove the Opaque4 and replace it by input 2. In debug builds,
// leave the non constant test in instead to sanity check that it
// never fails (if it does, that subgraph was constructed so, at
// runtime, a Halt node is executed).
#ifdef ASSERT
igvn.replace_node(opaq, opaq->in(1));
#else
igvn.replace_node(opaq, opaq->in(2));
#endif
void Compile::process_for_post_loop_opts_igvn(PhaseIterGVN& igvn) {
// Verify that all previous optimizations produced a valid graph
// at least to this point, even if no loop optimizations were done.
PhaseIdealLoop::verify(igvn);

C->set_post_loop_opts_phase(); // no more loop opts allowed

if (_for_post_loop_igvn.length() > 0) {
while (_for_post_loop_igvn.length() > 0) {
Node* n = _for_post_loop_igvn.pop();
n->remove_flag(Node::Flag_for_post_loop_opts_igvn);
igvn._worklist.push(n);
}
igvn.optimize();
assert(_for_post_loop_igvn.length() == 0, "no more delayed nodes allowed");
}
assert(opaque4_count() == 0, "should be empty");
}

// StringOpts and late inlining of string methods
Expand Down Expand Up @@ -2387,7 +2374,6 @@ void Compile::Optimize() {
}
if (!failing()) {
// Verify that last round of loop opts produced a valid graph
TracePhase tp("idealLoopVerify", &timers[_t_idealLoopVerify]);
PhaseIdealLoop::verify(igvn);
}
}
Expand Down Expand Up @@ -2423,21 +2409,9 @@ void Compile::Optimize() {

if (failing()) return;

// Ensure that major progress is now clear
C->clear_major_progress();

{
// Verify that all previous optimizations produced a valid graph
// at least to this point, even if no loop optimizations were done.
TracePhase tp("idealLoopVerify", &timers[_t_idealLoopVerify]);
PhaseIdealLoop::verify(igvn);
}
C->clear_major_progress(); // ensure that major progress is now clear

if (range_check_cast_count() > 0) {
// No more loop optimizations. Remove all range check dependent CastIINodes.
C->remove_range_check_casts(igvn);
igvn.optimize();
}
process_for_post_loop_opts_igvn(igvn);

#ifdef ASSERT
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
Expand All @@ -2464,11 +2438,6 @@ void Compile::Optimize() {
}
#endif

if (opaque4_count() > 0) {
C->remove_opaque4_nodes(igvn);
igvn.optimize();
}

if (C->max_vector_size() > 0) {
C->optimize_logic_cones(igvn);
igvn.optimize();
Expand Down Expand Up @@ -4768,8 +4737,6 @@ Node* Compile::constrained_convI2L(PhaseGVN* phase, Node* value, const TypeInt*
// ConvI2L node may be eliminated independently of the range check, causing the data path
// to become TOP while the control path is still there (although it's unreachable).
value->set_req(0, ctrl);
// Save CastII node to remove it after loop optimizations.
phase->C->add_range_check_cast(value);
value = phase->transform(value);
}
const TypeLong* ltype = TypeLong::make(itype->_lo, itype->_hi, itype->_widen);
Expand Down
Loading
Loading