diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp index d00d9cf941a..bb24ff28bf3 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp @@ -45,6 +45,8 @@ bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) { ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state(); if ((state->iu_barriers_count() + state->load_reference_barriers_count()) > 0) { + assert(C->post_loop_opts_phase(), "no loop opts allowed"); + C->reset_post_loop_opts_phase(); // ... but we know what we are doing bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion; C->clear_major_progress(); PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand); @@ -58,12 +60,10 @@ bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) { return false; } C->clear_major_progress(); - if (C->range_check_cast_count() > 0) { - // No more loop optimizations. Remove all range check dependent CastIINodes. - C->remove_range_check_casts(igvn); - igvn.optimize(); - } + + C->process_for_post_loop_opts_igvn(igvn); } + C->set_post_loop_opts_phase(); // now for real! } return true; } @@ -2470,12 +2470,12 @@ void MemoryGraphFixer::collect_memory_nodes() { Node* m = _memory_nodes[c->in(k)->_idx]; assert(m != NULL, "expect memory state"); if (u->in(k) != m) { - phi = NULL; + phi = NodeSentinel; } } } } - if (phi == NULL) { + if (phi == NodeSentinel) { phi = new PhiNode(c, Type::MEMORY, _phase->C->get_adr_type(_alias)); for (uint k = 1; k < c->req(); k++) { Node* m = _memory_nodes[c->in(k)->_idx]; @@ -2484,8 +2484,11 @@ void MemoryGraphFixer::collect_memory_nodes() { } } } - assert(phi != NULL, ""); - regions.map(c->_idx, phi); + if (phi != NULL) { + regions.map(c->_idx, phi); + } else { + assert(c->unique_ctrl_out()->Opcode() == Op_Halt, "expected memory state"); + } } Node* current_region = regions[c->_idx]; if (current_region != prev_region) { @@ -2496,7 +2499,7 @@ void MemoryGraphFixer::collect_memory_nodes() { } } else if (prev_mem == NULL || prev_mem->is_Phi() || ctrl_or_self(prev_mem) != c) { Node* m = _memory_nodes[_phase->idom(c)->_idx]; - assert(m != NULL, "expect memory state"); + assert(m != NULL || c->Opcode() == Op_Halt, "expect memory state"); if (m != prev_mem) { _memory_nodes.map(c->_idx, m); progress = true; @@ -2520,7 +2523,8 @@ void MemoryGraphFixer::collect_memory_nodes() { Node* c = rpo_list.at(i); if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) { Node* n = regions[c->_idx]; - if (n->is_Phi() && n->_idx >= last && n->in(0) == c) { + assert(n != NULL || c->unique_ctrl_out()->Opcode() == Op_Halt, "expected memory state"); + if (n != NULL && n->is_Phi() && n->_idx >= last && n->in(0) == c) { _phase->register_new_node(n, c); } } @@ -2529,10 +2533,12 @@ void MemoryGraphFixer::collect_memory_nodes() { Node* c = rpo_list.at(i); if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) { Node* n = regions[c->_idx]; + assert(n != NULL || c->unique_ctrl_out()->Opcode() == Op_Halt, "expected memory state"); for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { Node* u = c->fast_out(i); if (u->is_Phi() && u->bottom_type() == Type::MEMORY && u != n) { + assert(c->unique_ctrl_out()->Opcode() != Op_Halt, "expected memory state"); if (u->adr_type() == TypePtr::BOTTOM) { fix_memory_uses(u, n, n, c); } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) { diff --git a/src/hotspot/share/opto/castnode.cpp b/src/hotspot/share/opto/castnode.cpp index d6cd108487f..d499188d9ab 100644 --- a/src/hotspot/share/opto/castnode.cpp +++ b/src/hotspot/share/opto/castnode.cpp @@ -231,37 +231,56 @@ Node *CastIINode::Ideal(PhaseGVN *phase, bool can_reshape) { // Do not narrow the type of range check dependent CastIINodes to // avoid corruption of the graph if a CastII is replaced by TOP but // the corresponding range check is not removed. - if (can_reshape && !_range_check_dependency && !phase->C->major_progress()) { - const TypeInt* this_type = this->type()->is_int(); - const TypeInt* in_type = phase->type(in(1))->isa_int(); - if (in_type != NULL && this_type != NULL && - (in_type->_lo != this_type->_lo || - in_type->_hi != this_type->_hi)) { - jint lo1 = this_type->_lo; - jint hi1 = this_type->_hi; - int w1 = this_type->_widen; - - if (lo1 >= 0) { - // Keep a range assertion of >=0. - lo1 = 0; hi1 = max_jint; - } else if (hi1 < 0) { - // Keep a range assertion of <0. - lo1 = min_jint; hi1 = -1; - } else { - lo1 = min_jint; hi1 = max_jint; - } - const TypeInt* wtype = TypeInt::make(MAX2(in_type->_lo, lo1), - MIN2(in_type->_hi, hi1), - MAX2((int)in_type->_widen, w1)); - if (wtype != type()) { - set_type(wtype); - return this; + if (can_reshape && !_range_check_dependency) { + if (phase->C->post_loop_opts_phase()) { + const TypeInt* this_type = this->type()->is_int(); + const TypeInt* in_type = phase->type(in(1))->isa_int(); + if (in_type != NULL && this_type != NULL && + (in_type->_lo != this_type->_lo || + in_type->_hi != this_type->_hi)) { + jint lo1 = this_type->_lo; + jint hi1 = this_type->_hi; + int w1 = this_type->_widen; + + if (lo1 >= 0) { + // Keep a range assertion of >=0. + lo1 = 0; hi1 = max_jint; + } else if (hi1 < 0) { + // Keep a range assertion of <0. + lo1 = min_jint; hi1 = -1; + } else { + lo1 = min_jint; hi1 = max_jint; + } + const TypeInt* wtype = TypeInt::make(MAX2(in_type->_lo, lo1), + MIN2(in_type->_hi, hi1), + MAX2((int)in_type->_widen, w1)); + if (wtype != type()) { + set_type(wtype); + return this; + } } + } else { + phase->C->record_for_post_loop_opts_igvn(this); } } return NULL; } +Node* CastIINode::Identity(PhaseGVN* phase) { + Node* progress = ConstraintCastNode::Identity(phase); + if (progress != this) { + return progress; + } + if (_range_check_dependency) { + if (phase->C->post_loop_opts_phase()) { + return this->in(1); + } else { + phase->C->record_for_post_loop_opts_igvn(this); + } + } + return this; +} + uint CastIINode::cmp(const Node &n) const { return ConstraintCastNode::cmp(n) && ((CastIINode&)n)._range_check_dependency == _range_check_dependency; } diff --git a/src/hotspot/share/opto/castnode.hpp b/src/hotspot/share/opto/castnode.hpp index eaf3e32957e..8add9cbc074 100644 --- a/src/hotspot/share/opto/castnode.hpp +++ b/src/hotspot/share/opto/castnode.hpp @@ -75,6 +75,7 @@ class CastIINode: public ConstraintCastNode { } virtual int Opcode() const; virtual uint ideal_reg() const { return Op_RegI; } + virtual Node* Identity(PhaseGVN* phase); virtual const Type* Value(PhaseGVN* phase) const; virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); const bool has_range_check() { diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index b7afde9a319..8207c714a3c 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -367,6 +367,15 @@ void Compile::remove_useless_late_inlines(GrowableArray* inlines inlines->trunc_to(inlines->length()-shift); } +void Compile::remove_useless_nodes(GrowableArray& node_list, Unique_Node_List& useful) { + for (int i = node_list.length() - 1; i >= 0; i--) { + Node* node = node_list.at(i); + if (!useful.member(node)) { + node_list.delete_at(i); // replaces i-th with last element which is known to be useful (already processed) + } + } +} + // Disconnect all useless nodes by disconnecting those at the boundary. void Compile::remove_useless_nodes(Unique_Node_List &useful) { uint next = 0; @@ -401,13 +410,6 @@ void Compile::remove_useless_nodes(Unique_Node_List &useful) { remove_macro_node(n); } } - // Remove useless CastII nodes with range check dependency - for (int i = range_check_cast_count() - 1; i >= 0; i--) { - Node* cast = range_check_cast_node(i); - if (!useful.member(cast)) { - remove_range_check_cast(cast); - } - } // Remove useless expensive nodes for (int i = C->expensive_count()-1; i >= 0; i--) { Node* n = C->expensive_node(i); @@ -415,13 +417,7 @@ void Compile::remove_useless_nodes(Unique_Node_List &useful) { remove_expensive_node(n); } } - // Remove useless Opaque4 nodes - for (int i = opaque4_count() - 1; i >= 0; i--) { - Node* opaq = opaque4_node(i); - if (!useful.member(opaq)) { - remove_opaque4_node(opaq); - } - } + remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass remove_useless_coarsened_locks(useful); // remove useless coarsened locks nodes BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); bs->eliminate_useless_gc_barriers(useful); @@ -677,6 +673,7 @@ Compile::Compile( ciEnv* ci_env, C2Compiler* compiler, ciMethod* target, int osr _inner_loops(0), _scratch_const_size(-1), _in_scratch_emit_size(false), + _for_post_loop_igvn(comp_arena(), 8, 0, NULL), _coarsened_locks (comp_arena(), 8, 0, NULL), _dead_node_list(comp_arena()), _dead_node_count(0), @@ -705,6 +702,7 @@ Compile::Compile( ciEnv* ci_env, C2Compiler* compiler, ciMethod* target, int osr _print_inlining_output(NULL), _interpreter_frame_size(0), _max_node_limit(MaxNodeLimit), + _post_loop_opts_phase(false), _has_reserved_stack_access(target->has_reserved_stack_access()) { C = this; CompileWrapper cw(this); @@ -1007,6 +1005,7 @@ Compile::Compile( ciEnv* ci_env, _Compile_types(mtCompiler), _dead_node_list(comp_arena()), _dead_node_count(0), + _for_post_loop_igvn(comp_arena(), 8, 0, NULL), _congraph(NULL), _replay_inline_data(NULL), _number_of_mh_late_inlines(0), @@ -1019,6 +1018,7 @@ Compile::Compile( ciEnv* ci_env, _allowed_reasons(0), _interpreter_frame_size(0), _max_node_limit(MaxNodeLimit), + _post_loop_opts_phase(false), _has_reserved_stack_access(false) { C = this; @@ -1203,8 +1203,6 @@ void Compile::Init(int aliaslevel) { _macro_nodes = new(comp_arena()) GrowableArray(comp_arena(), 8, 0, NULL); _predicate_opaqs = new(comp_arena()) GrowableArray(comp_arena(), 8, 0, NULL); _expensive_nodes = new(comp_arena()) GrowableArray(comp_arena(), 8, 0, NULL); - _range_check_casts = new(comp_arena()) GrowableArray(comp_arena(), 8, 0, NULL); - _opaque4_nodes = new(comp_arena()) GrowableArray(comp_arena(), 8, 0, NULL); register_library_intrinsics(); #ifdef ASSERT _type_verify_symmetry = true; @@ -1977,46 +1975,35 @@ void Compile::cleanup_loop_predicates(PhaseIterGVN &igvn) { assert(predicate_count()==0, "should be clean!"); } -void Compile::add_range_check_cast(Node* n) { - assert(n->isa_CastII()->has_range_check(), "CastII should have range check dependency"); - assert(!_range_check_casts->contains(n), "duplicate entry in range check casts"); - _range_check_casts->append(n); -} - -// Remove all range check dependent CastIINodes. -void Compile::remove_range_check_casts(PhaseIterGVN &igvn) { - for (int i = range_check_cast_count(); i > 0; i--) { - Node* cast = range_check_cast_node(i-1); - assert(cast->isa_CastII()->has_range_check(), "CastII should have range check dependency"); - igvn.replace_node(cast, cast->in(1)); +void Compile::record_for_post_loop_opts_igvn(Node* n) { + if (!n->for_post_loop_opts_igvn()) { + assert(!_for_post_loop_igvn.contains(n), "duplicate"); + n->add_flag(Node::Flag_for_post_loop_opts_igvn); + _for_post_loop_igvn.append(n); } - assert(range_check_cast_count() == 0, "should be empty"); } -void Compile::add_opaque4_node(Node* n) { - assert(n->Opcode() == Op_Opaque4, "Opaque4 only"); - assert(!_opaque4_nodes->contains(n), "duplicate entry in Opaque4 list"); - _opaque4_nodes->append(n); +void Compile::remove_from_post_loop_opts_igvn(Node* n) { + n->remove_flag(Node::Flag_for_post_loop_opts_igvn); + _for_post_loop_igvn.remove(n); } -// Remove all Opaque4 nodes. -void Compile::remove_opaque4_nodes(PhaseIterGVN &igvn) { - for (int i = opaque4_count(); i > 0; i--) { - Node* opaq = opaque4_node(i-1); - assert(opaq->Opcode() == Op_Opaque4, "Opaque4 only"); - // With Opaque4 nodes, the expectation is that the test of input 1 - // is always equal to the constant value of input 2. So we can - // remove the Opaque4 and replace it by input 2. In debug builds, - // leave the non constant test in instead to sanity check that it - // never fails (if it does, that subgraph was constructed so, at - // runtime, a Halt node is executed). -#ifdef ASSERT - igvn.replace_node(opaq, opaq->in(1)); -#else - igvn.replace_node(opaq, opaq->in(2)); -#endif +void Compile::process_for_post_loop_opts_igvn(PhaseIterGVN& igvn) { + // Verify that all previous optimizations produced a valid graph + // at least to this point, even if no loop optimizations were done. + PhaseIdealLoop::verify(igvn); + + C->set_post_loop_opts_phase(); // no more loop opts allowed + + if (_for_post_loop_igvn.length() > 0) { + while (_for_post_loop_igvn.length() > 0) { + Node* n = _for_post_loop_igvn.pop(); + n->remove_flag(Node::Flag_for_post_loop_opts_igvn); + igvn._worklist.push(n); + } + igvn.optimize(); + assert(_for_post_loop_igvn.length() == 0, "no more delayed nodes allowed"); } - assert(opaque4_count() == 0, "should be empty"); } // StringOpts and late inlining of string methods @@ -2387,7 +2374,6 @@ void Compile::Optimize() { } if (!failing()) { // Verify that last round of loop opts produced a valid graph - TracePhase tp("idealLoopVerify", &timers[_t_idealLoopVerify]); PhaseIdealLoop::verify(igvn); } } @@ -2423,21 +2409,9 @@ void Compile::Optimize() { if (failing()) return; - // Ensure that major progress is now clear - C->clear_major_progress(); - - { - // Verify that all previous optimizations produced a valid graph - // at least to this point, even if no loop optimizations were done. - TracePhase tp("idealLoopVerify", &timers[_t_idealLoopVerify]); - PhaseIdealLoop::verify(igvn); - } + C->clear_major_progress(); // ensure that major progress is now clear - if (range_check_cast_count() > 0) { - // No more loop optimizations. Remove all range check dependent CastIINodes. - C->remove_range_check_casts(igvn); - igvn.optimize(); - } + process_for_post_loop_opts_igvn(igvn); #ifdef ASSERT BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); @@ -2464,11 +2438,6 @@ void Compile::Optimize() { } #endif - if (opaque4_count() > 0) { - C->remove_opaque4_nodes(igvn); - igvn.optimize(); - } - if (C->max_vector_size() > 0) { C->optimize_logic_cones(igvn); igvn.optimize(); @@ -4768,8 +4737,6 @@ Node* Compile::constrained_convI2L(PhaseGVN* phase, Node* value, const TypeInt* // ConvI2L node may be eliminated independently of the range check, causing the data path // to become TOP while the control path is still there (although it's unreachable). value->set_req(0, ctrl); - // Save CastII node to remove it after loop optimizations. - phase->C->add_range_check_cast(value); value = phase->transform(value); } const TypeLong* ltype = TypeLong::make(itype->_lo, itype->_hi, itype->_widen); diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index efa10d29fcc..fe706ce70ec 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -383,6 +383,9 @@ class Compile : public Phase { int _fixed_slots; // count of frame slots not allocated by the register // allocator i.e. locks, original deopt pc, etc. uintx _max_node_limit; // Max unique node count during a single compilation. + + bool _post_loop_opts_phase; // Loop opts are finished. + // For deopt int _orig_pc_slot; int _orig_pc_slot_offset_in_bytes; @@ -433,8 +436,7 @@ class Compile : public Phase { GrowableArray* _macro_nodes; // List of nodes which need to be expanded before matching. GrowableArray* _predicate_opaqs; // List of Opaque1 nodes for the loop predicates. GrowableArray* _expensive_nodes; // List of nodes that are expensive to compute and that we'd better not let the GVN freely common - GrowableArray* _range_check_casts; // List of CastII nodes with a range check dependency - GrowableArray* _opaque4_nodes; // List of Opaque4 nodes that have a default value + GrowableArray _for_post_loop_igvn; // List of nodes for IGVN after loop opts are over GrowableArray _coarsened_locks; // List of coarsened Lock and Unlock nodes ConnectionGraph* _congraph; #ifndef PRODUCT @@ -855,30 +857,17 @@ class Compile : public Phase { _predicate_opaqs->append(n); } - // Range check dependent CastII nodes that can be removed after loop optimizations - void add_range_check_cast(Node* n); - void remove_range_check_cast(Node* n) { - if (_range_check_casts->contains(n)) { - _range_check_casts->remove(n); - } - } - Node* range_check_cast_node(int idx) const { return _range_check_casts->at(idx); } - int range_check_cast_count() const { return _range_check_casts->length(); } - // Remove all range check dependent CastIINodes. - void remove_range_check_casts(PhaseIterGVN &igvn); void add_coarsened_locks(GrowableArray& locks); void remove_coarsened_lock(Node* n); bool coarsened_locks_consistent(); - void add_opaque4_node(Node* n); - void remove_opaque4_node(Node* n) { - if (_opaque4_nodes->contains(n)) { - _opaque4_nodes->remove(n); - } - } - Node* opaque4_node(int idx) const { return _opaque4_nodes->at(idx); } - int opaque4_count() const { return _opaque4_nodes->length(); } - void remove_opaque4_nodes(PhaseIterGVN &igvn); + bool post_loop_opts_phase() { return _post_loop_opts_phase; } + void set_post_loop_opts_phase() { _post_loop_opts_phase = true; } + void reset_post_loop_opts_phase() { _post_loop_opts_phase = false; } + + void record_for_post_loop_opts_igvn(Node* n); + void remove_from_post_loop_opts_igvn(Node* n); + void process_for_post_loop_opts_igvn(PhaseIterGVN& igvn); // remove the opaque nodes that protect the predicates so that the unused checks and // uncommon traps will be eliminated from the graph. @@ -1128,6 +1117,8 @@ class Compile : public Phase { _vector_reboxing_late_inlines.push(cg); } + void remove_useless_nodes (GrowableArray& node_list, Unique_Node_List &useful); + void remove_useless_late_inlines(GrowableArray* inlines, Unique_Node_List &useful); void remove_useless_coarsened_locks(Unique_Node_List& useful); diff --git a/src/hotspot/share/opto/convertnode.cpp b/src/hotspot/share/opto/convertnode.cpp index 835b101be88..24530e52c8e 100644 --- a/src/hotspot/share/opto/convertnode.cpp +++ b/src/hotspot/share/opto/convertnode.cpp @@ -287,49 +287,49 @@ Node *ConvI2LNode::Ideal(PhaseGVN *phase, bool can_reshape) { const TypeLong* this_type = this->type()->is_long(); Node* this_changed = NULL; - // If _major_progress, then more loop optimizations follow. Do NOT - // remove this node's type assertion until no more loop ops can happen. - // The progress bit is set in the major loop optimizations THEN comes the - // call to IterGVN and any chance of hitting this code. Cf. Opaque1Node. - if (can_reshape && !phase->C->major_progress()) { - const TypeInt* in_type = phase->type(in(1))->isa_int(); - if (in_type != NULL && this_type != NULL && - (in_type->_lo != this_type->_lo || - in_type->_hi != this_type->_hi)) { - // Although this WORSENS the type, it increases GVN opportunities, - // because I2L nodes with the same input will common up, regardless - // of slightly differing type assertions. Such slight differences - // arise routinely as a result of loop unrolling, so this is a - // post-unrolling graph cleanup. Choose a type which depends only - // on my input. (Exception: Keep a range assertion of >=0 or <0.) - jlong lo1 = this_type->_lo; - jlong hi1 = this_type->_hi; - int w1 = this_type->_widen; - if (lo1 != (jint)lo1 || - hi1 != (jint)hi1 || - lo1 > hi1) { - // Overflow leads to wraparound, wraparound leads to range saturation. - lo1 = min_jint; hi1 = max_jint; - } else if (lo1 >= 0) { - // Keep a range assertion of >=0. - lo1 = 0; hi1 = max_jint; - } else if (hi1 < 0) { - // Keep a range assertion of <0. - lo1 = min_jint; hi1 = -1; - } else { - lo1 = min_jint; hi1 = max_jint; - } - const TypeLong* wtype = TypeLong::make(MAX2((jlong)in_type->_lo, lo1), - MIN2((jlong)in_type->_hi, hi1), - MAX2((int)in_type->_widen, w1)); - if (wtype != type()) { - set_type(wtype); - // Note: this_type still has old type value, for the logic below. - this_changed = this; - } + if (can_reshape) { + // Do NOT remove this node's type assertion until no more loop ops can happen. + if (phase->C->post_loop_opts_phase()) { + const TypeInt* in_type = phase->type(in(1))->isa_int(); + if (in_type != NULL && this_type != NULL && + (in_type->_lo != this_type->_lo || + in_type->_hi != this_type->_hi)) { + // Although this WORSENS the type, it increases GVN opportunities, + // because I2L nodes with the same input will common up, regardless + // of slightly differing type assertions. Such slight differences + // arise routinely as a result of loop unrolling, so this is a + // post-unrolling graph cleanup. Choose a type which depends only + // on my input. (Exception: Keep a range assertion of >=0 or <0.) + jlong lo1 = this_type->_lo; + jlong hi1 = this_type->_hi; + int w1 = this_type->_widen; + if (lo1 != (jint)lo1 || + hi1 != (jint)hi1 || + lo1 > hi1) { + // Overflow leads to wraparound, wraparound leads to range saturation. + lo1 = min_jint; hi1 = max_jint; + } else if (lo1 >= 0) { + // Keep a range assertion of >=0. + lo1 = 0; hi1 = max_jint; + } else if (hi1 < 0) { + // Keep a range assertion of <0. + lo1 = min_jint; hi1 = -1; + } else { + lo1 = min_jint; hi1 = max_jint; } + const TypeLong* wtype = TypeLong::make(MAX2((jlong)in_type->_lo, lo1), + MIN2((jlong)in_type->_hi, hi1), + MAX2((int)in_type->_widen, w1)); + if (wtype != type()) { + set_type(wtype); + // Note: this_type still has old type value, for the logic below. + this_changed = this; + } + } + } else { + phase->C->record_for_post_loop_opts_igvn(this); + } } - #ifdef _LP64 // Convert ConvI2L(AddI(x, y)) to AddL(ConvI2L(x), ConvI2L(y)) // but only if x and y have subranges that cannot cause 32-bit overflow, diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 67711d287f9..2d2a33d49b2 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -3061,6 +3061,8 @@ bool PhaseIdealLoop::only_has_infinite_loops() { // Create a PhaseLoop. Build the ideal Loop tree. Map each Ideal Node to // its corresponding LoopNode. If 'optimize' is true, do some loop cleanups. void PhaseIdealLoop::build_and_optimize() { + assert(!C->post_loop_opts_phase(), "no loop opts allowed"); + bool do_split_ifs = (_mode == LoopOptsDefault || _mode == LoopOptsLastRound); bool skip_loop_opts = (_mode == LoopOptsNone); #if INCLUDE_SHENANDOAHGC diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index 4238edb87c3..3939264c0cb 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -1024,6 +1024,8 @@ class PhaseIdealLoop : public PhaseTransform { // is useful to verify that all inputs properly dominate their uses. static void verify(PhaseIterGVN& igvn) { #ifdef ASSERT + ResourceMark rm; + Compile::TracePhase tp("idealLoopVerify", &timers[_t_idealLoopVerify]); PhaseIdealLoop v(igvn); #endif } diff --git a/src/hotspot/share/opto/node.cpp b/src/hotspot/share/opto/node.cpp index 02bb6bb16de..e69ec048748 100644 --- a/src/hotspot/share/opto/node.cpp +++ b/src/hotspot/share/opto/node.cpp @@ -499,24 +499,23 @@ Node *Node::clone() const { n->_in[i] = x; if (x != NULL) x->add_out(n); } - if (is_macro()) + if (is_macro()) { C->add_macro_node(n); - if (is_expensive()) + } + if (is_expensive()) { C->add_expensive_node(n); + } if (n->is_reduction()) { // Do not copy reduction information. This must be explicitly set by the calling code. n->remove_flag(Node::Flag_is_reduction); } + if (for_post_loop_opts_igvn()) { + // Don't add cloned node to Compile::_for_post_loop_opts_igvn list automatically. + // If it is applicable, it will happen anyway when the cloned node is registered with IGVN. + n->remove_flag(Node::Flag_for_post_loop_opts_igvn); + } BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); bs->register_potential_barrier_node(n); - // If the cloned node is a range check dependent CastII, add it to the list. - CastIINode* cast = n->isa_CastII(); - if (cast != NULL && cast->has_range_check()) { - C->add_range_check_cast(cast); - } - if (n->Opcode() == Op_Opaque4) { - C->add_opaque4_node(n); - } n->set_idx(C->next_unique()); // Get new unique index as well debug_only( n->verify_construction() ); @@ -624,14 +623,9 @@ void Node::destruct() { if (is_expensive()) { compile->remove_expensive_node(this); } - CastIINode* cast = isa_CastII(); - if (cast != NULL && cast->has_range_check()) { - compile->remove_range_check_cast(cast); - } - if (Opcode() == Op_Opaque4) { - compile->remove_opaque4_node(this); + if (for_post_loop_opts_igvn()) { + compile->remove_from_post_loop_opts_igvn(this); } - if (is_SafePoint()) { as_SafePoint()->delete_replaced_nodes(); } @@ -1035,7 +1029,7 @@ bool Node::verify_jvms(const JVMState* using_jvms) const { //------------------------------init_NodeProperty------------------------------ void Node::init_NodeProperty() { assert(_max_classes <= max_jushort, "too many NodeProperty classes"); - assert(_max_flags <= max_jushort, "too many NodeProperty flags"); + assert(_max_flags <= max_juint, "too many NodeProperty flags"); } #endif @@ -1389,12 +1383,8 @@ static void kill_dead_code( Node *dead, PhaseIterGVN *igvn ) { if (dead->is_expensive()) { igvn->C->remove_expensive_node(dead); } - CastIINode* cast = dead->isa_CastII(); - if (cast != NULL && cast->has_range_check()) { - igvn->C->remove_range_check_cast(cast); - } - if (dead->Opcode() == Op_Opaque4) { - igvn->C->remove_opaque4_node(dead); + if (dead->for_post_loop_opts_igvn()) { + igvn->C->remove_from_post_loop_opts_igvn(dead); } BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); bs->unregister_potential_barrier_node(dead); diff --git a/src/hotspot/share/opto/node.hpp b/src/hotspot/share/opto/node.hpp index 63b707bca3c..1a8b0d0296f 100644 --- a/src/hotspot/share/opto/node.hpp +++ b/src/hotspot/share/opto/node.hpp @@ -740,38 +740,40 @@ class Node { // Flags are sorted by usage frequency. enum NodeFlags { - Flag_is_Copy = 0x01, // should be first bit to avoid shift - Flag_rematerialize = Flag_is_Copy << 1, - Flag_needs_anti_dependence_check = Flag_rematerialize << 1, - Flag_is_macro = Flag_needs_anti_dependence_check << 1, - Flag_is_Con = Flag_is_macro << 1, - Flag_is_cisc_alternate = Flag_is_Con << 1, - Flag_is_dead_loop_safe = Flag_is_cisc_alternate << 1, - Flag_may_be_short_branch = Flag_is_dead_loop_safe << 1, - Flag_avoid_back_to_back_before = Flag_may_be_short_branch << 1, - Flag_avoid_back_to_back_after = Flag_avoid_back_to_back_before << 1, - Flag_has_call = Flag_avoid_back_to_back_after << 1, - Flag_is_reduction = Flag_has_call << 1, - Flag_is_scheduled = Flag_is_reduction << 1, - Flag_has_vector_mask_set = Flag_is_scheduled << 1, - Flag_is_expensive = Flag_has_vector_mask_set << 1, - _max_flags = (Flag_is_expensive << 1) - 1 // allow flags combination + Flag_is_Copy = 1 << 0, // should be first bit to avoid shift + Flag_rematerialize = 1 << 1, + Flag_needs_anti_dependence_check = 1 << 2, + Flag_is_macro = 1 << 3, + Flag_is_Con = 1 << 4, + Flag_is_cisc_alternate = 1 << 5, + Flag_is_dead_loop_safe = 1 << 6, + Flag_may_be_short_branch = 1 << 7, + Flag_avoid_back_to_back_before = 1 << 8, + Flag_avoid_back_to_back_after = 1 << 9, + Flag_has_call = 1 << 10, + Flag_is_reduction = 1 << 11, + Flag_is_scheduled = 1 << 12, + Flag_has_vector_mask_set = 1 << 13, + Flag_is_expensive = 1 << 14, + Flag_for_post_loop_opts_igvn = 1 << 15, + _last_flag = Flag_for_post_loop_opts_igvn, + _max_flags = (Flag_for_post_loop_opts_igvn << 1) - 1 // allow flags combination }; private: juint _class_id; - jushort _flags; + juint _flags; protected: // These methods should be called from constructors only. void init_class_id(juint c) { _class_id = c; // cast out const } - void init_flags(jushort fl) { + void init_flags(uint fl) { assert(fl <= _max_flags, "invalid node flag"); _flags |= fl; } - void clear_flag(jushort fl) { + void clear_flag(uint fl) { assert(fl <= _max_flags, "invalid node flag"); _flags &= ~fl; } @@ -779,11 +781,11 @@ class Node { public: const juint class_id() const { return _class_id; } - const jushort flags() const { return _flags; } + const juint flags() const { return _flags; } - void add_flag(jushort fl) { init_flags(fl); } + void add_flag(juint fl) { init_flags(fl); } - void remove_flag(jushort fl) { clear_flag(fl); } + void remove_flag(juint fl) { clear_flag(fl); } // Return a dense integer opcode number virtual int Opcode() const; @@ -961,6 +963,8 @@ class Node { // Used in lcm to mark nodes that have scheduled bool is_scheduled() const { return (_flags & Flag_is_scheduled) != 0; } + bool for_post_loop_opts_igvn() const { return (_flags & Flag_for_post_loop_opts_igvn) != 0; } + //----------------- Optimization // Get the worst-case Type output for this Node. diff --git a/src/hotspot/share/opto/opaquenode.cpp b/src/hotspot/share/opto/opaquenode.cpp index cccd5b581af..d11f1229d18 100644 --- a/src/hotspot/share/opto/opaquenode.cpp +++ b/src/hotspot/share/opto/opaquenode.cpp @@ -34,14 +34,14 @@ uint Opaque1Node::cmp( const Node &n ) const { } //------------------------------Identity--------------------------------------- -// If _major_progress, then more loop optimizations follow. Do NOT remove -// the opaque Node until no more loop ops can happen. Note the timing of -// _major_progress; it's set in the major loop optimizations THEN comes the -// call to IterGVN and any chance of hitting this code. Hence there's no -// phase-ordering problem with stripping Opaque1 in IGVN followed by some -// more loop optimizations that require it. +// Do NOT remove the opaque Node until no more loop ops can happen. Node* Opaque1Node::Identity(PhaseGVN* phase) { - return phase->C->major_progress() ? this : in(1); + if (phase->C->post_loop_opts_phase()) { + return in(1); + } else { + phase->C->record_for_post_loop_opts_igvn(this); + } + return this; } //============================================================================= @@ -60,6 +60,25 @@ uint Opaque2Node::cmp( const Node &n ) const { return (&n == this); // Always fail except on self } +Node* Opaque4Node::Identity(PhaseGVN* phase) { + if (phase->C->post_loop_opts_phase()) { + // With Opaque4 nodes, the expectation is that the test of input 1 + // is always equal to the constant value of input 2. So we can + // remove the Opaque4 and replace it by input 2. In debug builds, + // leave the non constant test in instead to sanity check that it + // never fails (if it does, that subgraph was constructed so, at + // runtime, a Halt node is executed). +#ifdef ASSERT + return this->in(1); +#else + return this->in(2); +#endif + } else { + phase->C->record_for_post_loop_opts_igvn(this); + } + return this; +} + const Type* Opaque4Node::Value(PhaseGVN* phase) const { return phase->type(in(1)); } diff --git a/src/hotspot/share/opto/opaquenode.hpp b/src/hotspot/share/opto/opaquenode.hpp index 023c527915f..7420c19fe17 100644 --- a/src/hotspot/share/opto/opaquenode.hpp +++ b/src/hotspot/share/opto/opaquenode.hpp @@ -114,12 +114,11 @@ class Opaque3Node : public Opaque2Node { // GraphKit::must_be_not_null(). class Opaque4Node : public Node { public: - Opaque4Node(Compile* C, Node *tst, Node* final_tst) : Node(NULL, tst, final_tst) { - // Put it on the Opaque4 nodes list to be removed after all optimizations - C->add_opaque4_node(this); - } + Opaque4Node(Compile* C, Node *tst, Node* final_tst) : Node(NULL, tst, final_tst) {} + virtual int Opcode() const; virtual const Type *bottom_type() const { return TypeInt::BOOL; } + virtual Node* Identity(PhaseGVN* phase); virtual const Type* Value(PhaseGVN* phase) const; }; diff --git a/src/hotspot/share/opto/phaseX.cpp b/src/hotspot/share/opto/phaseX.cpp index 5aec3b953b6..9cf53dc10e0 100644 --- a/src/hotspot/share/opto/phaseX.cpp +++ b/src/hotspot/share/opto/phaseX.cpp @@ -1515,12 +1515,8 @@ void PhaseIterGVN::remove_globally_dead_node( Node *dead ) { if (dead->is_expensive()) { C->remove_expensive_node(dead); } - CastIINode* cast = dead->isa_CastII(); - if (cast != NULL && cast->has_range_check()) { - C->remove_range_check_cast(cast); - } - if (dead->Opcode() == Op_Opaque4) { - C->remove_opaque4_node(dead); + if (dead->for_post_loop_opts_igvn()) { + C->remove_from_post_loop_opts_igvn(dead); } BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); bs->unregister_potential_barrier_node(dead); diff --git a/src/hotspot/share/runtime/vmStructs.cpp b/src/hotspot/share/runtime/vmStructs.cpp index 463dad9fdb2..5552bad6a62 100644 --- a/src/hotspot/share/runtime/vmStructs.cpp +++ b/src/hotspot/share/runtime/vmStructs.cpp @@ -957,7 +957,7 @@ typedef PaddedEnd PaddedObjectMonitor; c2_nonstatic_field(Node, _outmax, node_idx_t) \ c2_nonstatic_field(Node, _idx, const node_idx_t) \ c2_nonstatic_field(Node, _class_id, juint) \ - c2_nonstatic_field(Node, _flags, jushort) \ + c2_nonstatic_field(Node, _flags, juint) \ \ c2_nonstatic_field(Compile, _root, RootNode*) \ c2_nonstatic_field(Compile, _unique, uint) \ diff --git a/test/hotspot/jtreg/gc/shenandoah/compiler/TestBarrierExpansionDeadMemPhi.java b/test/hotspot/jtreg/gc/shenandoah/compiler/TestBarrierExpansionDeadMemPhi.java new file mode 100644 index 00000000000..80d73392a7b --- /dev/null +++ b/test/hotspot/jtreg/gc/shenandoah/compiler/TestBarrierExpansionDeadMemPhi.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2020, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/** + * @test + * @bug 8255400 + * @summary C2 failures after JDK-8255000 + * @requires vm.gc.Shenandoah + * @modules java.base/jdk.internal.misc:+open + * + * @run main/othervm -XX:-UseOnStackReplacement -XX:-BackgroundCompilation -XX:-TieredCompilation -XX:+UseShenandoahGC TestBarrierExpansionDeadMemPhi + * + * + */ + +import jdk.internal.misc.Unsafe; +import java.util.Arrays; +import java.lang.reflect.Field; + +public class TestBarrierExpansionDeadMemPhi { + + static final jdk.internal.misc.Unsafe UNSAFE = Unsafe.getUnsafe(); + + static final long F_OFFSET; + + static class A { + int f; + } + + static { + try { + Field fField = A.class.getDeclaredField("f"); + F_OFFSET = UNSAFE.objectFieldOffset(fField); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + static int test(Object[] array) { + int f = 0; + for (int i = 0; i < 100; i++) { + f += UNSAFE.getInt(array[i], F_OFFSET); + } + return f; + } + + static public void main(String[] args) { + Object[] array = new Object[100]; + Arrays.fill(array, new A()); + + for (int i = 0; i < 20000; i++) { + test(array); + } + } +}