Merge branch 'master' into poly_inline

src/hotspot/share/compiler/compilerDirectives.hpp

@@ -61,6 +61,7 @@
     cflags(PrintIntrinsics,         bool, PrintIntrinsics, PrintIntrinsics) \
 NOT_PRODUCT(cflags(TraceOptoPipelining, bool, TraceOptoPipelining, TraceOptoPipelining)) \
 NOT_PRODUCT(cflags(TraceOptoOutput,     bool, TraceOptoOutput, TraceOptoOutput)) \
+NOT_PRODUCT(cflags(TraceMergeStores, bool, TraceMergeStores, TraceMergeStores)) \
     cflags(TraceSpilling,           bool, TraceSpilling, TraceSpilling) \
     cflags(Vectorize,               bool, false, Vectorize) \
     cflags(VectorizeDebug,          uintx, 0, VectorizeDebug) \

src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp

@@ -607,12 +607,15 @@ Node* G1BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) c
   Node* adr = access.addr().node();
   Node* obj = access.base();
 
+  bool anonymous = (decorators & C2_UNSAFE_ACCESS) != 0;
   bool mismatched = (decorators & C2_MISMATCHED) != 0;
   bool unknown = (decorators & ON_UNKNOWN_OOP_REF) != 0;
   bool in_heap = (decorators & IN_HEAP) != 0;
+  bool in_native = (decorators & IN_NATIVE) != 0;
   bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
   bool is_unordered = (decorators & MO_UNORDERED) != 0;
-  bool need_cpu_mem_bar = !is_unordered || mismatched || !in_heap;
+  bool is_mixed = !in_heap && !in_native;
+  bool need_cpu_mem_bar = !is_unordered || mismatched || is_mixed;
 
   Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : kit->top();
   Node* load = CardTableBarrierSetC2::load_at_resolved(access, val_type);

src/hotspot/share/gc/shared/c2/barrierSetC2.cpp

@@ -40,13 +40,15 @@ void* C2Access::barrier_set_state() const {
 }
 
 bool C2Access::needs_cpu_membar() const {
-  bool mismatched = (_decorators & C2_MISMATCHED) != 0;
+  bool mismatched   = (_decorators & C2_MISMATCHED) != 0;
   bool is_unordered = (_decorators & MO_UNORDERED) != 0;
   bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0;
-  bool in_heap = (_decorators & IN_HEAP) != 0;
+  bool in_heap   = (_decorators & IN_HEAP) != 0;
+  bool in_native = (_decorators & IN_NATIVE) != 0;
+  bool is_mixed  = !in_heap && !in_native;
 
-  bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
-  bool is_read = (_decorators & C2_READ_ACCESS) != 0;
+  bool is_write  = (_decorators & C2_WRITE_ACCESS) != 0;
+  bool is_read   = (_decorators & C2_READ_ACCESS) != 0;
   bool is_atomic = is_read && is_write;
 
   if (is_atomic) {
@@ -60,9 +62,11 @@ bool C2Access::needs_cpu_membar() const {
     // the barriers get omitted and the unsafe reference begins to "pollute"
     // the alias analysis of the rest of the graph, either Compile::can_alias
     // or Compile::must_alias will throw a diagnostic assert.)
-    if (!in_heap || !is_unordered || (mismatched && !_addr.type()->isa_aryptr())) {
+    if (is_mixed || !is_unordered || (mismatched && !_addr.type()->isa_aryptr())) {
       return true;
     }
+  } else {
+    assert(!is_mixed, "not unsafe");
   }
 
   return false;
@@ -78,7 +82,7 @@ Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) cons
   bool requires_atomic_access = (decorators & MO_UNORDERED) == 0;
 
   bool in_native = (decorators & IN_NATIVE) != 0;
-  assert(!in_native, "not supported yet");
+  assert(!in_native || (unsafe && !access.is_oop()), "not supported yet");
 
   if (access.type() == T_DOUBLE) {
     Node* new_val = kit->dstore_rounding(val.node());

src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp

@@ -45,6 +45,8 @@ bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) {
   ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
   if ((state->iu_barriers_count() +
        state->load_reference_barriers_count()) > 0) {
+    assert(C->post_loop_opts_phase(), "no loop opts allowed");
+    C->reset_post_loop_opts_phase(); // ... but we know what we are doing
     bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion;
     C->clear_major_progress();
     PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand);
@@ -58,12 +60,10 @@ bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) {
         return false;
       }
       C->clear_major_progress();
-      if (C->range_check_cast_count() > 0) {
-        // No more loop optimizations. Remove all range check dependent CastIINodes.
-        C->remove_range_check_casts(igvn);
-        igvn.optimize();
-      }
+
+      C->process_for_post_loop_opts_igvn(igvn);
     }
+    C->set_post_loop_opts_phase(); // now for real!
   }
   return true;
 }
@@ -2470,12 +2470,12 @@ void MemoryGraphFixer::collect_memory_nodes() {
                   Node* m = _memory_nodes[c->in(k)->_idx];
                   assert(m != NULL, "expect memory state");
                   if (u->in(k) != m) {
-                    phi = NULL;
+                    phi = NodeSentinel;
                   }
                 }
               }
             }
-            if (phi == NULL) {
+            if (phi == NodeSentinel) {
               phi = new PhiNode(c, Type::MEMORY, _phase->C->get_adr_type(_alias));
               for (uint k = 1; k < c->req(); k++) {
                 Node* m = _memory_nodes[c->in(k)->_idx];
@@ -2484,8 +2484,11 @@ void MemoryGraphFixer::collect_memory_nodes() {
               }
             }
           }
-          assert(phi != NULL, "");
-          regions.map(c->_idx, phi);
+          if (phi != NULL) {
+            regions.map(c->_idx, phi);
+          } else {
+            assert(c->unique_ctrl_out()->Opcode() == Op_Halt, "expected memory state");
+          }
         }
         Node* current_region = regions[c->_idx];
         if (current_region != prev_region) {
@@ -2496,7 +2499,7 @@ void MemoryGraphFixer::collect_memory_nodes() {
         }
       } else if (prev_mem == NULL || prev_mem->is_Phi() || ctrl_or_self(prev_mem) != c) {
         Node* m = _memory_nodes[_phase->idom(c)->_idx];
-        assert(m != NULL, "expect memory state");
+        assert(m != NULL || c->Opcode() == Op_Halt, "expect memory state");
         if (m != prev_mem) {
           _memory_nodes.map(c->_idx, m);
           progress = true;
@@ -2520,7 +2523,8 @@ void MemoryGraphFixer::collect_memory_nodes() {
     Node* c = rpo_list.at(i);
     if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) {
       Node* n = regions[c->_idx];
-      if (n->is_Phi() && n->_idx >= last && n->in(0) == c) {
+      assert(n != NULL || c->unique_ctrl_out()->Opcode() == Op_Halt, "expected memory state");
+      if (n != NULL && n->is_Phi() && n->_idx >= last && n->in(0) == c) {
         _phase->register_new_node(n, c);
       }
     }
@@ -2529,10 +2533,12 @@ void MemoryGraphFixer::collect_memory_nodes() {
     Node* c = rpo_list.at(i);
     if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) {
       Node* n = regions[c->_idx];
+      assert(n != NULL || c->unique_ctrl_out()->Opcode() == Op_Halt, "expected memory state");
       for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
         Node* u = c->fast_out(i);
         if (u->is_Phi() && u->bottom_type() == Type::MEMORY &&
             u != n) {
+          assert(c->unique_ctrl_out()->Opcode() != Op_Halt, "expected memory state");
           if (u->adr_type() == TypePtr::BOTTOM) {
             fix_memory_uses(u, n, n, c);
           } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) {

src/hotspot/share/oops/accessDecorators.hpp

@@ -174,11 +174,11 @@ const DecoratorSet ON_DECORATOR_MASK  = ON_STRONG_OOP_REF | ON_WEAK_OOP_REF |
                                         ON_PHANTOM_OOP_REF | ON_UNKNOWN_OOP_REF;
 
 // === Access Location ===
-// Accesses can take place in, e.g. the heap, old or young generation and different native roots.
+// Accesses can take place in, e.g. the heap, old or young generation, different native roots, or native memory off the heap.
 // The location is important to the GC as it may imply different actions. The following decorators are used:
 // * IN_HEAP: The access is performed in the heap. Many barriers such as card marking will
 //   be omitted if this decorator is not set.
-// * IN_NATIVE: The access is performed in an off-heap data structure pointing into the Java heap.
+// * IN_NATIVE: The access is performed in an off-heap data structure.
 const DecoratorSet IN_HEAP            = UCONST64(1) << 19;
 const DecoratorSet IN_NATIVE          = UCONST64(1) << 20;
 const DecoratorSet IN_DECORATOR_MASK  = IN_HEAP | IN_NATIVE;

src/hotspot/share/opto/addnode.cpp

@@ -704,9 +704,9 @@ Node* AddPNode::Ideal_base_and_offset(Node* ptr, PhaseTransform* phase,
 //------------------------------unpack_offsets----------------------------------
 // Collect the AddP offset values into the elements array, giving up
 // if there are more than length.
-int AddPNode::unpack_offsets(Node* elements[], int length) {
+int AddPNode::unpack_offsets(Node* elements[], int length) const {
   int count = 0;
-  Node* addr = this;
+  Node const* addr = this;
   Node* base = addr->in(AddPNode::Base);
   while (addr->is_AddP()) {
     if (addr->in(AddPNode::Base) != base) {

src/hotspot/share/opto/addnode.hpp

@@ -154,7 +154,7 @@ class AddPNode : public Node {
 
   // Collect the AddP offset values into the elements array, giving up
   // if there are more than length.
-  int unpack_offsets(Node* elements[], int length);
+  int unpack_offsets(Node* elements[], int length) const;
 
   // Do not match base-ptr edge
   virtual uint match_edge(uint idx) const;

src/hotspot/share/opto/c2_globals.hpp

@@ -354,6 +354,12 @@
   notproduct(bool, TraceNewVectors, false,                                  \
           "Trace creation of Vector nodes")                                 \
                                                                             \
+  diagnostic(bool, MergeStores, true,                                       \
+          "Optimize stores by combining values into larger store")          \
+                                                                            \
+  develop(bool, TraceMergeStores, false,                                    \
+          "Trace creation of merged stores")                                \
+                                                                            \
   product_pd(bool, OptoBundling,                                            \
           "Generate nops to fill i-cache lines")                            \
                                                                             \

src/hotspot/share/opto/castnode.cpp

@@ -231,37 +231,56 @@ Node *CastIINode::Ideal(PhaseGVN *phase, bool can_reshape) {
   // Do not narrow the type of range check dependent CastIINodes to
   // avoid corruption of the graph if a CastII is replaced by TOP but
   // the corresponding range check is not removed.
-  if (can_reshape && !_range_check_dependency && !phase->C->major_progress()) {
-    const TypeInt* this_type = this->type()->is_int();
-    const TypeInt* in_type = phase->type(in(1))->isa_int();
-    if (in_type != NULL && this_type != NULL &&
-        (in_type->_lo != this_type->_lo ||
-         in_type->_hi != this_type->_hi)) {
-      jint lo1 = this_type->_lo;
-      jint hi1 = this_type->_hi;
-      int w1  = this_type->_widen;
-
-      if (lo1 >= 0) {
-        // Keep a range assertion of >=0.
-        lo1 = 0;        hi1 = max_jint;
-      } else if (hi1 < 0) {
-        // Keep a range assertion of <0.
-        lo1 = min_jint; hi1 = -1;
-      } else {
-        lo1 = min_jint; hi1 = max_jint;
-      }
-      const TypeInt* wtype = TypeInt::make(MAX2(in_type->_lo, lo1),
-                                           MIN2(in_type->_hi, hi1),
-                                           MAX2((int)in_type->_widen, w1));
-      if (wtype != type()) {
-        set_type(wtype);
-        return this;
+  if (can_reshape && !_range_check_dependency) {
+    if (phase->C->post_loop_opts_phase()) {
+      const TypeInt* this_type = this->type()->is_int();
+      const TypeInt* in_type = phase->type(in(1))->isa_int();
+      if (in_type != NULL && this_type != NULL &&
+          (in_type->_lo != this_type->_lo ||
+           in_type->_hi != this_type->_hi)) {
+        jint lo1 = this_type->_lo;
+        jint hi1 = this_type->_hi;
+        int w1  = this_type->_widen;
+
+        if (lo1 >= 0) {
+          // Keep a range assertion of >=0.
+          lo1 = 0;        hi1 = max_jint;
+        } else if (hi1 < 0) {
+          // Keep a range assertion of <0.
+          lo1 = min_jint; hi1 = -1;
+        } else {
+          lo1 = min_jint; hi1 = max_jint;
+        }
+        const TypeInt* wtype = TypeInt::make(MAX2(in_type->_lo, lo1),
+                                             MIN2(in_type->_hi, hi1),
+                                             MAX2((int)in_type->_widen, w1));
+        if (wtype != type()) {
+          set_type(wtype);
+          return this;
+        }
       }
+    } else {
+      phase->C->record_for_post_loop_opts_igvn(this);
     }
   }
   return NULL;
 }
 
+Node* CastIINode::Identity(PhaseGVN* phase) {
+  Node* progress = ConstraintCastNode::Identity(phase);
+  if (progress != this) {
+    return progress;
+  }
+  if (_range_check_dependency) {
+    if (phase->C->post_loop_opts_phase()) {
+      return this->in(1);
+    } else {
+      phase->C->record_for_post_loop_opts_igvn(this);
+    }
+  }
+  return this;
+}
+
 uint CastIINode::cmp(const Node &n) const {
   return ConstraintCastNode::cmp(n) && ((CastIINode&)n)._range_check_dependency == _range_check_dependency;
 }

src/hotspot/share/opto/castnode.hpp

@@ -75,6 +75,7 @@ class CastIINode: public ConstraintCastNode {
   }
   virtual int Opcode() const;
   virtual uint ideal_reg() const { return Op_RegI; }
+  virtual Node* Identity(PhaseGVN* phase);
   virtual const Type* Value(PhaseGVN* phase) const;
   virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
   const bool has_range_check() {