8268125: ZGC: Clone oop array gets wrong acopy stub

Nils Eliasson · Nils Eliasson · commit e36136fa9345 · 2021-06-15T11:33:12.000Z
Reviewed-by: vlivanov
diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp
@@ -852,3 +852,5 @@ void BarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac
 
   phase->igvn().replace_node(ac, call);
 }
+
+#undef XTOP
diff --git a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp
@@ -39,6 +39,7 @@
 #include "opto/output.hpp"
 #include "opto/regalloc.hpp"
 #include "opto/rootnode.hpp"
+#include "opto/runtime.hpp"
 #include "opto/type.hpp"
 #include "utilities/growableArray.hpp"
 #include "utilities/macros.hpp"
@@ -253,11 +254,51 @@ static const TypeFunc* clone_type() {
   return TypeFunc::make(domain, range);
 }
 
+#define XTOP LP64_ONLY(COMMA phase->top())
+
 void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
   Node* const src = ac->in(ArrayCopyNode::Src);
+
   if (ac->is_clone_array()) {
-    // Clone primitive array
-    BarrierSetC2::clone_at_expansion(phase, ac);
+    const TypeAryPtr* ary_ptr = src->get_ptr_type()->isa_aryptr();
+    BasicType bt;
+    if (ary_ptr == NULL) {
+      // ary_ptr can be null iff we are running with StressReflectiveCode
+      // This code will be unreachable
+      assert(StressReflectiveCode, "Guard against surprises");
+      bt = T_LONG;
+    } else {
+      bt = ary_ptr->elem()->array_element_basic_type();
+      if (is_reference_type(bt)) {
+        // Clone object array
+        bt = T_OBJECT;
+      } else {
+        // Clone primitive array
+        bt = T_LONG;
+      }
+    }
+
+    Node* ctrl = ac->in(TypeFunc::Control);
+    Node* mem = ac->in(TypeFunc::Memory);
+    Node* src = ac->in(ArrayCopyNode::Src);
+    Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
+    Node* dest = ac->in(ArrayCopyNode::Dest);
+    Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
+    Node* length = ac->in(ArrayCopyNode::Length);
+
+    Node* payload_src = phase->basic_plus_adr(src, src_offset);
+    Node* payload_dst = phase->basic_plus_adr(dest, dest_offset);
+
+    const char* copyfunc_name = "arraycopy";
+    address     copyfunc_addr = phase->basictype2arraycopy(bt, NULL, NULL, true, copyfunc_name, true);
+
+    const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
+    const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type();
+
+    Node* call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
+    phase->transform_later(call);
+
+    phase->igvn().replace_node(ac, call);
     return;
   }
 
@@ -289,6 +330,8 @@ void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* a
   phase->igvn().replace_node(ac, call);
 }
 
+#undef XTOP
+
 // == Dominating barrier elision ==
 
 static bool block_has_safepoint(const Block* block, uint from, uint to) {
diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp
@@ -4117,6 +4117,8 @@ bool LibraryCallKit::inline_unsafe_copyMemory() {
   return true;
 }
 
+#undef XTOP
+
 //------------------------clone_coping-----------------------------------
 // Helper function for inline_native_clone.
 void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array) {
diff --git a/src/hotspot/share/opto/macroArrayCopy.cpp b/src/hotspot/share/opto/macroArrayCopy.cpp
@@ -1224,6 +1224,8 @@ bool PhaseMacroExpand::generate_unchecked_arraycopy(Node** ctrl, MergeMemNode**
   return false;
 }
 
+#undef XTOP
+
 void PhaseMacroExpand::expand_arraycopy_node(ArrayCopyNode *ac) {
   Node* ctrl = ac->in(TypeFunc::Control);
   Node* io = ac->in(TypeFunc::I_O);
diff --git a/test/hotspot/jtreg/compiler/arraycopy/TestObjectArrayClone.java b/test/hotspot/jtreg/compiler/arraycopy/TestObjectArrayClone.java

Original file line number	Diff line number	Diff line change
`@@ -852,3 +852,5 @@ void BarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac`
`852`	`852`
`853`	`853`	`phase->igvn().replace_node(ac, call);`
`854`	`854`	`}`
	`855`	`+`
	`856`	`+#undef XTOP`
Original file line number	Diff line number	Diff line change
`@@ -4117,6 +4117,8 @@ bool LibraryCallKit::inline_unsafe_copyMemory() {`
`4117`	`4117`	`return true;`
`4118`	`4118`	`}`
`4119`	`4119`
	`4120`	`+#undef XTOP`
	`4121`	`+`
`4120`	`4122`	`//------------------------clone_coping-----------------------------------`
`4121`	`4123`	`// Helper function for inline_native_clone.`
`4122`	`4124`	`void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array) {`
Original file line number	Diff line number	Diff line change
`@@ -1224,6 +1224,8 @@ bool PhaseMacroExpand::generate_unchecked_arraycopy(Node ctrl, MergeMemNode`
`1224`	`1224`	`return false;`
`1225`	`1225`	`}`
`1226`	`1226`
	`1227`	`+#undef XTOP`
	`1228`	`+`
`1227`	`1229`	`void PhaseMacroExpand::expand_arraycopy_node(ArrayCopyNode *ac) {`
`1228`	`1230`	`Node* ctrl = ac->in(TypeFunc::Control);`
`1229`	`1231`	`Node* io = ac->in(TypeFunc::I_O);`