Skip to content

Commit d7b5cb6

Browse files
author
Vladimir Kozlov
committed
8271368: [BACKOUT] JDK-8266054 VectorAPI rotate operation optimization
Reviewed-by: dholmes, iklam
1 parent ecd4455 commit d7b5cb6

57 files changed

Lines changed: 219 additions & 4380 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎src/hotspot/cpu/aarch64/matcher_aarch64.hpp‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -138,11 +138,6 @@
138138
return false;
139139
}
140140

141-
// Does the CPU supports vector constant rotate instructions?
142-
static constexpr bool supports_vector_constant_rotates(int shift) {
143-
return false;
144-
}
145-
146141
// Does the CPU supports vector unsigned comparison instructions?
147142
static const bool supports_vector_comparison_unsigned(int vlen, BasicType bt) {
148143
// Not supported on SVE yet.

‎src/hotspot/cpu/arm/matcher_arm.hpp‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -131,11 +131,6 @@
131131
return false; // not supported
132132
}
133133

134-
// Does the CPU supports vector constant rotate instructions?
135-
static constexpr bool supports_vector_constant_rotates(int shift) {
136-
return false;
137-
}
138-
139134
// Does the CPU supports vector unsigned comparison instructions?
140135
static constexpr bool supports_vector_comparison_unsigned(int vlen, BasicType bt) {
141136
return false;

‎src/hotspot/cpu/ppc/matcher_ppc.hpp‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -138,11 +138,6 @@
138138
return false;
139139
}
140140

141-
// Does the CPU supports vector constant rotate instructions?
142-
static constexpr bool supports_vector_constant_rotates(int shift) {
143-
return false;
144-
}
145-
146141
// Does the CPU supports vector unsigned comparison instructions?
147142
static constexpr bool supports_vector_comparison_unsigned(int vlen, BasicType bt) {
148143
return false;

‎src/hotspot/cpu/s390/matcher_s390.hpp‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -128,11 +128,6 @@
128128
return false;
129129
}
130130

131-
// Does the CPU supports vector constant rotate instructions?
132-
static constexpr bool supports_vector_constant_rotates(int shift) {
133-
return false;
134-
}
135-
136131
// Does the CPU supports vector unsigned comparison instructions?
137132
static constexpr bool supports_vector_comparison_unsigned(int vlen, BasicType bt) {
138133
return false;

‎src/hotspot/cpu/x86/matcher_x86.hpp‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -158,11 +158,6 @@
158158
return true;
159159
}
160160

161-
// Does the CPU supports vector constant rotate instructions?
162-
static constexpr bool supports_vector_constant_rotates(int shift) {
163-
return -0x80 <= shift && shift < 0x80;
164-
}
165-
166161
// Does the CPU supports vector unsigned comparison instructions?
167162
static const bool supports_vector_comparison_unsigned(int vlen, BasicType bt) {
168163
int vlen_in_bytes = vlen * type2aelembytes(bt);

‎src/hotspot/cpu/x86/x86.ad‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1638,9 +1638,6 @@ const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType
16381638
break;
16391639
case Op_RotateRightV:
16401640
case Op_RotateLeftV:
1641-
if (bt != T_INT && bt != T_LONG) {
1642-
return false;
1643-
} // fallthrough
16441641
case Op_MacroLogicV:
16451642
if (!VM_Version::supports_evex() ||
16461643
((size_in_bits != 512) && !VM_Version::supports_avx512vl())) {

‎src/hotspot/share/opto/library_call.hpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -336,7 +336,6 @@ class LibraryCallKit : public GraphKit {
336336
};
337337

338338
bool arch_supports_vector(int op, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args = false);
339-
bool arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt, bool has_scalar_args = false);
340339

341340
void clear_upper_avx() {
342341
#ifdef X86

‎src/hotspot/share/opto/superword.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2488,8 +2488,9 @@ void SuperWord::output() {
24882488
} else if (VectorNode::is_scalar_rotate(n)) {
24892489
Node* in1 = low_adr->in(1);
24902490
Node* in2 = p->at(0)->in(2);
2491+
assert(in2->bottom_type()->isa_int(), "Shift must always be an int value");
24912492
// If rotation count is non-constant or greater than 8bit value create a vector.
2492-
if (!in2->is_Con() || !Matcher::supports_vector_constant_rotates(in2->get_int())) {
2493+
if (!in2->is_Con() || -0x80 > in2->get_int() || in2->get_int() >= 0x80) {
24932494
in2 = vector_opd(p, 2);
24942495
}
24952496
vn = VectorNode::make(opc, in1, in2, vlen, velt_basic_type(n));

‎src/hotspot/share/opto/vectorIntrinsics.cpp‎

Lines changed: 11 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -59,48 +59,6 @@ static bool check_vbox(const TypeInstPtr* vbox_type) {
5959
}
6060
#endif
6161

62-
bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt, bool has_scalar_args) {
63-
bool is_supported = true;
64-
// has_scalar_args flag is true only for non-constant scalar shift count,
65-
// since in this case shift needs to be broadcasted.
66-
if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
67-
(has_scalar_args &&
68-
!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed))) {
69-
is_supported = false;
70-
}
71-
72-
int lshiftopc, rshiftopc;
73-
switch(elem_bt) {
74-
case T_BYTE:
75-
lshiftopc = Op_LShiftI;
76-
rshiftopc = Op_URShiftB;
77-
break;
78-
case T_SHORT:
79-
lshiftopc = Op_LShiftI;
80-
rshiftopc = Op_URShiftS;
81-
break;
82-
case T_INT:
83-
lshiftopc = Op_LShiftI;
84-
rshiftopc = Op_URShiftI;
85-
break;
86-
case T_LONG:
87-
lshiftopc = Op_LShiftL;
88-
rshiftopc = Op_URShiftL;
89-
break;
90-
default:
91-
assert(false, "Unexpected type");
92-
}
93-
int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
94-
int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
95-
if (!is_supported &&
96-
arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed) &&
97-
arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed) &&
98-
arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
99-
is_supported = true;
100-
}
101-
return is_supported;
102-
}
103-
10462
Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
10563
assert(EnableVectorSupport, "");
10664

@@ -154,29 +112,17 @@ bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type
154112
return false;
155113
}
156114

157-
if (VectorNode::is_vector_rotate(sopc)) {
158-
if(!arch_supports_vector_rotate(sopc, num_elem, type, has_scalar_args)) {
115+
// Check that architecture supports this op-size-type combination.
116+
if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
159117
#ifndef PRODUCT
160-
if (C->print_intrinsics()) {
161-
tty->print_cr(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
162-
NodeClassNames[sopc], type2name(type), num_elem);
163-
}
164-
#endif
165-
return false;
118+
if (C->print_intrinsics()) {
119+
tty->print_cr(" ** Rejected vector op (%s,%s,%d) because architecture does not support it",
120+
NodeClassNames[sopc], type2name(type), num_elem);
166121
}
167-
} else {
168-
// Check that architecture supports this op-size-type combination.
169-
if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
170-
#ifndef PRODUCT
171-
if (C->print_intrinsics()) {
172-
tty->print_cr(" ** Rejected vector op (%s,%s,%d) because architecture does not support it",
173-
NodeClassNames[sopc], type2name(type), num_elem);
174-
}
175122
#endif
176-
return false;
177-
} else {
178-
assert(Matcher::match_rule_supported(sopc), "must be supported");
179-
}
123+
return false;
124+
} else {
125+
assert(Matcher::match_rule_supported(sopc), "must be supported");
180126
}
181127

182128
if (num_elem == 1) {
@@ -1554,9 +1500,7 @@ bool LibraryCallKit::inline_vector_broadcast_int() {
15541500
BasicType elem_bt = elem_type->basic_type();
15551501
int num_elem = vlen->get_con();
15561502
int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1557-
bool is_shift = VectorNode::is_shift_opcode(opc);
1558-
bool is_rotate = VectorNode::is_rotate_opcode(opc);
1559-
if (opc == 0 || (!is_shift && !is_rotate)) {
1503+
if (opc == 0 || !VectorNode::is_shift_opcode(opc)) {
15601504
if (C->print_intrinsics()) {
15611505
tty->print_cr(" ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
15621506
}
@@ -1569,37 +1513,18 @@ bool LibraryCallKit::inline_vector_broadcast_int() {
15691513
}
15701514
return false; // operation not supported
15711515
}
1572-
Node* cnt = argument(5);
15731516
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
15741517
const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1575-
const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
15761518

1577-
// If CPU supports vector constant rotate instructions pass it directly
1578-
bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
1579-
Matcher::supports_vector_constant_rotates(cnt_type->get_con());
1580-
bool has_scalar_args = is_rotate ? !is_const_rotate : true;
1581-
if (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args)) {
1519+
if (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, true /*has_scalar_args*/)) {
15821520
if (C->print_intrinsics()) {
15831521
tty->print_cr(" ** not supported: arity=0 op=int/%d vlen=%d etype=%s ismask=no",
15841522
sopc, num_elem, type2name(elem_bt));
15851523
}
15861524
return false; // not supported
15871525
}
15881526
Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1589-
Node* opd2 = NULL;
1590-
if (is_shift) {
1591-
opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
1592-
} else {
1593-
assert(is_rotate, "unexpected operation");
1594-
if (!is_const_rotate) {
1595-
const Type * type_bt = Type::get_const_basic_type(elem_bt);
1596-
cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
1597-
opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, type_bt));
1598-
} else {
1599-
// Constant shift value.
1600-
opd2 = cnt;
1601-
}
1602-
}
1527+
Node* opd2 = vector_shift_count(argument(5), opc, elem_bt, num_elem);
16031528
if (opd1 == NULL || opd2 == NULL) {
16041529
return false;
16051530
}

‎src/hotspot/share/opto/vectornode.cpp‎

Lines changed: 23 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -142,9 +142,9 @@ int VectorNode::opcode(int sopc, BasicType bt) {
142142
case Op_RoundDoubleMode:
143143
return (bt == T_DOUBLE ? Op_RoundDoubleModeV : 0);
144144
case Op_RotateLeft:
145-
return (is_integral_type(bt) ? Op_RotateLeftV : 0);
145+
return (bt == T_LONG || bt == T_INT ? Op_RotateLeftV : 0);
146146
case Op_RotateRight:
147-
return (is_integral_type(bt) ? Op_RotateRightV : 0);
147+
return (bt == T_LONG || bt == T_INT ? Op_RotateRightV : 0);
148148
case Op_SqrtF:
149149
return (bt == T_FLOAT ? Op_SqrtVF : 0);
150150
case Op_SqrtD:
@@ -261,7 +261,7 @@ bool VectorNode::implemented(int opc, uint vlen, BasicType bt) {
261261
// For rotate operation we will do a lazy de-generation into
262262
// OrV/LShiftV/URShiftV pattern if the target does not support
263263
// vector rotation instruction.
264-
if (VectorNode::is_vector_rotate(vopc)) {
264+
if (vopc == Op_RotateLeftV || vopc == Op_RotateRightV) {
265265
return is_vector_rotate_supported(vopc, vlen, bt);
266266
}
267267
return vopc > 0 && Matcher::match_rule_supported_vector(vopc, vlen, bt);
@@ -295,8 +295,15 @@ bool VectorNode::is_roundopD(Node* n) {
295295
return false;
296296
}
297297

298+
bool VectorNode::is_scalar_rotate(Node* n) {
299+
if (n->Opcode() == Op_RotateLeft || n->Opcode() == Op_RotateRight) {
300+
return true;
301+
}
302+
return false;
303+
}
304+
298305
bool VectorNode::is_vector_rotate_supported(int vopc, uint vlen, BasicType bt) {
299-
assert(VectorNode::is_vector_rotate(vopc), "wrong opcode");
306+
assert(vopc == Op_RotateLeftV || vopc == Op_RotateRightV, "wrong opcode");
300307

301308
// If target defines vector rotation patterns then no
302309
// need for degeneration.
@@ -340,23 +347,6 @@ bool VectorNode::is_shift(Node* n) {
340347
return is_shift_opcode(n->Opcode());
341348
}
342349

343-
bool VectorNode::is_rotate_opcode(int opc) {
344-
switch (opc) {
345-
case Op_RotateRight:
346-
case Op_RotateLeft:
347-
return true;
348-
default:
349-
return false;
350-
}
351-
}
352-
353-
bool VectorNode::is_scalar_rotate(Node* n) {
354-
if (is_rotate_opcode(n->Opcode())) {
355-
return true;
356-
}
357-
return false;
358-
}
359-
360350
bool VectorNode::is_vshift_cnt(Node* n) {
361351
switch (n->Opcode()) {
362352
case Op_LShiftCntV:
@@ -588,16 +578,6 @@ VectorNode* VectorNode::shift_count(int opc, Node* cnt, uint vlen, BasicType bt)
588578
}
589579
}
590580

591-
bool VectorNode::is_vector_rotate(int opc) {
592-
switch (opc) {
593-
case Op_RotateLeftV:
594-
case Op_RotateRightV:
595-
return true;
596-
default:
597-
return false;
598-
}
599-
}
600-
601581
bool VectorNode::is_vector_shift(int opc) {
602582
assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode");
603583
switch (opc) {
@@ -1151,66 +1131,42 @@ MacroLogicVNode* MacroLogicVNode::make(PhaseGVN& gvn, Node* in1, Node* in2, Node
11511131

11521132
Node* VectorNode::degenerate_vector_rotate(Node* src, Node* cnt, bool is_rotate_left,
11531133
int vlen, BasicType bt, PhaseGVN* phase) {
1154-
assert(is_integral_type(bt), "sanity");
1134+
assert(bt == T_INT || bt == T_LONG, "sanity");
11551135
const TypeVect* vt = TypeVect::make(bt, vlen);
11561136

1157-
int shift_mask = (type2aelembytes(bt) * 8) - 1;
1158-
int shiftLOpc = (bt == T_LONG) ? Op_LShiftL : Op_LShiftI;
1159-
auto urshiftopc = [=]() {
1160-
switch(bt) {
1161-
case T_INT: return Op_URShiftI;
1162-
case T_LONG: return Op_URShiftL;
1163-
case T_BYTE: return Op_URShiftB;
1164-
case T_SHORT: return Op_URShiftS;
1165-
default: return (Opcodes)0;
1166-
}
1167-
};
1168-
int shiftROpc = urshiftopc();
1137+
int shift_mask = (bt == T_INT) ? 0x1F : 0x3F;
1138+
int shiftLOpc = (bt == T_INT) ? Op_LShiftI : Op_LShiftL;
1139+
int shiftROpc = (bt == T_INT) ? Op_URShiftI: Op_URShiftL;
11691140

11701141
// Compute shift values for right rotation and
11711142
// later swap them in case of left rotation.
11721143
Node* shiftRCnt = NULL;
11731144
Node* shiftLCnt = NULL;
1174-
const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
1175-
bool is_binary_vector_op = false;
1176-
if (cnt_type && cnt_type->is_con()) {
1177-
// Constant shift.
1178-
int shift = cnt_type->get_con() & shift_mask;
1145+
if (cnt->is_Con() && cnt->bottom_type()->isa_int()) {
1146+
// Constant shift case.
1147+
int shift = cnt->get_int() & shift_mask;
11791148
shiftRCnt = phase->intcon(shift);
11801149
shiftLCnt = phase->intcon(shift_mask + 1 - shift);
1181-
} else if (VectorNode::is_invariant_vector(cnt)) {
1182-
// Scalar variable shift, handle replicates generated by auto vectorizer.
1150+
} else {
1151+
// Variable shift case.
11831152
assert(VectorNode::is_invariant_vector(cnt), "Broadcast expected");
11841153
cnt = cnt->in(1);
11851154
if (bt == T_LONG) {
11861155
// Shift count vector for Rotate vector has long elements too.
11871156
assert(cnt->Opcode() == Op_ConvI2L, "ConvI2L expected");
11881157
cnt = cnt->in(1);
11891158
}
1190-
shiftRCnt = cnt;
1159+
shiftRCnt = phase->transform(new AndINode(cnt, phase->intcon(shift_mask)));
11911160
shiftLCnt = phase->transform(new SubINode(phase->intcon(shift_mask + 1), shiftRCnt));
1192-
} else {
1193-
// Vector variable shift.
1194-
assert(cnt->bottom_type()->isa_vect(), "Unexpected shift");
1195-
const Type* elem_ty = Type::get_const_basic_type(bt);
1196-
Node* shift_mask_node = (bt == T_LONG) ? (Node*)(phase->longcon(shift_mask + 1L)) :
1197-
(Node*)(phase->intcon(shift_mask + 1));
1198-
Node* vector_mask = phase->transform(VectorNode::scalar2vector(shift_mask_node,vlen, elem_ty));
1199-
int subVopc = VectorNode::opcode((bt == T_LONG) ? Op_SubL : Op_SubI, bt);
1200-
shiftRCnt = cnt;
1201-
shiftLCnt = phase->transform(VectorNode::make(subVopc, vector_mask, shiftRCnt, vt));
1202-
is_binary_vector_op = true;
12031161
}
12041162

12051163
// Swap the computed left and right shift counts.
12061164
if (is_rotate_left) {
12071165
swap(shiftRCnt,shiftLCnt);
12081166
}
12091167

1210-
if (!is_binary_vector_op) {
1211-
shiftLCnt = phase->transform(new LShiftCntVNode(shiftLCnt, vt));
1212-
shiftRCnt = phase->transform(new RShiftCntVNode(shiftRCnt, vt));
1213-
}
1168+
shiftLCnt = phase->transform(new LShiftCntVNode(shiftLCnt, vt));
1169+
shiftRCnt = phase->transform(new RShiftCntVNode(shiftRCnt, vt));
12141170

12151171
return new OrVNode(phase->transform(VectorNode::make(shiftLOpc, src, shiftLCnt, vlen, bt)),
12161172
phase->transform(VectorNode::make(shiftROpc, src, shiftRCnt, vlen, bt)),

0 commit comments

Comments
 (0)