Skip to content

Commit b05c40c

Browse files
author
Jatin Bhateja
committed
8266951: Partial in-lining for vectorized mismatch operation using AVX512 masked instructions
Reviewed-by: psandoz, vlivanov
1 parent f768fbf commit b05c40c

22 files changed

Lines changed: 725 additions & 97 deletions

‎src/hotspot/cpu/x86/assembler_x86.cpp‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2572,6 +2572,13 @@ void Assembler::knotwl(KRegister dst, KRegister src) {
25722572
emit_int16(0x44, (0xC0 | encode));
25732573
}
25742574

2575+
void Assembler::knotql(KRegister dst, KRegister src) {
2576+
assert(VM_Version::supports_avx512bw(), "");
2577+
InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2578+
int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2579+
emit_int16(0x44, (0xC0 | encode));
2580+
}
2581+
25752582
// This instruction produces ZF or CF flags
25762583
void Assembler::kortestbl(KRegister src1, KRegister src2) {
25772584
assert(VM_Version::supports_avx512dq(), "");

‎src/hotspot/cpu/x86/assembler_x86.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1480,6 +1480,7 @@ class Assembler : public AbstractAssembler {
14801480
void kmovql(Register dst, KRegister src);
14811481

14821482
void knotwl(KRegister dst, KRegister src);
1483+
void knotql(KRegister dst, KRegister src);
14831484

14841485
void kortestbl(KRegister dst, KRegister src);
14851486
void kortestwl(KRegister dst, KRegister src);

‎src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1923,7 +1923,7 @@ void C2_MacroAssembler::reduce8L(int opcode, Register dst, Register src1, XMMReg
19231923
}
19241924

19251925
void C2_MacroAssembler::genmask(KRegister dst, Register len, Register temp) {
1926-
assert(ArrayCopyPartialInlineSize <= 64,"");
1926+
assert(ArrayOperationPartialInlineSize > 0 && ArrayOperationPartialInlineSize <= 64, "invalid");
19271927
mov64(temp, -1L);
19281928
bzhiq(temp, temp, len);
19291929
kmovql(dst, temp);
@@ -2140,11 +2140,37 @@ void C2_MacroAssembler::get_elem(BasicType typ, XMMRegister dst, XMMRegister src
21402140
}
21412141
}
21422142

2143+
void C2_MacroAssembler::evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, XMMRegister src2, int comparison, int vector_len) {
2144+
switch(typ) {
2145+
case T_BYTE:
2146+
case T_BOOLEAN:
2147+
evpcmpb(kdmask, ksmask, src1, src2, comparison, vector_len);
2148+
break;
2149+
case T_SHORT:
2150+
case T_CHAR:
2151+
evpcmpw(kdmask, ksmask, src1, src2, comparison, vector_len);
2152+
break;
2153+
case T_INT:
2154+
case T_FLOAT:
2155+
evpcmpd(kdmask, ksmask, src1, src2, comparison, vector_len);
2156+
break;
2157+
case T_LONG:
2158+
case T_DOUBLE:
2159+
evpcmpq(kdmask, ksmask, src1, src2, comparison, vector_len);
2160+
break;
2161+
default:
2162+
assert(false,"Should not reach here.");
2163+
break;
2164+
}
2165+
}
2166+
21432167
void C2_MacroAssembler::evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, AddressLiteral adr, int comparison, int vector_len, Register scratch) {
21442168
switch(typ) {
2169+
case T_BOOLEAN:
21452170
case T_BYTE:
21462171
evpcmpb(kdmask, ksmask, src1, adr, comparison, /*signed*/ true, vector_len, scratch);
21472172
break;
2173+
case T_CHAR:
21482174
case T_SHORT:
21492175
evpcmpw(kdmask, ksmask, src1, adr, comparison, /*signed*/ true, vector_len, scratch);
21502176
break;

‎src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@
139139

140140
// blend
141141
void evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, AddressLiteral adr, int comparison, int vector_len, Register scratch = rscratch1);
142+
void evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, XMMRegister src2, int comparison, int vector_len);
142143
void evpblend(BasicType typ, XMMRegister dst, KRegister kmask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len);
143144

144145
void load_vector_mask(XMMRegister dst, XMMRegister src, int vlen_in_bytes, BasicType elem_bt);

‎src/hotspot/cpu/x86/vm_version_x86.cpp‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1419,12 +1419,12 @@ void VM_Version::get_processor_features() {
14191419
}
14201420
#ifdef COMPILER2
14211421
if (UseAVX > 2) {
1422-
if (FLAG_IS_DEFAULT(ArrayCopyPartialInlineSize) ||
1423-
(!FLAG_IS_DEFAULT(ArrayCopyPartialInlineSize) &&
1424-
ArrayCopyPartialInlineSize != 0 &&
1425-
ArrayCopyPartialInlineSize != 32 &&
1426-
ArrayCopyPartialInlineSize != 16 &&
1427-
ArrayCopyPartialInlineSize != 64)) {
1422+
if (FLAG_IS_DEFAULT(ArrayOperationPartialInlineSize) ||
1423+
(!FLAG_IS_DEFAULT(ArrayOperationPartialInlineSize) &&
1424+
ArrayOperationPartialInlineSize != 0 &&
1425+
ArrayOperationPartialInlineSize != 16 &&
1426+
ArrayOperationPartialInlineSize != 32 &&
1427+
ArrayOperationPartialInlineSize != 64)) {
14281428
int inline_size = 0;
14291429
if (MaxVectorSize >= 64 && AVX3Threshold == 0) {
14301430
inline_size = 64;
@@ -1433,18 +1433,18 @@ void VM_Version::get_processor_features() {
14331433
} else if (MaxVectorSize >= 16) {
14341434
inline_size = 16;
14351435
}
1436-
if(!FLAG_IS_DEFAULT(ArrayCopyPartialInlineSize)) {
1437-
warning("Setting ArrayCopyPartialInlineSize as %d", inline_size);
1436+
if(!FLAG_IS_DEFAULT(ArrayOperationPartialInlineSize)) {
1437+
warning("Setting ArrayOperationPartialInlineSize as %d", inline_size);
14381438
}
1439-
ArrayCopyPartialInlineSize = inline_size;
1439+
ArrayOperationPartialInlineSize = inline_size;
14401440
}
14411441

1442-
if (ArrayCopyPartialInlineSize > MaxVectorSize) {
1443-
ArrayCopyPartialInlineSize = MaxVectorSize >= 16 ? MaxVectorSize : 0;
1444-
if (ArrayCopyPartialInlineSize) {
1445-
warning("Setting ArrayCopyPartialInlineSize as MaxVectorSize" INTX_FORMAT ")", MaxVectorSize);
1442+
if (ArrayOperationPartialInlineSize > MaxVectorSize) {
1443+
ArrayOperationPartialInlineSize = MaxVectorSize >= 16 ? MaxVectorSize : 0;
1444+
if (ArrayOperationPartialInlineSize) {
1445+
warning("Setting ArrayOperationPartialInlineSize as MaxVectorSize" INTX_FORMAT ")", MaxVectorSize);
14461446
} else {
1447-
warning("Setting ArrayCopyPartialInlineSize as " INTX_FORMAT, ArrayCopyPartialInlineSize);
1447+
warning("Setting ArrayOperationPartialInlineSize as " INTX_FORMAT, ArrayOperationPartialInlineSize);
14481448
}
14491449
}
14501450
}

‎src/hotspot/cpu/x86/x86.ad‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1578,6 +1578,7 @@ const bool Matcher::match_rule_supported(int opcode) {
15781578
}
15791579
break;
15801580

1581+
case Op_VectorCmpMasked:
15811582
case Op_VectorMaskGen:
15821583
case Op_LoadVectorMasked:
15831584
case Op_StoreVectorMasked:
@@ -1678,6 +1679,7 @@ const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType
16781679
break;
16791680
case Op_ClearArray:
16801681
case Op_VectorMaskGen:
1682+
case Op_VectorCmpMasked:
16811683
case Op_LoadVectorMasked:
16821684
case Op_StoreVectorMasked:
16831685
if (!is_LP64 || !VM_Version::supports_avx512bw()) {
@@ -8084,7 +8086,34 @@ instruct vprorate(vec dst, vec src, vec shift) %{
80848086
%}
80858087

80868088
#ifdef _LP64
8087-
// ---------------------------------- Masked Block Copy ------------------------------------
8089+
// ---------------------------------- Masked Operations ------------------------------------
8090+
8091+
instruct vmask_cmp_node(rRegI dst, vec src1, vec src2, kReg mask, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
8092+
match(Set dst (VectorCmpMasked src1 (Binary src2 mask)));
8093+
effect(TEMP_DEF dst, TEMP ktmp1, TEMP ktmp2, KILL cr);
8094+
format %{ "vector_mask_cmp $src1, $src2, $mask \t! vector mask comparison" %}
8095+
ins_encode %{
8096+
assert(vector_length_encoding(this, $src1) == vector_length_encoding(this, $src2), "mismatch");
8097+
assert(vector_element_basic_type(this, $src1) == vector_element_basic_type(this, $src2), "mismatch");
8098+
8099+
Label DONE;
8100+
int vlen_enc = vector_length_encoding(this, $src1);
8101+
BasicType elem_bt = vector_element_basic_type(this, $src1);
8102+
8103+
__ knotql($ktmp2$$KRegister, $mask$$KRegister);
8104+
__ mov64($dst$$Register, -1L);
8105+
__ evpcmp(elem_bt, $ktmp1$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, Assembler::eq, vlen_enc);
8106+
__ kortestql($ktmp2$$KRegister, $ktmp1$$KRegister);
8107+
__ jccb(Assembler::carrySet, DONE);
8108+
__ kmovql($dst$$Register, $ktmp1$$KRegister);
8109+
__ notq($dst$$Register);
8110+
__ tzcntq($dst$$Register, $dst$$Register);
8111+
__ bind(DONE);
8112+
%}
8113+
ins_pipe( pipe_slow );
8114+
%}
8115+
8116+
80888117
instruct vmasked_load64(vec dst, memory mem, kReg mask) %{
80898118
match(Set dst (LoadVectorMasked mem mask));
80908119
format %{ "vector_masked_load $dst, $mem, $mask \t! vector masked copy" %}

‎src/hotspot/share/adlc/formssel.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -789,6 +789,7 @@ bool InstructForm::captures_bottom_type(FormDict &globals) const {
789789
!strcmp(_matrule->_rChild->_opType,"ShenandoahCompareAndExchangeN") ||
790790
#endif
791791
!strcmp(_matrule->_rChild->_opType,"StrInflatedCopy") ||
792+
!strcmp(_matrule->_rChild->_opType,"VectorCmpMasked")||
792793
!strcmp(_matrule->_rChild->_opType,"VectorMaskGen")||
793794
!strcmp(_matrule->_rChild->_opType,"CompareAndExchangeP") ||
794795
!strcmp(_matrule->_rChild->_opType,"CompareAndExchangeN"))) return true;

‎src/hotspot/share/opto/arraycopynode.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -738,7 +738,7 @@ bool ArrayCopyNode::modifies(intptr_t offset_lo, intptr_t offset_hi, PhaseTransf
738738

739739
// As an optimization, choose optimum vector size for copy length known at compile time.
740740
int ArrayCopyNode::get_partial_inline_vector_lane_count(BasicType type, int const_len) {
741-
int lane_count = ArrayCopyPartialInlineSize/type2aelembytes(type);
741+
int lane_count = ArrayOperationPartialInlineSize/type2aelembytes(type);
742742
if (const_len > 0) {
743743
int size_in_bytes = const_len * type2aelembytes(type);
744744
if (size_in_bytes <= 16)

‎src/hotspot/share/opto/c2_globals.hpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -82,9 +82,10 @@
8282
"actual size could be less depending on elements type") \
8383
range(0, max_jint) \
8484
\
85-
product(intx, ArrayCopyPartialInlineSize, -1, DIAGNOSTIC, \
86-
"Partial inline size used for array copy acceleration.") \
87-
range(-1, 64) \
85+
product(intx, ArrayOperationPartialInlineSize, 0, DIAGNOSTIC, \
86+
"Partial inline size used for small array operations" \
87+
"(e.g. copy,cmp) acceleration.") \
88+
range(0, 64) \
8889
\
8990
product(bool, AlignVector, true, \
9091
"Perform vector store/load alignment in loop") \

‎src/hotspot/share/opto/castnode.hpp‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,11 @@ class CastIINode: public ConstraintCastNode {
7878
: ConstraintCastNode(n, t, carry_dependency), _range_check_dependency(range_check_dependency) {
7979
init_class_id(Class_CastII);
8080
}
81+
CastIINode(Node* ctrl, Node* n, const Type* t, bool carry_dependency = false, bool range_check_dependency = false)
82+
: ConstraintCastNode(n, t, carry_dependency), _range_check_dependency(range_check_dependency) {
83+
init_class_id(Class_CastII);
84+
init_req(0, ctrl);
85+
}
8186
virtual int Opcode() const;
8287
virtual uint ideal_reg() const { return Op_RegI; }
8388
virtual Node* Identity(PhaseGVN* phase);
@@ -103,6 +108,11 @@ class CastIINode: public ConstraintCastNode {
103108

104109
class CastLLNode: public ConstraintCastNode {
105110
public:
111+
CastLLNode(Node* ctrl, Node* n, const Type* t, bool carry_dependency = false)
112+
: ConstraintCastNode(n, t, carry_dependency) {
113+
init_class_id(Class_CastLL);
114+
init_req(0, ctrl);
115+
}
106116
CastLLNode(Node* n, const Type* t, bool carry_dependency = false)
107117
: ConstraintCastNode(n, t, carry_dependency){
108118
init_class_id(Class_CastLL);

0 commit comments

Comments
 (0)