Skip to content

Commit 43a373a

Browse files
malwaregarrymalwaregarry
authored
[Doris] Updated Doris to 2.1.4 (#949)
* [Doris] Update deployment in CI * [Doris] Update group by columns to match fetch columns * [Doris] Prevent generation of null for some expressions * [Doris] Add bug 36070 * [Doris] Add new error message * [Doris] Update CI script and pin to version 2.1.3 * [Doris] Add bug 36072 * [Doris] Fix bugs in TLP Aggregate oracle * [Doris] Add bug 34342 * [Doris] Add bug 36343 * [Doris] Add bug 36346 * [Doris] Update bugs list * [Doris] Update to version 2.1.4 * [Doris] Refactor tests * [Doris] Add bug 36351
1 parent e01300f commit 43a373a

14 files changed

+188
-110
lines changed

‎.github/workflows/main.yml‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -544,26 +544,26 @@ jobs:
544544
- name: Set up Apache Doris
545545
run: |
546546
sudo sysctl -w vm.max_map_count=2000000
547-
LASTEST_TAG=$(curl -s GET https://api.github.com/repos/apache/doris/releases | jq -r '.[].tag_name' | sed -n 1p)
548-
LASTEST_TAG_BIG_VERSION=$(echo ${LASTEST_TAG} | awk '{split($1, arr, "."); print arr[1]"."arr[2]}')
549-
curl -LJO "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=doris/${LASTEST_TAG_BIG_VERSION}/${LASTEST_TAG}/apache-doris-fe-${LASTEST_TAG}-bin-x86_64.tar.xz"
550-
curl -LJO "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=doris/${LASTEST_TAG_BIG_VERSION}/${LASTEST_TAG}/apache-doris-be-${LASTEST_TAG}-bin-x86_64.tar.xz"
551-
curl -LJO "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=doris/${LASTEST_TAG_BIG_VERSION}/${LASTEST_TAG}/apache-doris-dependencies-${LASTEST_TAG}-bin-x86_64.tar.xz"
552-
mkdir ./doris
553-
tar xf apache-doris-fe-${LASTEST_TAG}-bin-x86_64.tar.xz -C ./doris && mv doris/apache-doris-fe-${LASTEST_TAG}-bin-x86_64 doris/fe
554-
tar xf apache-doris-be-${LASTEST_TAG}-bin-x86_64.tar.xz -C ./doris && mv doris/apache-doris-be-${LASTEST_TAG}-bin-x86_64 doris/be
555-
tar xf apache-doris-dependencies-${LASTEST_TAG}-bin-x86_64.tar.xz -C ./doris && mv doris/apache-doris-dependencies-${LASTEST_TAG}-bin-x86_64 doris/dependencies
556-
cp doris/dependencies/*.jar doris/be/lib/
557-
doris/fe/bin/start_fe.sh --daemon
558-
doris/be/bin/start_be.sh --daemon
559-
sudo apt install libnet-ifconfig-wrapper-perl --assume-yes
560-
IP=$(ifconfig eth0 | grep inet | grep -v inet6 | awk '{print $2}')
547+
wget -q https://apache-doris-releases.oss-accelerate.aliyuncs.com/apache-doris-2.1.4-bin-x64.tar.gz
548+
tar zxf apache-doris-2.1.4-bin-x64.tar.gz
549+
mv apache-doris-2.1.4-bin-x64 apache-doris
550+
sudo swapoff -a
551+
cd apache-doris/fe
552+
./bin/start_fe.sh --daemon
553+
cd ../be
554+
./bin/start_be.sh --daemon
555+
556+
sleep 30
557+
IP=$(hostname -I | awk '{print $1}')
561558
mysql -u root -h 127.0.0.1 --port 9030 -e "ALTER SYSTEM ADD BACKEND '${IP}:9050';"
562559
mysql -u root -h 127.0.0.1 --port 9030 -e "CREATE USER 'sqlancer' IDENTIFIED BY 'sqlancer'; GRANT ALL ON *.* TO sqlancer;"
563560
- name: Build SQLancer
564561
run: mvn -B package -DskipTests=true
565562
- name: Run Tests
566-
run: DORIS_AVAILABLE=true mvn -Dtest=TestDoris test
563+
run: |
564+
DORIS_AVAILABLE=true mvn -Dtest=TestDorisNoREC test
565+
DORIS_AVAILABLE=true mvn -Dtest=TestDorisPQS test
566+
DORIS_AVAILABLE=true mvn -Dtest=TestDorisTLP test
567567
568568
reducer:
569569
name: Reducer Tests

‎src/sqlancer/doris/DorisBugs.java‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,43 +1,43 @@
11
package sqlancer.doris;
22

33
public final class DorisBugs {
4-
// https://github.com/apache/doris/issues/17697
5-
// Logical bug about where true not in (columns)
6-
public static boolean bug17697 = true;
7-
8-
// https://github.com/apache/doris/issues/17700
9-
// Cannot use between and in boolean column
10-
public static boolean bug17700 = true;
11-
12-
// https://github.com/apache/doris/issues/17701
13-
// Wrong result of `where column not in (values)`
14-
public static boolean bug17701 = true;
15-
16-
// https://github.com/apache/doris/issues/17705
17-
// Different result caused by `where` split and union all
18-
public static boolean bug17705 = true;
19-
204
// https://github.com/apache/doris/issues/19370
215
// Internal Error occur in GroupBy&Having sql
226
// fixed by https://github.com/apache/doris/pull/19559
23-
public static boolean bug19370 = true;
7+
public static boolean bug19370;
248

259
// https://github.com/apache/doris/issues/19374
2610
// Different result of having not ($value in column) and having ($value not in column)
2711
// fixed by https://github.com/apache/doris/pull/19471
28-
public static boolean bug19374 = true;
12+
public static boolean bug19374;
2913

3014
// https://github.com/apache/doris/issues/19611
3115
// ERROR occur in nested subqueries with same column name and union
3216
public static boolean bug19611 = true;
3317

34-
// https://github.com/apache/doris/issues/19613
35-
// Wrong result when right outer join and where false
36-
public static boolean bug19613 = true;
18+
// https://github.com/apache/doris/issues/36070
19+
// Expression evaluate to NULL but is treated as FALSE in where clause
20+
public static boolean bug36070 = true;
21+
22+
// https://github.com/apache/doris/issues/36072
23+
// SELECT DISTINCT does not work with aggregate key column
24+
public static boolean bug36072 = true;
25+
26+
// https://github.com/apache/doris/issues/36342
27+
// Wrong result with INNER JOIN and CURRENT_TIMESTAMP
28+
public static boolean bug36342 = true;
29+
30+
// https://github.com/apache/doris/issues/36343
31+
// Wrong result with SELECT DISTINCT and UNIQUE model
32+
public static boolean bug36343 = true;
33+
34+
// https://github.com/apache/doris/issues/36346
35+
// Wrong result with LEFT JOIN SELECT DISTINCT and IN operation
36+
public static boolean bug36346 = true;
3737

38-
// https://github.com/apache/doris/issues/19614
39-
// Wrong result when value like column from table_join
40-
public static boolean bug19614 = true;
38+
// https://github.com/apache/doris/issues/36351
39+
// Wrong result with TINYINT column with value -1049190528
40+
public static boolean bug36351 = true;
4141

4242
private DorisBugs() {
4343

‎src/sqlancer/doris/DorisErrors.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ public static List<String> getExpressionErrors() {
5757
if (DorisBugs.bug19611) {
5858
errors.add("Duplicated inline view column alias");
5959
}
60+
errors.add("Arithmetic overflow");
6061

6162
return errors;
6263
}
@@ -72,6 +73,7 @@ public static List<String> getInsertErrors() {
7273
errors.add("Only value columns of unique table could be updated");
7374
errors.add("Only unique olap table could be updated");
7475
errors.add("Number out of range");
76+
errors.add("Arithmetic overflow");
7577

7678
return errors;
7779
}

‎src/sqlancer/doris/DorisSchema.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import java.sql.SQLException;
55
import java.sql.Statement;
66
import java.util.ArrayList;
7+
import java.util.Arrays;
78
import java.util.Collections;
89
import java.util.HashMap;
910
import java.util.List;
@@ -30,7 +31,14 @@ public enum DorisTableDataModel {
3031
UNIQUE, AGGREGATE, DUPLICATE;
3132

3233
public static DorisTableDataModel getRandom() {
33-
return Randomly.fromOptions(values());
34+
List<DorisTableDataModel> validOptions = new ArrayList<>(Arrays.asList(values()));
35+
if (DorisBugs.bug36072) {
36+
validOptions.remove(AGGREGATE);
37+
}
38+
if (DorisBugs.bug36343) {
39+
validOptions.remove(UNIQUE);
40+
}
41+
return Randomly.fromList(validOptions);
3442
}
3543
}
3644

‎src/sqlancer/doris/gen/DorisNewExpressionGenerator.java‎

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,12 @@
77
import java.util.Set;
88
import java.util.stream.Collectors;
99

10+
import sqlancer.IgnoreMeException;
1011
import sqlancer.Randomly;
1112
import sqlancer.common.ast.newast.NewOrderingTerm;
1213
import sqlancer.common.ast.newast.Node;
1314
import sqlancer.common.gen.TypedExpressionGenerator;
15+
import sqlancer.doris.DorisBugs;
1416
import sqlancer.doris.DorisProvider.DorisGlobalState;
1517
import sqlancer.doris.DorisSchema.DorisColumn;
1618
import sqlancer.doris.DorisSchema.DorisDataType;
@@ -117,10 +119,12 @@ public DorisExpression generateExpression(DorisDataType type, int depth) {
117119
return function.getCall(type, this, depth + 1);
118120
}
119121
}
120-
if (globalState.getDbmsSpecificOptions().testCasts && Randomly.getBooleanWithRatherLowProbability()) {
122+
if (!DorisBugs.bug36070 && type != DorisDataType.NULL && globalState.getDbmsSpecificOptions().testCasts
123+
&& Randomly.getBooleanWithRatherLowProbability()) {
121124
return new DorisCastOperation(DorisExprToNode.cast(generateExpression(getRandomType(), depth + 1)), type);
122125
}
123-
if (globalState.getDbmsSpecificOptions().testCase && Randomly.getBooleanWithRatherLowProbability()) {
126+
if (!DorisBugs.bug36070 && globalState.getDbmsSpecificOptions().testCase
127+
&& Randomly.getBooleanWithRatherLowProbability()) {
124128
DorisExpression expr = generateExpression(DorisDataType.BOOLEAN, depth + 1);
125129
List<DorisExpression> conditions = new ArrayList<>();
126130
List<DorisExpression> cases = new ArrayList<>();
@@ -192,7 +196,7 @@ DorisExpression generateBooleanExpression(int depth) {
192196
allowAggregateFunctions = false;
193197
}
194198
List<BooleanExpression> validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values()));
195-
if (!globalState.getDbmsSpecificOptions().testIn) {
199+
if (DorisBugs.bug36346 || !globalState.getDbmsSpecificOptions().testIn) {
196200
validOptions.remove(BooleanExpression.IN_OPERATION);
197201
}
198202
if (!globalState.getDbmsSpecificOptions().testBinaryLogicals) {
@@ -201,7 +205,7 @@ DorisExpression generateBooleanExpression(int depth) {
201205
if (!globalState.getDbmsSpecificOptions().testBinaryComparisons) {
202206
validOptions.remove(BooleanExpression.BINARY_COMPARISON);
203207
}
204-
if (!globalState.getDbmsSpecificOptions().testBetween) {
208+
if (DorisBugs.bug36070 || !globalState.getDbmsSpecificOptions().testBetween) {
205209
validOptions.remove(BooleanExpression.BETWEEN);
206210
}
207211

@@ -306,8 +310,11 @@ public DorisExpression isNull(DorisExpression predicate) {
306310
}
307311

308312
public DorisExpression generateConstant(DorisDataType type, boolean isNullable) {
309-
if (isNullable && Randomly.getBooleanWithSmallProbability()) {
310-
createConstant(DorisDataType.NULL);
313+
if (!isNullable) {
314+
return createConstantWithoutNull(type);
315+
}
316+
if (Randomly.getBooleanWithSmallProbability()) {
317+
return createConstant(DorisDataType.NULL);
311318
}
312319
return createConstant(type);
313320
}
@@ -320,12 +327,29 @@ public DorisExpression generateConstant(DorisDataType type) {
320327
return createConstant(type);
321328
}
322329

330+
public DorisExpression createConstantWithoutNull(DorisDataType type) {
331+
DorisExpression constant = createConstant(type);
332+
int loopCount = 0;
333+
while (constant instanceof DorisConstant.DorisNullConstant && loopCount < 1000) {
334+
constant = createConstant(type);
335+
loopCount++;
336+
}
337+
if (constant instanceof DorisConstant.DorisNullConstant) {
338+
throw new IgnoreMeException();
339+
}
340+
return constant;
341+
}
342+
323343
public DorisExpression createConstant(DorisDataType type) {
324344
Randomly r = globalState.getRandomly();
325345
long timestamp;
326346
switch (type) {
327347
case INT:
328348
if (globalState.getDbmsSpecificOptions().testIntConstants) {
349+
long number = r.getInteger();
350+
if (DorisBugs.bug36351 && number == -1049190528) {
351+
number = 0;
352+
}
329353
return DorisConstant.createIntConstant(r.getInteger());
330354
}
331355
return DorisConstant.createNullConstant();
@@ -368,6 +392,9 @@ public DorisExpression createConstant(DorisDataType type) {
368392
if (globalState.getDbmsSpecificOptions().testDateTimeConstants) {
369393
// [1970-01-01 08:00:00, 3000-01-01 00:00:00]
370394
timestamp = globalState.getRandomly().getLong(0, 32503651200L);
395+
if (DorisBugs.bug36342) {
396+
return DorisConstant.createDatetimeConstant(timestamp);
397+
}
371398
return Randomly.fromOptions(DorisConstant.createDatetimeConstant(timestamp),
372399
DorisConstant.createDatetimeConstant());
373400
}

‎src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -74,9 +74,21 @@ public void check() throws SQLException {
7474

7575
state.getState().getLocalState().log(
7676
"--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult + "\n-- " + secondResult);
77-
if (firstResult == null && secondResult != null
78-
|| firstResult != null && (!firstResult.contentEquals(secondResult)
79-
&& !ComparatorHelper.isEqualDouble(firstResult, secondResult))) {
77+
if (firstResult == null && secondResult == null) {
78+
return;
79+
}
80+
if (firstResult == null) {
81+
throw new AssertionError();
82+
}
83+
firstResult = firstResult.replace("\0", "");
84+
if (firstResult.contentEquals("0") && secondResult == null) {
85+
return;
86+
}
87+
if (secondResult == null) {
88+
throw new AssertionError();
89+
}
90+
secondResult = secondResult.replace("\0", "");
91+
if (!firstResult.contentEquals(secondResult) && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) {
8092
throw new AssertionError();
8193
}
8294

@@ -94,6 +106,11 @@ private String createMetamorphicUnionQuery(DorisSelect select,
94106
DorisSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList());
95107
DorisSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList());
96108
DorisSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList());
109+
if (Randomly.getBooleanWithSmallProbability()) {
110+
leftSelect.setGroupByExpressions(groupByExpression);
111+
middleSelect.setGroupByExpressions(groupByExpression);
112+
rightSelect.setGroupByExpressions(groupByExpression);
113+
}
97114
metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM (";
98115
metamorphicQuery += DorisToStringVisitor.asString(leftSelect) + " UNION ALL "
99116
+ DorisToStringVisitor.asString(middleSelect) + " UNION ALL "
@@ -185,9 +202,6 @@ private DorisSelect getSelect(List<Node<DorisExpression>> aggregates, List<Node<
185202
leftSelect.setFromList(from);
186203
leftSelect.setWhereClause(whereClause);
187204
leftSelect.setJoinList(joinList);
188-
if (Randomly.getBooleanWithSmallProbability()) {
189-
leftSelect.setGroupByExpressions(groupByExpression);
190-
}
191205
return leftSelect;
192206
}
193207

‎src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,13 +46,14 @@ public void check() throws SQLException {
4646
s = state.getSchema();
4747
targetTables = s.getRandomTableNonEmptyTables();
4848
gen = new DorisNewExpressionGenerator(state).setColumns(targetTables.getColumns());
49+
List<DorisColumnValue> allColumnValues = targetTables.getColumns().stream()
50+
.map(c -> new DorisColumnValue(c, null)).collect(Collectors.toList());
4951
HashSet<DorisColumnValue> columnOfLeafNode = new HashSet<>();
5052
gen.setColumnOfLeafNode(columnOfLeafNode);
5153
initializeTernaryPredicateVariants();
5254
select = new DorisSelect();
53-
columnOfLeafNode.addAll(targetTables.getColumns().stream().map(c -> new DorisColumnValue(c, null))
54-
.collect(Collectors.toList()));
55-
groupByExpression = new ArrayList<>(columnOfLeafNode);
55+
columnOfLeafNode.addAll(allColumnValues);
56+
groupByExpression = new ArrayList<>(allColumnValues);
5657
select.setFetchColumns(generateFetchColumns());
5758
List<DorisTable> tables = targetTables.getTables();
5859
List<TableReferenceNode<DorisExpression, DorisTable>> tableList = tables.stream()

‎src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ public DorisQueryPartitioningGroupByTester(DorisGlobalState state) {
2727
@Override
2828
public void check() throws SQLException {
2929
super.check();
30-
select.setGroupByExpressions(groupByExpression);
30+
select.setGroupByExpressions(select.getFetchColumns());
3131
select.setWhereClause(null);
3232
String originalQueryString = DorisToStringVisitor.asString(select);
3333

‎src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ public void check() throws SQLException {
3131
if (Randomly.getBoolean()) {
3232
select.setWhereClause(DorisExprToNode.cast(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)));
3333
}
34+
select.setFetchColumns(groupByExpression);
3435
boolean orderBy = Randomly.getBoolean();
3536
if (orderBy) {
3637
List<Node<DorisExpression>> constants = new ArrayList<>();

‎test/sqlancer/dbms/TestConfig.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ public class TestConfig {
44
public static final String NUM_QUERIES = "1000";
55
public static final String SECONDS = "300";
66

7+
public static final String DORIS_ENV = "DORIS_AVAILABLE";
78
public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE";
89

910
public static boolean isEnvironmentTrue(String key) {

0 commit comments

Comments
 (0)