Skip to content

Commit e696c6e

Browse files
authored
Support Apache Doris (#758)
* support doris init commit * support doris complete Signed-off-by: ChaseHuangxu <huangxu@smail.nju.edu.cn> * code format & correct github workflow main.yml about doris & add doris to README.md support dbms * correct README.md support dbms * change sql union method to union all in TLPDistinct check Signed-off-by: ChaseHuangxu <huangxu@smail.nju.edu.cn> * Limit the data format inserted by decimal type, and support Doris function call Signed-off-by: ChaseHuangxu <huangxu@smail.nju.edu.cn> * correct github/workflows/DBMS Tests(doris) Signed-off-by: ChaseHuangxu <huangxu@smail.nju.edu.cn> --------- Signed-off-by: ChaseHuangxu <huangxu@smail.nju.edu.cn>
1 parent 9e17ede commit e696c6e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+5476
-2
lines changed

.github/workflows/main.yml

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -442,11 +442,52 @@ jobs:
442442
- name: Setup Yugabyte
443443
run: |
444444
docker pull yugabytedb/yugabyte:latest
445-
docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false
445+
docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false
446446
sleep 5
447447
- name: Run Tests
448448
run: YUGABYTE_AVAILABLE=true mvn -Dtest=TestYugabyte test
449449

450+
451+
doris:
452+
name: DBMS Tests (Apache Doris)
453+
runs-on: ubuntu-latest
454+
steps:
455+
- uses: actions/checkout@v2
456+
with:
457+
fetch-depth: 0
458+
- name: Set up JDK 11
459+
uses: actions/setup-java@v1
460+
with:
461+
java-version: 11
462+
- name: install mysql client
463+
run: |
464+
sudo apt update
465+
sudo apt install mysql-client --assume-yes
466+
- name: Set up Apache Doris
467+
run: |
468+
sudo sysctl -w vm.max_map_count=2000000
469+
LASTEST_TAG=$(curl -s GET https://api.github.com/repos/apache/doris/releases | jq -r '.[].tag_name' | sed -n 1p)
470+
LASTEST_TAG_BIG_VERSION=$(echo ${LASTEST_TAG} | awk '{split($1, arr, "."); print arr[1]"."arr[2]}')
471+
curl -LJO "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=doris/${LASTEST_TAG_BIG_VERSION}/${LASTEST_TAG}/apache-doris-fe-${LASTEST_TAG}-bin-x86_64.tar.xz"
472+
curl -LJO "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=doris/${LASTEST_TAG_BIG_VERSION}/${LASTEST_TAG}/apache-doris-be-${LASTEST_TAG}-bin-x86_64.tar.xz"
473+
curl -LJO "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=doris/${LASTEST_TAG_BIG_VERSION}/${LASTEST_TAG}/apache-doris-dependencies-${LASTEST_TAG}-bin-x86_64.tar.xz"
474+
mkdir ./doris
475+
tar xf apache-doris-fe-${LASTEST_TAG}-bin-x86_64.tar.xz -C ./doris && mv doris/apache-doris-fe-${LASTEST_TAG}-bin-x86_64 doris/fe
476+
tar xf apache-doris-be-${LASTEST_TAG}-bin-x86_64.tar.xz -C ./doris && mv doris/apache-doris-be-${LASTEST_TAG}-bin-x86_64 doris/be
477+
tar xf apache-doris-dependencies-${LASTEST_TAG}-bin-x86_64.tar.xz -C ./doris && mv doris/apache-doris-dependencies-${LASTEST_TAG}-bin-x86_64 doris/dependencies
478+
cp doris/dependencies/*.jar doris/be/lib/
479+
doris/fe/bin/start_fe.sh --daemon
480+
doris/be/bin/start_be.sh --daemon
481+
sudo apt install libnet-ifconfig-wrapper-perl --assume-yes
482+
IP=$(ifconfig eth0 | grep inet | grep -v inet6 | awk '{print $2}')
483+
mysql -u root -h 127.0.0.1 --port 9030 -e "ALTER SYSTEM ADD BACKEND '${IP}:9050';"
484+
mysql -u root -h 127.0.0.1 --port 9030 -e "CREATE USER 'sqlancer' IDENTIFIED BY 'sqlancer'; GRANT ALL ON *.* TO sqlancer;"
485+
- name: Build SQLancer
486+
run: mvn -B package -DskipTests=true
487+
- name: Run Tests
488+
run: DORIS_AVAILABLE=true mvn -Dtest=TestDoris test
489+
490+
450491
java13:
451492
name: Java 13 Compatibility (DuckDB)
452493
runs-on: ubuntu-latest

README.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,8 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp
6969
| Databend | Working | Typed | |
7070
| QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. |
7171
| CnosDB |Working | Typed | The implementation of CnosDB currently uses Restful API. |
72-
| Materialize |Working | Typed | |
72+
| Materialize |Working | Typed |
73+
| Apache Doris | Preliminary | Typed | This is a preliminary implementation, which only contains the common logic of Doris. We have found some errors through it, and hope to improve it in the future.
7374

7475

7576
# Using SQLancer

src/check_names.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,4 +52,5 @@ def verify_all_dbs(name_to_files: dict[str:List[str]]):
5252
name_to_files["SQLite3"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "sqlite3"))
5353
name_to_files["TiDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "tidb"))
5454
name_to_files["Y"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "yugabyte")) # has both YCQL and YSQL prefixes
55+
name_to_files["Doris"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "doris"))
5556
verify_all_dbs(name_to_files)

src/sqlancer/doris/DorisBugs.java

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
package sqlancer.doris;
2+
3+
public final class DorisBugs {
4+
// https://github.com/apache/doris/issues/17697
5+
// Logical bug about where true not in (columns)
6+
public static boolean bug17697 = true;
7+
8+
// https://github.com/apache/doris/issues/17700
9+
// Cannot use between and in boolean column
10+
public static boolean bug17700 = true;
11+
12+
// https://github.com/apache/doris/issues/17701
13+
// Wrong result of `where column not in (values)`
14+
public static boolean bug17701 = true;
15+
16+
// https://github.com/apache/doris/issues/17705
17+
// Different result caused by `where` split and union all
18+
public static boolean bug17705 = true;
19+
20+
// https://github.com/apache/doris/issues/19370
21+
// Internal Error occur in GroupBy&Having sql
22+
// fixed by https://github.com/apache/doris/pull/19559
23+
public static boolean bug19370 = true;
24+
25+
// https://github.com/apache/doris/issues/19374
26+
// Different result of having not ($value in column) and having ($value not in column)
27+
// fixed by https://github.com/apache/doris/pull/19471
28+
public static boolean bug19374 = true;
29+
30+
// https://github.com/apache/doris/issues/19611
31+
// ERROR occur in nested subqueries with same column name and union
32+
public static boolean bug19611 = true;
33+
34+
// https://github.com/apache/doris/issues/19613
35+
// Wrong result when right outer join and where false
36+
public static boolean bug19613 = true;
37+
38+
// https://github.com/apache/doris/issues/19614
39+
// Wrong result when value like column from table_join
40+
public static boolean bug19614 = true;
41+
42+
private DorisBugs() {
43+
44+
}
45+
}
Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
package sqlancer.doris;
2+
3+
import sqlancer.common.query.ExpectedErrors;
4+
5+
public final class DorisErrors {
6+
7+
private DorisErrors() {
8+
}
9+
10+
public static void addExpressionErrors(ExpectedErrors errors) {
11+
// SQL syntax error
12+
errors.add("Syntax error");
13+
errors.add("Please check your sql, we meet an error when parsing");
14+
errors.add("but returns type");
15+
errors.add("is not a number");
16+
17+
// Not in line with Doris' logic
18+
errors.add("Unexpected exception: null");
19+
errors.add("Cross join can't be used with ON clause");
20+
errors.add("BetweenPredicate needs to be rewritten into a CompoundPredicate");
21+
errors.add("can't be assigned to some PlanNode");
22+
errors.add("can not cast from origin type");
23+
errors.add("not produced by aggregation output");
24+
errors.add("cannot combine"); // cannot combine SELECT DISTINCT with aggregate functions or GROUP BY
25+
errors.add("Invalid type");
26+
errors.add("cannot be cast to");
27+
28+
// functions
29+
errors.add("No matching function with signature");
30+
errors.add("Invalid number format");
31+
errors.add("group_concat requires");
32+
errors.add("function's argument should be");
33+
errors.add("requires a numeric parameter");
34+
errors.add("out of bounds");
35+
errors.add("function do not support");
36+
errors.add("parameter must be");
37+
errors.add("Not supported input arguments types");
38+
errors.add("No matching function with signature");
39+
errors.add("function");
40+
errors.add("Invalid");
41+
errors.add("Incorrect");
42+
43+
// regex
44+
45+
// To avoid bugs
46+
if (DorisBugs.bug19370) {
47+
errors.add("failed to initialize storage");
48+
}
49+
if (DorisBugs.bug19374) {
50+
errors.add("the size of the result sets mismatch");
51+
}
52+
if (DorisBugs.bug19611) {
53+
errors.add("Duplicated inline view column alias");
54+
}
55+
}
56+
57+
public static void addInsertErrors(ExpectedErrors errors) {
58+
errors.add("Insert has filtered data in strict mode");
59+
errors.add("Only value columns of unique table could be updated");
60+
errors.add("Only unique olap table could be updated");
61+
errors.add("Number out of range");
62+
}
63+
64+
}
Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
package sqlancer.doris;
2+
3+
import java.sql.SQLException;
4+
import java.util.ArrayList;
5+
import java.util.Arrays;
6+
import java.util.List;
7+
8+
import com.beust.jcommander.Parameter;
9+
import com.beust.jcommander.Parameters;
10+
11+
import sqlancer.DBMSSpecificOptions;
12+
import sqlancer.OracleFactory;
13+
import sqlancer.common.oracle.CompositeTestOracle;
14+
import sqlancer.common.oracle.TestOracle;
15+
import sqlancer.doris.DorisOptions.DorisOracleFactory;
16+
import sqlancer.doris.DorisProvider.DorisGlobalState;
17+
import sqlancer.doris.oracle.DorisNoRECOracle;
18+
import sqlancer.doris.oracle.DorisPivotedQuerySynthesisOracle;
19+
import sqlancer.doris.oracle.tlp.DorisQueryPartitioningAggregateTester;
20+
import sqlancer.doris.oracle.tlp.DorisQueryPartitioningDistinctTester;
21+
import sqlancer.doris.oracle.tlp.DorisQueryPartitioningGroupByTester;
22+
import sqlancer.doris.oracle.tlp.DorisQueryPartitioningHavingTester;
23+
import sqlancer.doris.oracle.tlp.DorisQueryPartitioningWhereTester;
24+
25+
@Parameters(commandDescription = "Apache Doris (default port: " + DorisOptions.DEFAULT_PORT + ", default host: "
26+
+ DorisOptions.DEFAULT_HOST + ")")
27+
public class DorisOptions implements DBMSSpecificOptions<DorisOracleFactory> {
28+
public static final String DEFAULT_HOST = "localhost";
29+
public static final int DEFAULT_PORT = 9030;
30+
31+
@Parameter(names = { "--max-num-tables" }, description = "The maximum number of tables/views that can be created")
32+
public int maxNumTables = 10;
33+
34+
@Parameter(names = { "--max-num-indexes" }, description = "The maximum number of indexes that can be created")
35+
public int maxNumIndexes = 20;
36+
37+
@Parameter(names = "--test-default-values", description = "Allow generating DEFAULT values in tables", arity = 1)
38+
public boolean testDefaultValues = true;
39+
40+
@Parameter(names = "--test-not-null", description = "Allow generating NOT NULL constraints in tables", arity = 1)
41+
public boolean testNotNullConstraints = true;
42+
43+
@Parameter(names = "--test-functions", description = "Allow generating functions in expressions", arity = 1)
44+
public boolean testFunctions;
45+
46+
@Parameter(names = "--test-casts", description = "Allow generating casts in expressions", arity = 1)
47+
public boolean testCasts = true;
48+
49+
@Parameter(names = "--test-between", description = "Allow generating the BETWEEN operator in expressions", arity = 1)
50+
public boolean testBetween = true;
51+
52+
@Parameter(names = "--test-in", description = "Allow generating the IN operator in expressions", arity = 1)
53+
public boolean testIn = true;
54+
55+
@Parameter(names = "--test-case", description = "Allow generating the CASE operator in expressions", arity = 1)
56+
public boolean testCase = true;
57+
58+
@Parameter(names = "--test-binary-logicals", description = "Allow generating AND and OR in expressions", arity = 1)
59+
public boolean testBinaryLogicals = true;
60+
61+
@Parameter(names = "--test-int-constants", description = "Allow generating INTEGER constants", arity = 1)
62+
public boolean testIntConstants = true;
63+
64+
@Parameter(names = "--test-float-constants", description = "Allow generating floating-point constants", arity = 1)
65+
public boolean testFloatConstants = true;
66+
67+
@Parameter(names = "--test-decimal-constants", description = "Allow generating DECIMAL constants", arity = 1)
68+
public boolean testDecimalConstants = true;
69+
70+
@Parameter(names = "--test-date-constants", description = "Allow generating DATE constants", arity = 1)
71+
public boolean testDateConstants = true;
72+
73+
@Parameter(names = "--test-datetime-constants", description = "Allow generating DATETIME constants", arity = 1)
74+
public boolean testDateTimeConstants = true;
75+
76+
@Parameter(names = "--test-varchar-constants", description = "Allow generating VARCHAR constants", arity = 1)
77+
public boolean testStringConstants = true;
78+
79+
@Parameter(names = "--test-boolean-constants", description = "Allow generating boolean constants", arity = 1)
80+
public boolean testBooleanConstants = true;
81+
82+
@Parameter(names = "--test-binary-comparisons", description = "Allow generating binary comparison operators (e.g., >= or LIKE)", arity = 1)
83+
public boolean testBinaryComparisons = true;
84+
85+
@Parameter(names = "--max-num-deletes", description = "The maximum number of DELETE statements that are issued for a database", arity = 1)
86+
public int maxNumDeletes = 1;
87+
88+
@Parameter(names = "--max-num-updates", description = "The maximum number of UPDATE statements that are issued for a database", arity = 1)
89+
public int maxNumUpdates;
90+
91+
@Parameter(names = "--max-num-table-alters", description = "The maximum number of ALTER TABLE statements that are issued for a database", arity = 1)
92+
public int maxNumTableAlters;
93+
94+
@Parameter(names = "--test-engine-type", description = "The engine type in Doris, only consider OLAP now", arity = 1)
95+
public String testEngineType = "OLAP"; // skip now
96+
97+
@Parameter(names = "--test-indexes", description = "Allow explicit indexes, Doris only supports creating indexes on single-column BITMAP", arity = 1)
98+
public boolean testIndexes = true; // skip now
99+
100+
@Parameter(names = "--test-column-aggr", description = "Allow test column aggregation (sum, min, max, replace, replace_if_not_null, hll_union, bitmap_untion)", arity = 1)
101+
public boolean testColumnAggr = true;
102+
103+
@Parameter(names = "--test-datemodel", description = "Allow generating Doris’s data model in tables. (Aggregate、Uniqe、Duplicate)", arity = 1)
104+
public boolean testDataModel = true;
105+
106+
@Parameter(names = "--test-distribution", description = "Allow generating data distribution in tables.", arity = 1)
107+
public boolean testDistribution = true; // must have it, skip now
108+
109+
@Parameter(names = "--test-rollup", description = "Allow generating rollups in tables.", arity = 1)
110+
public boolean testRollup = true; // skip now
111+
112+
@Parameter(names = "--oracle")
113+
public List<DorisOracleFactory> oracles = Arrays.asList(DorisOracleFactory.NOREC);
114+
115+
public enum DorisOracleFactory implements OracleFactory<DorisGlobalState> {
116+
NOREC {
117+
@Override
118+
public TestOracle<DorisGlobalState> create(DorisGlobalState globalState) throws SQLException {
119+
return new DorisNoRECOracle(globalState);
120+
}
121+
122+
},
123+
HAVING {
124+
@Override
125+
public TestOracle<DorisGlobalState> create(DorisGlobalState globalState) throws SQLException {
126+
return new DorisQueryPartitioningHavingTester(globalState);
127+
}
128+
},
129+
WHERE {
130+
@Override
131+
public TestOracle<DorisGlobalState> create(DorisGlobalState globalState) throws SQLException {
132+
return new DorisQueryPartitioningWhereTester(globalState);
133+
}
134+
},
135+
GROUP_BY {
136+
@Override
137+
public TestOracle<DorisGlobalState> create(DorisGlobalState globalState) throws SQLException {
138+
return new DorisQueryPartitioningGroupByTester(globalState);
139+
}
140+
},
141+
AGGREGATE {
142+
@Override
143+
public TestOracle<DorisGlobalState> create(DorisGlobalState globalState) throws SQLException {
144+
return new DorisQueryPartitioningAggregateTester(globalState);
145+
}
146+
147+
},
148+
DISTINCT {
149+
@Override
150+
public TestOracle<DorisGlobalState> create(DorisGlobalState globalState) throws SQLException {
151+
return new DorisQueryPartitioningDistinctTester(globalState);
152+
}
153+
},
154+
QUERY_PARTITIONING {
155+
@Override
156+
public TestOracle<DorisGlobalState> create(DorisGlobalState globalState) throws SQLException {
157+
List<TestOracle<DorisGlobalState>> oracles = new ArrayList<>();
158+
oracles.add(new DorisQueryPartitioningWhereTester(globalState));
159+
oracles.add(new DorisQueryPartitioningHavingTester(globalState));
160+
oracles.add(new DorisQueryPartitioningAggregateTester(globalState));
161+
oracles.add(new DorisQueryPartitioningDistinctTester(globalState));
162+
oracles.add(new DorisQueryPartitioningGroupByTester(globalState));
163+
return new CompositeTestOracle<DorisGlobalState>(oracles, globalState);
164+
}
165+
},
166+
PQS {
167+
@Override
168+
public TestOracle<DorisGlobalState> create(DorisGlobalState globalState) throws Exception {
169+
return new DorisPivotedQuerySynthesisOracle(globalState);
170+
}
171+
},
172+
ALL {
173+
@Override
174+
public TestOracle<DorisGlobalState> create(DorisGlobalState globalState) throws Exception {
175+
List<TestOracle<DorisGlobalState>> oracles = new ArrayList<>();
176+
oracles.add(new DorisNoRECOracle(globalState));
177+
oracles.add(new DorisQueryPartitioningWhereTester(globalState));
178+
oracles.add(new DorisQueryPartitioningHavingTester(globalState));
179+
oracles.add(new DorisQueryPartitioningAggregateTester(globalState));
180+
oracles.add(new DorisQueryPartitioningDistinctTester(globalState));
181+
oracles.add(new DorisQueryPartitioningGroupByTester(globalState));
182+
oracles.add(new DorisPivotedQuerySynthesisOracle(globalState));
183+
return new CompositeTestOracle<DorisGlobalState>(oracles, globalState);
184+
}
185+
}
186+
187+
}
188+
189+
@Override
190+
public List<DorisOracleFactory> getTestOracleFactory() {
191+
return oracles;
192+
}
193+
194+
}

0 commit comments

Comments
 (0)