Skip to content

Commit f09e98f

Browse files
hanyisonghanyisong
committed
chore(databend):code tidying and local optimization
1 parent 5e7c000 commit f09e98f

File tree

9 files changed

+53
-94
lines changed

9 files changed

+53
-94
lines changed

‎src/sqlancer/ProviderAdapter.java‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -45,12 +45,8 @@ public void generateAndTestDatabase(G globalState) throws Exception {
4545
for (int i = 0; i < globalState.getOptions().getNrQueries(); i++) {
4646
try (OracleRunReproductionState localState = globalState.getState().createLocalState()) {
4747
assert localState != null;
48-
try {
49-
oracle.check();
50-
globalState.getManager().incrementSelectQueryCount();
51-
} catch (IgnoreMeException e) {
52-
53-
}
48+
oracle.check();
49+
globalState.getManager().incrementSelectQueryCount();
5450
assert localState != null;
5551
localState.executedWithoutError();
5652
}

‎src/sqlancer/databend/DatabendErrors.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,24 +9,29 @@ private DatabendErrors() {
99

1010
public static void addExpressionErrors(ExpectedErrors errors) {
1111
errors.add("Division by zero");
12-
12+
errors.add("/ by zero");
13+
errors.add("ORDER BY position");
14+
errors.add("GROUP BY position");
1315
/*
1416
* TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE
1517
* databend2; CREATE TABLE t0(c0VARCHAR VARCHAR NULL, c1VARCHAR VARCHAR NULL, c2FLOAT FLOAT NOT NULL
16-
* DEFAULT(NULL)); CREATE TABLE t1(c0INT BIGINT NULL); INSERT INTO t0(c1varchar, c0varchar) VALUES ('067596',
17-
* '19'), ('', '87');
18+
* DEFAULT(NULL)); CREATE TABLE t1(c0INT BIGINT NULL); INSERT INTO t0(c1varchar, c0varchar) VALUES
19+
* ('067596','19'), ('', '87');
1820
*/
1921
errors.add("Can't cast column from null into non-nullable type");
2022
}
2123

2224
public static void addInsertErrors(ExpectedErrors errors) {
2325
errors.add("Division by zero");
26+
errors.add("/ by zero");
2427
errors.add("Can't cast column from null into non-nullable type");
2528
}
2629

2730
public static void addGroupByErrors(ExpectedErrors errors) {
2831
errors.add("Division by zero");
32+
errors.add("/ by zero");
2933
errors.add("Can't cast column from null into non-nullable type");
34+
errors.add("GROUP BY position");
3035
}
3136

3237
}

‎src/sqlancer/databend/DatabendProvider.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import sqlancer.databend.gen.DatabendInsertGenerator;
2424
import sqlancer.databend.gen.DatabendRandomQuerySynthesizer;
2525
import sqlancer.databend.gen.DatabendTableGenerator;
26+
import sqlancer.databend.gen.DatabendViewGenerator;
2627

2728
@AutoService(DatabaseProvider.class)
2829
public class DatabendProvider extends SQLProviderAdapter<DatabendGlobalState, DatabendOptions> {
@@ -38,8 +39,7 @@ public enum Action implements AbstractAction<DatabendGlobalState> {
3839
// DELETE(DatabendDeleteGenerator::generate), //
3940
// UPDATE(DatabendUpdateGenerator::getQuery), //
4041

41-
// CREATE_VIEW(DatabendViewGenerator::generate), //TODO 等待databend的create view语法 更加贴近mysql
42-
EXPLAIN((g) -> {
42+
CREATE_VIEW(DatabendViewGenerator::generate), EXPLAIN((g) -> {
4343
ExpectedErrors errors = new ExpectedErrors();
4444
DatabendErrors.addExpressionErrors(errors);
4545
DatabendErrors.addGroupByErrors(errors);
@@ -73,8 +73,8 @@ private static int mapActions(DatabendGlobalState globalState, Action a) {
7373
// return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1);
7474
// case DELETE:
7575
// return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1);
76-
// case CREATE_VIEW: //TODO 暂时关闭create view
77-
// return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1);
76+
case CREATE_VIEW:
77+
return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1);
7878
default:
7979
throw new AssertionError(a);
8080
}

‎src/sqlancer/databend/DatabendSchema.java‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
import sqlancer.IgnoreMeException;
1111
import sqlancer.Randomly;
1212
import sqlancer.SQLConnection;
13-
import sqlancer.common.DBMSCommon;
1413
import sqlancer.common.schema.AbstractRelationalTable;
1514
import sqlancer.common.schema.AbstractSchema;
1615
import sqlancer.common.schema.AbstractTableColumn;
@@ -237,9 +236,6 @@ public static DatabendSchema fromConnection(SQLConnection con, String databaseNa
237236
List<DatabendTable> databaseTables = new ArrayList<>();
238237
List<String> tableNames = getTableNames(con, databaseName);
239238
for (String tableName : tableNames) {
240-
if (DBMSCommon.matchesIndexName(tableName)) {
241-
continue; // TODO: unexpected?
242-
}
243239
List<DatabendColumn> databaseColumns = getTableColumns(con, tableName, databaseName);
244240
boolean isView = tableName.startsWith("v");
245241
DatabendTable t = new DatabendTable(tableName, databaseColumns, isView);
@@ -292,11 +288,7 @@ private static List<DatabendColumn> getTableColumns(SQLConnection con, String ta
292288
}
293289
}
294290
}
295-
// if (columns.stream().noneMatch(c -> c.isPrimaryKey())) {
296-
// TODO: implement an option to enable/disable rowids
297-
// columns.add(new DatabendColumn("rowid", new DatabendCompositeDataType(DatabendDataType.INT, 4), false,
298-
// false));
299-
// }
291+
300292
return columns;
301293
}
302294

‎src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java‎

Lines changed: 25 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,12 @@
55
import java.util.stream.Collectors;
66

77
import sqlancer.Randomly;
8+
import sqlancer.common.ast.newast.ColumnReferenceNode;
89
import sqlancer.common.ast.newast.Node;
910
import sqlancer.common.ast.newast.TableReferenceNode;
1011
import sqlancer.databend.DatabendProvider.DatabendGlobalState;
1112
import sqlancer.databend.DatabendSchema;
13+
import sqlancer.databend.DatabendSchema.DatabendColumn;
1214
import sqlancer.databend.DatabendSchema.DatabendTable;
1315
import sqlancer.databend.DatabendSchema.DatabendTables;
1416
import sqlancer.databend.ast.DatabendConstant;
@@ -23,38 +25,42 @@ private DatabendRandomQuerySynthesizer() {
2325

2426
public static DatabendSelect generateSelect(DatabendGlobalState globalState, int nrColumns) {
2527
DatabendTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables();
26-
DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState)
27-
.setColumns(targetTables.getColumns());
28+
List<DatabendColumn> targetColumns = targetTables.getColumns();
29+
DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState).setColumns(targetColumns);
2830
DatabendSelect select = new DatabendSelect();
2931
// TODO distinct
30-
// select.setDistinct(Randomly.getBoolean());
32+
select.setDistinct(Randomly.getBoolean());
3133
// boolean allowAggregates = Randomly.getBooleanWithSmallProbability();
3234
List<Node<DatabendExpression>> columns = new ArrayList<>();
35+
int freeColumns = targetColumns.size();
3336
for (int i = 0; i < nrColumns; i++) {
3437
// if (allowAggregates && Randomly.getBoolean()) {
35-
Node<DatabendExpression> expression = gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN);
38+
Node<DatabendExpression> expression = null;
39+
if (freeColumns > 0 && Randomly.getBoolean()) {
40+
expression = new ColumnReferenceNode<>(targetColumns.get(freeColumns - 1));
41+
freeColumns -= 1;
42+
} else {
43+
expression = gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN);
44+
}
3645
columns.add(expression);
37-
// } else {
38-
// columns.add(gen());
39-
// }
4046
}
4147
select.setFetchColumns(columns);
4248
List<DatabendTable> tables = targetTables.getTables();
4349
List<TableReferenceNode<DatabendExpression, DatabendTable>> tableList = tables.stream()
4450
.map(t -> new TableReferenceNode<DatabendExpression, DatabendTable>(t)).collect(Collectors.toList());
4551
List<Node<DatabendExpression>> joins = DatabendJoin.getJoins(tableList, globalState);
46-
select.setJoinList(joins.stream().collect(Collectors.toList()));
52+
select.setJoinList(joins);
4753
select.setFromList(tableList.stream().collect(Collectors.toList()));
4854
if (Randomly.getBoolean()) {
4955
select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN));
5056
}
51-
// if (Randomly.getBoolean()) {//TODO order by超过实际行数
52-
// select.setOrderByExpressions(gen.generateOrderBys());
53-
// }
57+
if (Randomly.getBoolean()) {
58+
select.setOrderByExpressions(gen.generateOrderBys());
59+
}
5460

55-
// if (Randomly.getBoolean()) { //TODO group by超过实际行数
56-
// select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1));
57-
// }
61+
if (Randomly.getBoolean()) { // TODO 该表达式生成的类型较多，需要对其限制
62+
select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1));
63+
}
5864

5965
if (Randomly.getBoolean()) {
6066
select.setLimitClause(
@@ -64,10 +70,11 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int
6470
select.setOffsetClause(
6571
DatabendConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)));
6672
}
67-
// TODO 待添加HavingClause
68-
// if (Randomly.getBoolean()) {
69-
// select.setHavingClause(gen.generateHavingClause());
70-
// }
73+
74+
if (Randomly.getBoolean()) {
75+
select.setHavingClause(gen.generateHavingClause());
76+
}
77+
7178
return select;
7279
}
7380

‎src/sqlancer/databend/gen/DatabendTableGenerator.java‎

Lines changed: 3 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -34,50 +34,26 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) {
3434
sb.append(columns.get(i).getName());
3535
sb.append(" ");
3636
sb.append(columns.get(i).getType());
37-
// if (globalState.getDbmsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability()
38-
// && columns.get(i).getType().getPrimitiveDataType() == DatabendDataType.VARCHAR) {
39-
// sb.append(" COLLATE ");
40-
// sb.append(getRandomCollate());
41-
// }
42-
// if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) {
43-
// sb.append(" UNIQUE");
44-
// }
37+
4538
if (globalState.getDbmsSpecificOptions().testNotNullConstraints
4639
&& Randomly.getBooleanWithRatherLowProbability()) {
4740
sb.append(" NOT NULL");
4841
} else {
4942
sb.append(" NULL"); // Databend 默认字段为非空，这个将它默认设置为允许空
5043
}
51-
// if (globalState.getDbmsSpecificOptions().testCheckConstraints //databend 无check约束
52-
// && Randomly.getBooleanWithRatherLowProbability()) {
53-
// sb.append(" CHECK(");
54-
// sb.append(DatabendToStringVisitor.asString(gen.generateExpression()));
55-
// DatabendErrors.addExpressionErrors(errors);
56-
// sb.append(")");
57-
// }
44+
5845
if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) {
5946
sb.append(" DEFAULT(");
6047
sb.append(DatabendToStringVisitor.asString(// 常量类型于字段类型等同
6148
gen.generateConstant(columns.get(i).getType().getPrimitiveDataType())));
6249
sb.append(")");
6350
}
6451
}
65-
// databend并没有索引
66-
// if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) {
67-
// errors.add("Invalid type for index");
68-
// List<DatabendColumn> primaryKeyColumns = Randomly.nonEmptySubset(columns);
69-
// sb.append(", PRIMARY KEY(");
70-
// sb.append(primaryKeyColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", ")));
71-
// sb.append(")");
72-
// }
52+
7353
sb.append(")");
7454
return new SQLQueryAdapter(sb.toString(), errors, true);
7555
}
7656

77-
public static String getRandomCollate() {
78-
return Randomly.fromOptions("NOCASE", "NOACCENT", "NOACCENT.NOCASE", "C", "POSIX");
79-
}
80-
8157
private static List<DatabendColumn> getNewColumns() {
8258
List<DatabendColumn> columns = new ArrayList<>();
8359
for (int i = 0; i < Randomly.smallNumber() + 1; i++) {

‎src/sqlancer/databend/gen/DatabendViewGenerator.java‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,6 @@ public static SQLQueryAdapter generate(DatabendGlobalState globalState) {
1717
StringBuilder sb = new StringBuilder("CREATE ");
1818
sb.append("VIEW ");
1919
sb.append(globalState.getSchema().getFreeViewName());
20-
// sb.append("(");
21-
// for (int i = 0; i < nrColumns; i++) {
22-
// if (i != 0) {
23-
// sb.append(", ");
24-
// }
25-
// sb.append("c");
26-
// sb.append(i);
27-
// }
28-
// sb.append(") AS ");
2920
sb.append(" AS ");
3021
sb.append(DatabendToStringVisitor
3122
.asString(DatabendRandomQuerySynthesizer.generateSelect(globalState, nrColumns)));

‎src/sqlancer/databend/test/DatabendNoRECOracle.java‎

Lines changed: 8 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import java.sql.ResultSet;
44
import java.sql.SQLException;
55
import java.sql.Statement;
6-
import java.util.Arrays;
76
import java.util.List;
87
import java.util.stream.Collectors;
98

@@ -48,15 +47,14 @@ public void check() throws SQLException {
4847
DatabendTables randomTables = s.getRandomTableNonEmptyTables(); // 随机获得nr张表
4948
List<DatabendColumn> columns = randomTables.getColumns();
5049
DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(state).setColumns(columns);
51-
52-
Node<DatabendExpression> randomWhereCondition = gen.generateExpression(DatabendDataType.BOOLEAN); // 生成随机where条件，形式为ast
53-
50+
Node<DatabendExpression> randomWhereCondition = gen.generateExpression(DatabendDataType.BOOLEAN); // 生成随机where条件
5451
List<DatabendTable> tables = randomTables.getTables();
5552
List<TableReferenceNode<DatabendExpression, DatabendTable>> tableList = tables.stream()
5653
.map(t -> new TableReferenceNode<DatabendExpression, DatabendTable>(t)).collect(Collectors.toList());
5754
List<Node<DatabendExpression>> joins = DatabendJoin.getJoins(tableList, state);
58-
int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); // 禁用优化
59-
int firstCount = getFirstQueryCount(con, tableList.stream().collect(Collectors.toList()), columns,
55+
int secondCount = getUnoptimizedQueryCount(tableList.stream().collect(Collectors.toList()),
56+
randomWhereCondition, joins);
57+
int firstCount = getOptimizedQueryCount(con, tableList.stream().collect(Collectors.toList()), columns,
6058
randomWhereCondition, joins);
6159
if (firstCount == -1 || secondCount == -1) {
6260
throw new IgnoreMeException();
@@ -67,23 +65,20 @@ public void check() throws SQLException {
6765
}
6866
}
6967

70-
private int getSecondQuery(List<Node<DatabendExpression>> tableList, Node<DatabendExpression> randomWhereCondition,
71-
List<Node<DatabendExpression>> joins) throws SQLException {
68+
private int getUnoptimizedQueryCount(List<Node<DatabendExpression>> tableList,
69+
Node<DatabendExpression> randomWhereCondition, List<Node<DatabendExpression>> joins) throws SQLException {
7270
DatabendSelect select = new DatabendSelect();
7371
// select.setGroupByClause(groupBys);
74-
// DatabendExpression isTrue = DatabendPostfixOperation.create(randomWhereCondition,
75-
// PostfixOperator.IS_TRUE);
7672
Node<DatabendExpression> asText = new NewPostfixTextNode<>(new DatabendCastOperation(
7773
new NewPostfixTextNode<DatabendExpression>(randomWhereCondition,
7874
" IS NOT NULL AND " + DatabendToStringVisitor.asString(randomWhereCondition)),
7975
new DatabendCompositeDataType(DatabendDataType.INT, 8)), "as count");
8076

81-
select.setFetchColumns(Arrays.asList(asText)); // ?
77+
select.setFetchColumns(List.of(asText));
8278
select.setFromList(tableList);
8379
select.setJoinList(joins);
8480
int secondCount = 0;
8581
unoptimizedQueryString = "SELECT SUM(count) FROM (" + DatabendToStringVisitor.asString(select) + ") as res";
86-
errors.add("canceling statement due to statement timeout");
8782
SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors);
8883
SQLancerResultSet rs;
8984
try {
@@ -101,17 +96,14 @@ private int getSecondQuery(List<Node<DatabendExpression>> tableList, Node<Databe
10196
return secondCount;
10297
}
10398

104-
private int getFirstQueryCount(SQLConnection con, List<Node<DatabendExpression>> tableList,
99+
private int getOptimizedQueryCount(SQLConnection con, List<Node<DatabendExpression>> tableList,
105100
List<DatabendColumn> columns, Node<DatabendExpression> randomWhereCondition,
106101
List<Node<DatabendExpression>> joins) throws SQLException {
107102
DatabendSelect select = new DatabendSelect();
108103
// select.setGroupByClause(groupBys);
109-
// DatabendAggregate aggr = new DatabendAggregate(
110104
List<Node<DatabendExpression>> allColumns = columns.stream()
111105
.map((c) -> new ColumnReferenceNode<DatabendExpression, DatabendColumn>(c))
112106
.collect(Collectors.toList());
113-
// DatabendAggregateFunction.COUNT);
114-
// select.setFetchColumns(Arrays.asList(aggr));
115107
select.setFetchColumns(allColumns);
116108
select.setFromList(tableList);
117109
select.setWhereClause(randomWhereCondition);

‎test/sqlancer/dbms/TestDatabend.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ public void testDatabendNoREC() {
1717
assertEquals(0,
1818
Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4",
1919
"--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend",
20-
"--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC),
20+
"--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC),
2121
"--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "NOREC"));
2222
}
2323

@@ -29,7 +29,7 @@ public void testDatabendTLPQueryPartitioning() {
2929
assertEquals(0,
3030
Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4",
3131
"--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend",
32-
"--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC),
32+
"--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC),
3333
"--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "QUERY_PARTITIONING"));
3434
}
3535

0 commit comments

Comments
 (0)