Skip to content

Commit ea05d14

Browse files
hanyisonghanyisong
committed
feature(databend): enhance norec, tlp, and implement pqs
Signed-off-by: Yisong Han <yisong8686@gmail.com>
1 parent 7560fb6 commit ea05d14

37 files changed

+1626
-292
lines changed

‎src/sqlancer/ProviderAdapter.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,12 @@ public void generateAndTestDatabase(G globalState) throws Exception {
4545
for (int i = 0; i < globalState.getOptions().getNrQueries(); i++) {
4646
try (OracleRunReproductionState localState = globalState.getState().createLocalState()) {
4747
assert localState != null;
48-
oracle.check();
49-
globalState.getManager().incrementSelectQueryCount();
48+
try {
49+
oracle.check();
50+
globalState.getManager().incrementSelectQueryCount();
51+
} catch (IgnoreMeException e) {
52+
53+
}
5054
assert localState != null;
5155
localState.executedWithoutError();
5256
}

‎src/sqlancer/databend/DatabendErrors.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@ public static void addExpressionErrors(ExpectedErrors errors) {
1212
errors.add("/ by zero");
1313
errors.add("ORDER BY position");
1414
errors.add("GROUP BY position");
15+
errors.add("downcast column error"); // bug
16+
errors.add("index out of bounds"); // bug
17+
errors.add("validity's length must be equal"); // bug
18+
errors.add("validity must be equal to the array's length"); // bug
1519
/*
1620
* TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE
1721
* databend2; CREATE TABLE t0(c0VARCHAR VARCHAR NULL, c1VARCHAR VARCHAR NULL, c2FLOAT FLOAT NOT NULL

‎src/sqlancer/databend/DatabendExpectedValueVisitor.java‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
package sqlancer.databend;
2+
3+
import java.util.List;
4+
5+
import sqlancer.common.ast.newast.ColumnReferenceNode;
6+
import sqlancer.common.ast.newast.NewAliasNode;
7+
import sqlancer.common.ast.newast.NewBetweenOperatorNode;
8+
import sqlancer.common.ast.newast.NewBinaryOperatorNode;
9+
import sqlancer.common.ast.newast.NewFunctionNode;
10+
import sqlancer.common.ast.newast.NewInOperatorNode;
11+
import sqlancer.common.ast.newast.NewOrderingTerm;
12+
import sqlancer.common.ast.newast.NewPostfixTextNode;
13+
import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode;
14+
import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode;
15+
import sqlancer.common.ast.newast.Node;
16+
import sqlancer.common.ast.newast.TableReferenceNode;
17+
import sqlancer.databend.DatabendSchema.DatabendColumn;
18+
import sqlancer.databend.ast.DatabendConstant;
19+
import sqlancer.databend.ast.DatabendExpression;
20+
import sqlancer.databend.ast.DatabendJoin;
21+
import sqlancer.databend.ast.DatabendSelect;
22+
23+
public class DatabendExpectedValueVisitor {
24+
25+
protected final StringBuilder sb = new StringBuilder();
26+
27+
private void print(Node<DatabendExpression> expr) {
28+
sb.append(DatabendToStringVisitor.asString(expr));
29+
sb.append(" -- ");
30+
sb.append(((DatabendExpression) expr).getExpectedValue());
31+
sb.append("\n");
32+
}
33+
34+
@SuppressWarnings("unchecked")
35+
public void visit(Node<DatabendExpression> expr) {
36+
assert expr != null;
37+
if (expr instanceof ColumnReferenceNode<?, ?>) {
38+
visit((ColumnReferenceNode<DatabendExpression, DatabendColumn>) expr);
39+
} else if (expr instanceof NewUnaryPostfixOperatorNode<?>) {
40+
visit((NewUnaryPostfixOperatorNode<DatabendExpression>) expr);
41+
} else if (expr instanceof NewUnaryPrefixOperatorNode<?>) {
42+
visit((NewUnaryPrefixOperatorNode<DatabendExpression>) expr);
43+
} else if (expr instanceof NewBinaryOperatorNode<?>) {
44+
visit((NewBinaryOperatorNode<DatabendExpression>) expr);
45+
} else if (expr instanceof TableReferenceNode<?, ?>) {
46+
visit((TableReferenceNode<DatabendExpression, ?>) expr);
47+
} else if (expr instanceof NewFunctionNode<?, ?>) {
48+
visit((NewFunctionNode<DatabendExpression, ?>) expr);
49+
} else if (expr instanceof NewBetweenOperatorNode<?>) {
50+
visit((NewBetweenOperatorNode<DatabendExpression>) expr);
51+
} else if (expr instanceof NewInOperatorNode<?>) {
52+
visit((NewInOperatorNode<DatabendExpression>) expr);
53+
} else if (expr instanceof NewOrderingTerm<?>) {
54+
visit((NewOrderingTerm<DatabendExpression>) expr);
55+
} else if (expr instanceof NewAliasNode<?>) {
56+
visit((NewAliasNode<DatabendExpression>) expr);
57+
} else if (expr instanceof NewPostfixTextNode<?>) {
58+
visit((NewPostfixTextNode<DatabendExpression>) expr);
59+
} else if (expr instanceof DatabendConstant) {
60+
visit((DatabendConstant) expr);
61+
} else if (expr instanceof DatabendSelect) {
62+
visit((DatabendSelect) expr);
63+
} else if (expr instanceof DatabendJoin) {
64+
visit((DatabendJoin) expr);
65+
} else {
66+
throw new AssertionError(expr);
67+
}
68+
}
69+
70+
public void visit(ColumnReferenceNode<DatabendExpression, DatabendColumn> c) {
71+
print(c);
72+
}
73+
74+
public void visit(NewUnaryPostfixOperatorNode<DatabendExpression> op) {
75+
print(op);
76+
visit(op.getExpr());
77+
}
78+
79+
public void visit(NewUnaryPrefixOperatorNode<DatabendExpression> op) {
80+
print(op);
81+
visit(op.getExpr());
82+
}
83+
84+
public void visit(NewBinaryOperatorNode<DatabendExpression> op) {
85+
print(op);
86+
visit(op.getLeft());
87+
visit(op.getRight());
88+
}
89+
90+
public void visit(TableReferenceNode<DatabendExpression, ?> t) {
91+
print(t);
92+
}
93+
94+
public void visit(NewFunctionNode<DatabendExpression, ?> fun) {
95+
print(fun);
96+
visit(fun.getArgs());
97+
}
98+
99+
public void visit(List<Node<DatabendExpression>> expressions) {
100+
for (Node<DatabendExpression> expression : expressions) {
101+
visit(expression);
102+
}
103+
}
104+
105+
public void visit(NewBetweenOperatorNode<DatabendExpression> op) {
106+
print(op);
107+
visit(op.getLeft());
108+
visit(op.getMiddle());
109+
visit(op.getRight());
110+
}
111+
112+
public void visit(NewInOperatorNode<DatabendExpression> op) {
113+
print(op);
114+
visit(op.getLeft());
115+
visit(op.getRight());
116+
}
117+
118+
public void visit(NewOrderingTerm<DatabendExpression> op) {
119+
print(op);
120+
visit(op.getExpr());
121+
}
122+
123+
public void visit(NewAliasNode<DatabendExpression> op) {
124+
print(op);
125+
visit(op.getExpr());
126+
}
127+
128+
public void visit(NewPostfixTextNode<DatabendExpression> postFixText) {
129+
print(postFixText);
130+
visit(postFixText.getExpr());
131+
}
132+
133+
public void visit(DatabendConstant constant) {
134+
print(constant);
135+
}
136+
137+
public void visit(DatabendSelect select) {
138+
print(select.getWhereClause());
139+
}
140+
141+
public void visit(DatabendJoin join) {
142+
print(join.getOnCondition());
143+
}
144+
145+
public String get() {
146+
return sb.toString();
147+
}
148+
149+
public static String asExpectedValues(Node<DatabendExpression> expr) {
150+
DatabendExpectedValueVisitor v = new DatabendExpectedValueVisitor();
151+
v.visit(expr);
152+
return v.get();
153+
}
154+
155+
}

‎src/sqlancer/databend/DatabendExprToNode.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package sqlancer.databend;
2+
3+
import java.util.List;
4+
import java.util.stream.Collectors;
5+
6+
import sqlancer.common.ast.newast.Node;
7+
import sqlancer.databend.ast.DatabendExpression;
8+
9+
public class DatabendExprToNode {
10+
11+
private DatabendExprToNode() {
12+
13+
}
14+
15+
@SuppressWarnings("unchecked")
16+
public static Node<DatabendExpression> cast(DatabendExpression expression) {
17+
return (Node<DatabendExpression>) expression;
18+
}
19+
20+
@SuppressWarnings("unchecked")
21+
public static List<Node<DatabendExpression>> casts(List<DatabendExpression> expressions) {
22+
return expressions.stream().map(e -> (Node<DatabendExpression>) e).collect(Collectors.toList());
23+
}
24+
25+
}

‎src/sqlancer/databend/DatabendOptions.java‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,12 @@
1515
import sqlancer.databend.DatabendOptions.DatabendOracleFactory;
1616
import sqlancer.databend.DatabendProvider.DatabendGlobalState;
1717
import sqlancer.databend.test.DatabendNoRECOracle;
18-
import sqlancer.databend.test.DatabendQueryPartitioningAggregateTester;
19-
import sqlancer.databend.test.DatabendQueryPartitioningDistinctTester;
20-
import sqlancer.databend.test.DatabendQueryPartitioningGroupByTester;
21-
import sqlancer.databend.test.DatabendQueryPartitioningHavingTester;
22-
import sqlancer.databend.test.DatabendQueryPartitioningWhereTester;
18+
import sqlancer.databend.test.DatabendPivotedQuerySynthesisOracle;
19+
import sqlancer.databend.test.tlp.DatabendQueryPartitioningAggregateTester;
20+
import sqlancer.databend.test.tlp.DatabendQueryPartitioningDistinctTester;
21+
import sqlancer.databend.test.tlp.DatabendQueryPartitioningGroupByTester;
22+
import sqlancer.databend.test.tlp.DatabendQueryPartitioningHavingTester;
23+
import sqlancer.databend.test.tlp.DatabendQueryPartitioningWhereTester;
2324

2425
@Parameters(commandDescription = "Databend")
2526
public class DatabendOptions implements DBMSSpecificOptions<DatabendOracleFactory> {
@@ -147,7 +148,13 @@ public TestOracle create(DatabendGlobalState globalState) throws SQLException {
147148
oracles.add(new DatabendQueryPartitioningGroupByTester(globalState));
148149
return new CompositeTestOracle(oracles, globalState);
149150
}
150-
};
151+
},
152+
PQS {
153+
@Override
154+
public TestOracle create(DatabendGlobalState globalState) throws Exception {
155+
return new DatabendPivotedQuerySynthesisOracle(globalState);
156+
}
157+
}
151158

152159
}
153160

‎src/sqlancer/databend/DatabendProvider.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,10 @@ public DatabendProvider() {
3535

3636
public enum Action implements AbstractAction<DatabendGlobalState> {
3737

38-
INSERT(DatabendInsertGenerator::getQuery),
39-
DELETE(DatabendDeleteGenerator::generate),
38+
INSERT(DatabendInsertGenerator::getQuery), DELETE(DatabendDeleteGenerator::generate),
4039
// TODO 等待databend实现update
4140
// UPDATE(DatabendUpdateGenerator::getQuery), //
42-
CREATE_VIEW(DatabendViewGenerator::generate),
43-
EXPLAIN((g) -> {
41+
CREATE_VIEW(DatabendViewGenerator::generate), EXPLAIN((g) -> {
4442
ExpectedErrors errors = new ExpectedErrors();
4543
DatabendErrors.addExpressionErrors(errors);
4644
DatabendErrors.addGroupByErrors(errors);

‎src/sqlancer/databend/DatabendSchema.java‎

Lines changed: 66 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,29 @@
11
package sqlancer.databend;
22

3+
import static sqlancer.databend.DatabendSchema.DatabendDataType.INT;
4+
35
import java.sql.ResultSet;
46
import java.sql.SQLException;
57
import java.sql.Statement;
68
import java.util.ArrayList;
79
import java.util.Collections;
10+
import java.util.HashMap;
811
import java.util.List;
12+
import java.util.Map;
13+
import java.util.stream.Collectors;
914

1015
import sqlancer.IgnoreMeException;
1116
import sqlancer.Randomly;
1217
import sqlancer.SQLConnection;
1318
import sqlancer.common.schema.AbstractRelationalTable;
19+
import sqlancer.common.schema.AbstractRowValue;
1420
import sqlancer.common.schema.AbstractSchema;
1521
import sqlancer.common.schema.AbstractTableColumn;
1622
import sqlancer.common.schema.AbstractTables;
1723
import sqlancer.common.schema.TableIndex;
1824
import sqlancer.databend.DatabendProvider.DatabendGlobalState;
1925
import sqlancer.databend.DatabendSchema.DatabendTable;
26+
import sqlancer.databend.ast.DatabendConstant;
2027

2128
public class DatabendSchema extends AbstractSchema<DatabendGlobalState, DatabendTable> {
2229

@@ -158,6 +165,55 @@ public DatabendTables(List<DatabendTable> tables) {
158165
super(tables);
159166
}
160167

168+
public DatabendRowValue getRandomRowValue(SQLConnection con) throws SQLException {
169+
String rowValueQuery = String.format("SELECT %s FROM %s ORDER BY 1 LIMIT 1", columnNamesAsString(
170+
c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()),
171+
tableNamesAsString());
172+
Map<DatabendColumn, DatabendConstant> values = new HashMap<>();
173+
try (Statement s = con.createStatement()) {
174+
ResultSet rs = s.executeQuery(rowValueQuery);
175+
if (!rs.next()) {
176+
throw new AssertionError("could not find random row " + rowValueQuery + "\n");
177+
}
178+
for (int i = 0; i < getColumns().size(); i++) {
179+
DatabendColumn column = getColumns().get(i);
180+
int columnIndex = rs.findColumn(column.getTable().getName() + column.getName());
181+
assert columnIndex == i + 1;
182+
DatabendConstant constant;
183+
if (rs.getString(columnIndex) == null) {
184+
constant = DatabendConstant.createNullConstant();
185+
} else {
186+
switch (column.getType().getPrimitiveDataType()) {
187+
case INT:
188+
constant = DatabendConstant.createIntConstant(rs.getLong(columnIndex));
189+
break;
190+
case BOOLEAN:
191+
constant = DatabendConstant.createBooleanConstant(rs.getBoolean(columnIndex));
192+
break;
193+
case VARCHAR:
194+
constant = DatabendConstant.createStringConstant(rs.getString(columnIndex));
195+
break;
196+
default:
197+
throw new IgnoreMeException();
198+
}
199+
}
200+
values.put(column, constant);
201+
}
202+
assert !rs.next();
203+
return new DatabendRowValue(this, values);
204+
} catch (SQLException e) {
205+
throw new IgnoreMeException();
206+
}
207+
}
208+
209+
}
210+
211+
public static class DatabendRowValue extends AbstractRowValue<DatabendTables, DatabendColumn, DatabendConstant> {
212+
213+
DatabendRowValue(DatabendTables tables, Map<DatabendColumn, DatabendConstant> values) {
214+
super(tables, values);
215+
}
216+
161217
}
162218

163219
public DatabendSchema(List<DatabendTable> databaseTables) {
@@ -168,6 +224,12 @@ public DatabendTables getRandomTableNonEmptyTables() {
168224
return new DatabendTables(Randomly.nonEmptySubset(getDatabaseTables()));
169225
}
170226

227+
public DatabendTables getRandomTableNonEmptyAndViewTables() {
228+
List<DatabendTable> tables = getDatabaseTables().stream().filter(t -> !t.isView()).collect(Collectors.toList());
229+
tables = Randomly.nonEmptySubset(tables);
230+
return new DatabendTables(tables);
231+
}
232+
171233
private static DatabendCompositeDataType getColumnType(String typeString) {
172234
DatabendDataType primitiveType;
173235
int size = -1;
@@ -176,19 +238,19 @@ private static DatabendCompositeDataType getColumnType(String typeString) {
176238
}
177239
switch (typeString) {
178240
case "INT":
179-
primitiveType = DatabendDataType.INT;
241+
primitiveType = INT;
180242
size = 4;
181243
break;
182244
case "SMALLINT":
183-
primitiveType = DatabendDataType.INT;
245+
primitiveType = INT;
184246
size = 2;
185247
break;
186248
case "BIGINT":
187-
primitiveType = DatabendDataType.INT;
249+
primitiveType = INT;
188250
size = 8;
189251
break;
190252
case "TINYINT":
191-
primitiveType = DatabendDataType.INT;
253+
primitiveType = INT;
192254
size = 1;
193255
break;
194256
case "VARCHAR":

0 commit comments

Comments
 (0)