Skip to content

Commit c633402

Browse files
Henry2SSHenry2SS
authored
[feature] (sql-digest) support sql digest (#8919)
1 parent 52a2db1 commit c633402

37 files changed

Lines changed: 735 additions & 5 deletions

‎docs/en/ecosystem/audit-plugin.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ create table doris_audit_tbl__
7272
frontend_ip varchar(32) comment "Frontend ip of executing this statement",
7373
cpu_time_ms bigint comment "Total scan cpu time in millisecond of this query",
7474
sql_hash varchar(50) comment "Hash value for this query",
75+
sql_digest varchar(48) comment "Sql digest for this query",
7576
peak_memory_bytes bigint comment "Peak memory bytes used on all backends of this query",
7677
stmt string comment "The original statement, trimed if longer than 2G"
7778
) engine=OLAP
@@ -97,4 +98,4 @@ The `dynamic_partition` attribute selects the number of days to keep the audit l
9798

9899
After that, connect to Doris and use the `INSTALL PLUGIN` command to complete the installation. After successful installation, you can see the installed plug-ins through `SHOW PLUGINS`, and the status is `INSTALLED`.
99100

100-
Upon completion, the plug-in will continuously import audit date into this table at specified intervals.
101+
Upon completion, the plug-in will continuously import audit date into this table at specified intervals.

‎docs/zh-CN/ecosystem/audit-plugin.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ create table doris_audit_tbl__
7272
frontend_ip varchar(32) comment "Frontend ip of executing this statement",
7373
cpu_time_ms bigint comment "Total scan cpu time in millisecond of this query",
7474
sql_hash varchar(48) comment "Hash value for this query",
75+
sql_digest varchar(48) comment "Sql digest for this query",
7576
peak_memory_bytes bigint comment "Peak memory bytes used on all backends of this query",
7677
stmt string comment "The original statement, trimed if longer than 2G "
7778
) engine=OLAP

‎fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticExpr.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -838,6 +838,36 @@ public String toSqlImpl() {
838838
return sb.toString();
839839
}
840840

841+
@Override
842+
public String toDigestImpl() {
843+
StringBuilder sb = new StringBuilder();
844+
sb.append(fnCall.toDigest()).append(" OVER (");
845+
boolean needsSpace = false;
846+
if (!partitionExprs.isEmpty()) {
847+
sb.append("PARTITION BY ").append(exprListToDigest(partitionExprs));
848+
needsSpace = true;
849+
}
850+
if (!orderByElements.isEmpty()) {
851+
List<String> orderByStrings = Lists.newArrayList();
852+
for (OrderByElement e : orderByElements) {
853+
orderByStrings.add(e.toDigest());
854+
}
855+
if (needsSpace) {
856+
sb.append(" ");
857+
}
858+
sb.append("ORDER BY ").append(Joiner.on(", ").join(orderByStrings));
859+
needsSpace = true;
860+
}
861+
if (window != null) {
862+
if (needsSpace) {
863+
sb.append(" ");
864+
}
865+
sb.append(window.toDigest());
866+
}
867+
sb.append(")");
868+
return sb.toString();
869+
}
870+
841871
private String exprListToSql(List<? extends Expr> exprs) {
842872
if (exprs == null || exprs.isEmpty())
843873
return "";
@@ -847,4 +877,15 @@ private String exprListToSql(List<? extends Expr> exprs) {
847877
}
848878
return Joiner.on(", ").join(strings);
849879
}
880+
881+
private String exprListToDigest(List<? extends Expr> exprs) {
882+
if (exprs == null || exprs.isEmpty()) {
883+
return "";
884+
}
885+
List<String> strings = Lists.newArrayList();
886+
for (Expr expr : exprs) {
887+
strings.add(expr.toDigest());
888+
}
889+
return Joiner.on(", ").join(strings);
890+
}
850891
}

‎fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticWindow.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,17 @@ public String toSql() {
167167
return sb.toString();
168168
}
169169

170+
public String toDigest() {
171+
StringBuilder sb = new StringBuilder();
172+
173+
if (expr != null) {
174+
sb.append(expr.toDigest()).append(" ");
175+
}
176+
177+
sb.append(type.toString());
178+
return sb.toString();
179+
}
180+
170181
public TAnalyticWindowBoundary toThrift(Type windowType) {
171182
TAnalyticWindowBoundary result = new TAnalyticWindowBoundary(type.toThrift());
172183

@@ -296,6 +307,21 @@ public String toSql() {
296307
return sb.toString();
297308
}
298309

310+
public String toDigest() {
311+
StringBuilder sb = new StringBuilder();
312+
sb.append(type_.toString()).append(" ");
313+
314+
if (rightBoundary_ == null) {
315+
sb.append(leftBoundary_.toDigest());
316+
} else {
317+
sb.append("BETWEEN ").append(leftBoundary_.toDigest()).append(" AND ");
318+
sb.append(rightBoundary_.toDigest());
319+
}
320+
321+
return sb.toString();
322+
}
323+
324+
299325
public TAnalyticWindow toThrift() {
300326
TAnalyticWindow result = new TAnalyticWindow(type_.toThrift());
301327

‎fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,15 @@ public String toSqlImpl() {
221221
}
222222
}
223223

224+
@Override
225+
public String toDigestImpl() {
226+
if (children.size() == 1) {
227+
return op.toString() + " " + getChild(0).toDigest();
228+
} else {
229+
return getChild(0).toDigest() + " " + op.toString() + " " + getChild(1).toDigest();
230+
}
231+
}
232+
224233
@Override
225234
protected void toThrift(TExprNode msg) {
226235
msg.node_type = TExprNodeType.ARITHMETIC_EXPR;

‎fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,14 @@ protected String toSqlImpl() {
7272
return "ARRAY(" + StringUtils.join(list, ", ") + ")";
7373
}
7474

75+
@Override
76+
public String toDigestImpl() {
77+
List<String> list = new ArrayList<>(children.size());
78+
children.forEach(v -> list.add(v.toDigestImpl()));
79+
80+
return "ARRAY(" + StringUtils.join(list, ", ") + ")";
81+
}
82+
7583
@Override
7684
public String getStringValue() {
7785
List<String> list = new ArrayList<>(children.size());

‎fe/fe-core/src/main/java/org/apache/doris/analysis/BetweenPredicate.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,14 @@ protected void toThrift(TExprNode msg) {
9696
public String toSqlImpl() {
9797
String notStr = (isNotBetween) ? "NOT " : "";
9898
return children.get(0).toSql() + " " + notStr + "BETWEEN " +
99-
children.get(1).toSql() + " AND " + children.get(2).toSql();
99+
children.get(1).toSql() + " AND " + children.get(2).toSql();
100+
}
101+
102+
@Override
103+
public String toDigestImpl() {
104+
String notStr = (isNotBetween) ? "NOT " : "";
105+
return children.get(0).toDigest() + " " + notStr + "BETWEEN " +
106+
children.get(1).toDigest() + " AND " + children.get(2).toDigest();
100107
}
101108

102109
@Override

‎fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,11 @@ public String toSqlImpl() {
242242
return getChild(0).toSql() + " " + op.toString() + " " + getChild(1).toSql();
243243
}
244244

245+
@Override
246+
public String toDigestImpl() {
247+
return getChild(0).toDigest() + " " + op.toString() + " " + getChild(1).toDigest();
248+
}
249+
245250
@Override
246251
protected void toThrift(TExprNode msg) {
247252
msg.node_type = TExprNodeType.BINARY_PRED;

‎fe/fe-core/src/main/java/org/apache/doris/analysis/CaseExpr.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,24 @@ public String toSqlImpl() {
129129
return output.toString();
130130
}
131131

132+
@Override
133+
public String toDigestImpl() {
134+
StringBuilder sb = new StringBuilder("CASE");
135+
int childIdx = 0;
136+
if (hasCaseExpr) {
137+
sb.append(" ").append(children.get(childIdx++).toDigest());
138+
}
139+
while (childIdx + 2 <= children.size()) {
140+
sb.append(" WHEN ").append(children.get(childIdx++).toDigest());
141+
sb.append(" THEN ").append(children.get(childIdx++).toDigest());
142+
}
143+
if (hasElseExpr) {
144+
sb.append(" ELSE ").append(children.get(children.size() - 1).toDigest());
145+
}
146+
sb.append(" END");
147+
return sb.toString();
148+
}
149+
132150
@Override
133151
public boolean isVectorized() {
134152
return false;

‎fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,27 @@ public String toSqlImpl() {
206206
}
207207
}
208208

209+
@Override
210+
public String toDigestImpl() {
211+
boolean isVerbose = ConnectContext.get() != null &&
212+
ConnectContext.get().getExecutor() != null &&
213+
ConnectContext.get().getExecutor().getParsedStmt() != null &&
214+
ConnectContext.get().getExecutor().getParsedStmt().getExplainOptions() != null &&
215+
ConnectContext.get().getExecutor().getParsedStmt().getExplainOptions().isVerbose();
216+
if (isImplicit && !isVerbose) {
217+
return getChild(0).toDigest();
218+
}
219+
if (isAnalyzed) {
220+
if (type.isStringType()) {
221+
return "CAST(" + getChild(0).toDigest() + " AS " + "CHARACTER" + ")";
222+
} else {
223+
return "CAST(" + getChild(0).toDigest() + " AS " + type.toString() + ")";
224+
}
225+
} else {
226+
return "CAST(" + getChild(0).toDigest() + " AS " + targetTypeDef.toString() + ")";
227+
}
228+
}
229+
209230
@Override
210231
protected void treeToThriftHelper(TExpr container) {
211232
if (noOp) {

0 commit comments

Comments
 (0)