Skip to content

Commit d271874

Browse files
dutyudutyu
authored
[Refactor](dialect) Add sql dialect converter plugins (#28890)
The current logic for SQL dialect conversion is all in the `fe-core` module, which may lead to the following issues: - Changes to the dialect conversion logic may occur frequently, requiring users to upgrade the Doris version frequently within the fe-core module, leading to a longer change cycle. - The cost of customized development is high, requiring users to replace the fe-core JAR package. Turning it into a plugin can address the above issues properly.
1 parent 1730e00 commit d271874

54 files changed

Lines changed: 1597 additions & 485 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎fe/fe-common/src/main/java/org/apache/doris/common/Config.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2397,6 +2397,9 @@ public class Config extends ConfigBase {
23972397
"Whether to enable the function of getting log files through http interface"})
23982398
public static boolean enable_get_log_file_api = false;
23992399

2400+
// This config is deprecated and has not taken effect anymore,
2401+
// please use dialect plugin: fe_plugins/http-dialect-converter for instead
2402+
@Deprecated
24002403
@ConfField(description = {"用于SQL方言转换的服务地址。",
24012404
"The service address for SQL dialect conversion."})
24022405
public static String sql_convertor_service = "";

‎fe/fe-core/pom.xml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -680,12 +680,6 @@ under the License.
680680
<artifactId>kryo-shaded</artifactId>
681681
</dependency>
682682

683-
<!-- trino-parser -->
684-
<dependency>
685-
<groupId>io.trino</groupId>
686-
<artifactId>trino-parser</artifactId>
687-
</dependency>
688-
689683
<!-- for arrow flight sql -->
690684
<dependency>
691685
<groupId>org.apache.arrow</groupId>

‎fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,7 @@
2727
import org.apache.doris.common.ErrorCode;
2828
import org.apache.doris.common.ErrorReport;
2929
import org.apache.doris.common.UserException;
30-
import org.apache.doris.nereids.parser.ParseDialect;
31-
import org.apache.doris.nereids.parser.spark.SparkSql3LogicalPlanBuilder;
30+
import org.apache.doris.nereids.parser.Dialect;
3231
import org.apache.doris.qe.ConnectContext;
3332
import org.apache.doris.rewrite.ExprRewriter;
3433
import org.apache.doris.thrift.TNullSide;
@@ -50,6 +49,8 @@
5049
public class InlineViewRef extends TableRef {
5150
private static final Logger LOG = LogManager.getLogger(InlineViewRef.class);
5251

52+
private static final String DEFAULT_TABLE_ALIAS_FOR_SPARK_SQL = "__auto_generated_subquery_name";
53+
5354
// Catalog or local view that is referenced.
5455
// Null for inline views parsed directly from a query string.
5556
private final View view;
@@ -198,12 +199,12 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
198199

199200
if (view == null && !hasExplicitAlias()) {
200201
String dialect = ConnectContext.get().getSessionVariable().getSqlDialect();
201-
ParseDialect.Dialect sqlDialect = ParseDialect.Dialect.getByName(dialect);
202-
if (ParseDialect.Dialect.SPARK_SQL != sqlDialect) {
202+
Dialect sqlDialect = Dialect.getByName(dialect);
203+
if (Dialect.SPARK_SQL != sqlDialect) {
203204
ErrorReport.reportAnalysisException(ErrorCode.ERR_DERIVED_MUST_HAVE_ALIAS);
204205
}
205206
hasExplicitAlias = true;
206-
aliases = new String[] { SparkSql3LogicalPlanBuilder.DEFAULT_TABLE_ALIAS };
207+
aliases = new String[] { DEFAULT_TABLE_ALIAS_FOR_SPARK_SQL };
207208
}
208209

209210
// Analyze the inline view query statement with its own analyzer

‎fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/DialectTransformException.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@
1818
package org.apache.doris.nereids.exceptions;
1919

2020
/**
21-
* DialectTransformException when have not supported transforming for the
22-
* {@link io.trino.sql.tree.Node}.
21+
* DialectTransformException when have not supported transforming for dialect converters.
2322
*/
2423
public class DialectTransformException extends UnsupportedOperationException {
2524

‎fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/UnsupportedDialectException.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,16 @@
1717

1818
package org.apache.doris.nereids.exceptions;
1919

20-
import org.apache.doris.nereids.parser.ParseDialect;
20+
import org.apache.doris.nereids.parser.Dialect;
2121

2222
/**
2323
* UnsupportedDialectException when not match any in
24-
* {@link org.apache.doris.nereids.parser.ParseDialect}.
24+
* {@link Dialect}.
2525
*/
2626
public class UnsupportedDialectException extends UnsupportedOperationException {
2727

28-
public UnsupportedDialectException(ParseDialect dialect) {
29-
super(String.format("Unsupported dialect name is %s, version is %s",
30-
dialect.getDialect().getDialectName(), dialect.getVersion().getVersionName()));
28+
public UnsupportedDialectException(Dialect dialect) {
29+
super(String.format("Unsupported dialect name is %s", dialect.getDialectName()));
3130
}
3231

3332
public UnsupportedDialectException(String type, String msg) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ComplexFnCallTransformer.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
package org.apache.doris.nereids.parser;
1919

2020
/**
21-
* Trino complex function transformer
21+
* Complex complex function transformer
2222
*/
2323
public abstract class ComplexFnCallTransformer extends AbstractFnCallTransformer {
2424

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/Dialect.java‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.parser;
19+
20+
import javax.annotation.Nullable;
21+
22+
/**
23+
* ParseDialect enum, maybe support other dialect.
24+
*/
25+
public enum Dialect {
26+
/**
27+
* Doris parser dialect
28+
*/
29+
DORIS("doris"),
30+
/**
31+
* Trino parser dialect
32+
*/
33+
TRINO("trino"),
34+
/**
35+
* Presto parser dialect
36+
*/
37+
PRESTO("presto"),
38+
/**
39+
* Spark sql parser dialect
40+
*/
41+
SPARK_SQL("spark_sql"),
42+
/**
43+
* Hive parser dialect
44+
*/
45+
HIVE("hive"),
46+
/**
47+
* Alibaba max compute parser dialect
48+
*/
49+
MAX_COMPUTE("max_compute"),
50+
/**
51+
* Mysql parser dialect
52+
*/
53+
MYSQL("mysql"),
54+
/**
55+
* Postgresql parser dialect
56+
*/
57+
POSTGRESQL("postgresql"),
58+
/**
59+
* Sqlserver parser dialect
60+
*/
61+
SQLSERVER("sqlserver"),
62+
/**
63+
* Clickhouse parser dialect
64+
*/
65+
CLICKHOUSE("clickhouse"),
66+
/**
67+
* Sap hana parser dialect
68+
*/
69+
SAP_HANA("sap_hana"),
70+
/**
71+
* OceanBase parser dialect
72+
*/
73+
OCEANBASE("oceanbase");
74+
75+
public static final int MAX_DIALECT_SIZE = Dialect.values().length;
76+
77+
private final String dialectName;
78+
79+
Dialect(String dialectName) {
80+
this.dialectName = dialectName;
81+
}
82+
83+
public String getDialectName() {
84+
return dialectName;
85+
}
86+
87+
/**
88+
* Get dialect by name
89+
*/
90+
public static @Nullable Dialect getByName(String dialectName) {
91+
if (dialectName == null) {
92+
return null;
93+
}
94+
for (Dialect dialect : Dialect.values()) {
95+
if (dialect.getDialectName().equals(dialectName.toLowerCase())) {
96+
return dialect;
97+
}
98+
}
99+
return null;
100+
}
101+
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderAssistant.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,7 @@
3131
/**
3232
* Logical plan builder assistant for buildIn dialect and other dialect.
3333
* The same logical in {@link org.apache.doris.nereids.parser.LogicalPlanBuilder}
34-
* and {@link org.apache.doris.nereids.parser.trino.TrinoLogicalPlanBuilder} can be
35-
* extracted to here.
34+
* can be extracted to here.
3635
*/
3736
public class LogicalPlanBuilderAssistant {
3837

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java‎

Lines changed: 25 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -18,20 +18,19 @@
1818
package org.apache.doris.nereids.parser;
1919

2020
import org.apache.doris.analysis.StatementBase;
21-
import org.apache.doris.common.Config;
21+
import org.apache.doris.catalog.Env;
2222
import org.apache.doris.common.Pair;
2323
import org.apache.doris.nereids.DorisLexer;
2424
import org.apache.doris.nereids.DorisParser;
2525
import org.apache.doris.nereids.StatementContext;
2626
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
27-
import org.apache.doris.nereids.parser.spark.SparkSql3LogicalPlanBuilder;
28-
import org.apache.doris.nereids.parser.trino.TrinoParser;
2927
import org.apache.doris.nereids.trees.expressions.Expression;
3028
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
3129
import org.apache.doris.nereids.types.DataType;
30+
import org.apache.doris.plugin.DialectConverterPlugin;
31+
import org.apache.doris.plugin.PluginMgr;
3232
import org.apache.doris.qe.SessionVariable;
3333

34-
import com.google.common.base.Strings;
3534
import com.google.common.collect.Lists;
3635
import org.antlr.v4.runtime.CharStreams;
3736
import org.antlr.v4.runtime.CommonTokenStream;
@@ -67,11 +66,13 @@ public List<StatementBase> parseSQL(String originStr) {
6766
* ParseSQL with dialect.
6867
*/
6968
public List<StatementBase> parseSQL(String sql, SessionVariable sessionVariable) {
70-
@Nullable ParseDialect.Dialect sqlDialect = ParseDialect.Dialect.getByName(sessionVariable.getSqlDialect());
71-
return parseSQLWithDialect(sql, sqlDialect, sessionVariable);
69+
return parseSQLWithDialect(sql, sessionVariable);
7270
}
7371

74-
private List<StatementBase> parseSQL(String originStr, @Nullable LogicalPlanBuilder logicalPlanBuilder) {
72+
/**
73+
* ParseSQL with logicalPlanBuilder.
74+
*/
75+
public List<StatementBase> parseSQL(String originStr, @Nullable LogicalPlanBuilder logicalPlanBuilder) {
7576
List<Pair<LogicalPlan, StatementContext>> logicalPlans = parseMultiple(originStr, logicalPlanBuilder);
7677
List<StatementBase> statementBases = Lists.newArrayList();
7778
for (Pair<LogicalPlan, StatementContext> parsedPlanToContext : logicalPlans) {
@@ -81,26 +82,28 @@ private List<StatementBase> parseSQL(String originStr, @Nullable LogicalPlanBuil
8182
}
8283

8384
private List<StatementBase> parseSQLWithDialect(String sql,
84-
@Nullable ParseDialect.Dialect sqlDialect,
8585
SessionVariable sessionVariable) {
86-
if (!Strings.isNullOrEmpty(Config.sql_convertor_service)) {
87-
// if sql convertor service is enabled, no need to parse sql again by specific dialect.
86+
@Nullable Dialect sqlDialect = Dialect.getByName(sessionVariable.getSqlDialect());
87+
if (sqlDialect == null) {
8888
return parseSQL(sql);
8989
}
90-
switch (sqlDialect) {
91-
case TRINO:
92-
final List<StatementBase> logicalPlans = TrinoParser.parse(sql, sessionVariable);
93-
if (CollectionUtils.isEmpty(logicalPlans)) {
94-
return parseSQL(sql);
95-
}
96-
return logicalPlans;
9790

98-
case SPARK_SQL:
99-
return parseSQL(sql, new SparkSql3LogicalPlanBuilder());
100-
101-
default:
102-
return parseSQL(sql);
91+
PluginMgr pluginMgr = Env.getCurrentEnv().getPluginMgr();
92+
List<DialectConverterPlugin> plugins = pluginMgr.getActiveDialectPluginList(sqlDialect);
93+
for (DialectConverterPlugin plugin : plugins) {
94+
try {
95+
List<StatementBase> statementBases = plugin.parseSqlWithDialect(sql, sessionVariable);
96+
if (CollectionUtils.isNotEmpty(statementBases)) {
97+
return statementBases;
98+
}
99+
} catch (Throwable throwable) {
100+
LOG.warn("Parse sql with dialect {} failed, plugin: {}, sql: {}.",
101+
sqlDialect, plugin.getClass().getSimpleName(), sql, throwable);
102+
}
103103
}
104+
105+
// fallback if any exception occurs before
106+
return parseSQL(sql);
104107
}
105108

106109
/**

0 commit comments

Comments
 (0)