From c0a4578bb02a283707b028720890c916ddd26ce5 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Thu, 7 Dec 2023 12:54:24 -0800 Subject: [PATCH 1/6] [SQL] Parser support for user-defined functions Signed-off-by: Mihai Budiu --- docs/sql/grammar.md | 11 +- .../SQL-compiler/src/main/codegen/config.fmpp | 2 + .../src/main/codegen/includes/ddl.ftl | 41 ++++- .../sqlCompiler/compiler/DBSPCompiler.java | 2 + .../calciteCompiler/CalciteCompiler.java | 166 +++++++----------- .../calciteCompiler/CustomFunctions.java | 124 +++++++++++++ .../SqlCreateFunctionDeclaration.java | 61 +++++++ .../org/dbsp/simulator/SimulatorTests.java | 1 - .../sqlCompiler/compiler/ParserTests.java | 8 + 9 files changed, 307 insertions(+), 109 deletions(-) create mode 100644 sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CustomFunctions.java create mode 100644 sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlCreateFunctionDeclaration.java diff --git a/docs/sql/grammar.md b/docs/sql/grammar.md index a718b15141c..0789bef39c5 100644 --- a/docs/sql/grammar.md +++ b/docs/sql/grammar.md @@ -9,6 +9,13 @@ statementList: statement : createTableStatement | createViewStatement + | createFunctionStatement + +generalType + : type [NOT NULL] + +createFunctionStatement + : CREATE FUNCTION name '(' [ columnDecl [, columnDecl ]* ] ')' generalType createTableStatement : CREATE TABLE name @@ -20,7 +27,7 @@ createViewStatement AS query tableElement - : columnName type [NOT [NULL]] ( columnConstraint )* + : columnName generalType ( columnConstraint )* | columnName | tableConstraint @@ -79,7 +86,7 @@ groupItem: | '(' expression [, expression ]* ')' columnDecl - : column type [ NOT NULL ] + : column generalType selectWithoutFrom : SELECT [ ALL | DISTINCT ] diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/config.fmpp b/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/config.fmpp index 4df83dc516d..cc1854f5ac8 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/config.fmpp +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/config.fmpp @@ -15,6 +15,7 @@ data: { "org.apache.calcite.sql.ddl.SqlDdlNodes" "org.apache.calcite.sql.ddl.SqlCreateType" "org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.SqlExtendedColumnDeclaration" + "org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.SqlCreateFunctionDeclaration" ] # List of new keywords. Example: "DATABASES", "TABLES". If the keyword is @@ -918,6 +919,7 @@ data: { "SqlCreateView" "SqlCreateExtendedTable" "SqlCreateType" + "SqlFunction" ] truncateStatementParserMethods: [ diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/includes/ddl.ftl b/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/includes/ddl.ftl index 11139352743..d96e3a294d8 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/includes/ddl.ftl +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/includes/ddl.ftl @@ -98,7 +98,7 @@ SqlExtendedColumnDeclaration ColumnAttribute(SqlExtendedColumnDeclaration column ) } -SqlNodeList AttributeDefList() : +SqlNodeList NonEmptyAttributeDefList() : { final Span s; final List list = new ArrayList(); @@ -114,6 +114,23 @@ SqlNodeList AttributeDefList() : } } +SqlNodeList AttributeDefList() : +{ + final Span s; + final List list = new ArrayList(); +} +{ + { s = span(); } + ( AttributeDef(list) + ( + AttributeDef(list) + )* + )? + { + return new SqlNodeList(list, s.end(this)); + } +} + void AttributeDef(List list) : { final SqlIdentifier id; @@ -135,6 +152,26 @@ void AttributeDef(List list) : } } +SqlCreateFunctionDeclaration SqlFunction(Span s, boolean replace) : +{ + final boolean ifNotExists; + final SqlIdentifier id; + final SqlNodeList parameters; + final SqlDataTypeSpec type; + final boolean nullable; +} +{ + ifNotExists = IfNotExistsOpt() + id = SimpleIdentifier() + parameters = AttributeDefList() + type = DataType() + nullable = NullableOptDefaultTrue() + { + return new SqlCreateFunctionDeclaration(s.end(this), replace, ifNotExists, + id, parameters, type.withNullable(nullable)); + } +} + SqlCreate SqlCreateType(Span s, boolean replace) : { final SqlIdentifier id; @@ -146,7 +183,7 @@ SqlCreate SqlCreateType(Span s, boolean replace) : id = CompoundIdentifier() ( - attributeDefList = AttributeDefList() + attributeDefList = NonEmptyAttributeDefList() | type = DataType() ) diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java index e1f41e6c931..9b4a6249e5d 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java @@ -223,6 +223,8 @@ private void compileInternal(String statements, boolean many, @Nullable String c .newline(); FrontEndStatement fe = this.frontend.compile( node.toString(), node, comment, this.inputTables, this.outputViews); + if (fe == null) + continue; this.midend.compile(fe); } } catch (SqlParseException e) { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteCompiler.java index 191052b196b..425ac052b4a 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteCompiler.java @@ -44,18 +44,43 @@ import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.rules.CoreRules; import org.apache.calcite.rel.rules.PruneEmptyRules; -import org.apache.calcite.rel.type.*; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rel.type.RelDataTypeFieldImpl; +import org.apache.calcite.rel.type.RelDataTypeSystem; +import org.apache.calcite.rel.type.RelDataTypeSystemImpl; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; -import org.apache.calcite.sql.*; -import org.apache.calcite.sql.ddl.*; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlBasicTypeNameSpec; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlDataTypeSpec; +import org.apache.calcite.sql.SqlExplainFormat; +import org.apache.calcite.sql.SqlExplainLevel; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlInsert; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlOperatorTable; +import org.apache.calcite.sql.SqlTypeNameSpec; +import org.apache.calcite.sql.SqlUnresolvedFunction; +import org.apache.calcite.sql.ddl.SqlColumnDeclaration; +import org.apache.calcite.sql.ddl.SqlCreateTable; +import org.apache.calcite.sql.ddl.SqlCreateView; +import org.apache.calcite.sql.ddl.SqlDropTable; +import org.apache.calcite.sql.ddl.SqlKeyConstraint; import org.apache.calcite.sql.fun.SqlLibrary; import org.apache.calcite.sql.fun.SqlLibraryOperatorTableFactory; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.calcite.sql.type.*; +import org.apache.calcite.sql.type.SqlTypeFactoryImpl; +import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.util.SqlOperatorTables; import org.apache.calcite.sql.util.SqlShuttle; import org.apache.calcite.sql.validate.SqlConformanceEnum; @@ -70,9 +95,17 @@ import org.dbsp.sqlCompiler.compiler.IErrorReporter; import org.dbsp.sqlCompiler.compiler.InputTableDescription; import org.dbsp.sqlCompiler.compiler.OutputViewDescription; -import org.dbsp.sqlCompiler.compiler.errors.*; +import org.dbsp.sqlCompiler.compiler.errors.CompilationError; +import org.dbsp.sqlCompiler.compiler.errors.InternalCompilerError; +import org.dbsp.sqlCompiler.compiler.errors.SourcePositionRange; +import org.dbsp.sqlCompiler.compiler.errors.UnimplementedException; +import org.dbsp.sqlCompiler.compiler.errors.UnsupportedException; import org.dbsp.sqlCompiler.compiler.frontend.CalciteObject; -import org.dbsp.sqlCompiler.compiler.frontend.statements.*; +import org.dbsp.sqlCompiler.compiler.frontend.statements.CreateTableStatement; +import org.dbsp.sqlCompiler.compiler.frontend.statements.CreateViewStatement; +import org.dbsp.sqlCompiler.compiler.frontend.statements.DropTableStatement; +import org.dbsp.sqlCompiler.compiler.frontend.statements.FrontEndStatement; +import org.dbsp.sqlCompiler.compiler.frontend.statements.TableModifyStatement; import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeDecimal; import org.dbsp.util.IWritesLogs; import org.dbsp.util.Linq; @@ -80,10 +113,14 @@ import org.dbsp.util.Utilities; import javax.annotation.Nullable; -import java.util.*; - -import static org.apache.calcite.sql.type.OperandTypes.family; -import static org.apache.calcite.sql.type.ReturnTypes.ARG1; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; /** * The calcite compiler compiles SQL into Calcite RelNode representations. @@ -121,8 +158,7 @@ public class CalciteCompiler implements IWritesLogs { * This class rewrites instances of the division operator in the SQL AST * into calls to a user-defined function DIVISION. We do this * because we don't like how Calcite infers result types for division, - * and we want to supply our own rules. - */ + * and we want to supply our own rules. */ public static class RewriteDivision extends SqlShuttle { @Override public SqlNode visit(SqlCall call) { @@ -146,88 +182,10 @@ public SqlNode visit(SqlCall call) { } } - static class SqlDivideFunction extends SqlFunction { - // Custom implementation of type inference DIVISION for our division operator. - static final SqlReturnTypeInference divResultInference = new SqlReturnTypeInference() { - @Override - public @org.checkerframework.checker.nullness.qual.Nullable - RelDataType inferReturnType(SqlOperatorBinding opBinding) { - // Default policy for division. - RelDataType result = ReturnTypes.QUOTIENT_NULLABLE.inferReturnType(opBinding); - List opTypes = opBinding.collectOperandTypes(); - // If all operands are integer or decimal, result is nullable - // otherwise it's not. - boolean nullable = true; - for (RelDataType type: opTypes) { - if (SqlTypeName.APPROX_TYPES.contains(type.getSqlTypeName())) { - nullable = false; - break; - } - } - if (nullable) - result = opBinding.getTypeFactory().createTypeWithNullability(result, true); - return result; - } - }; - - public SqlDivideFunction() { - super("DIVISION", - SqlKind.OTHER_FUNCTION, - divResultInference, - null, - OperandTypes.NUMERIC_NUMERIC, - SqlFunctionCategory.NUMERIC); - } - - @Override - public boolean isDeterministic() { - // TODO: change this when we learn how to constant-fold in the RexToLixTranslator - // https://issues.apache.org/jira/browse/CALCITE-3394 may give a solution - return false; - } - } - public void generateOutputForNextView(boolean generate) { this.generateOutputForNextView = generate; } - /** - * WRITELOG(format, arg) returns its argument 'arg' unchanged but also logs - * its value to stdout. Used for debugging. In the format string - * each occurrence of %% is replaced with the arg */ - public static class WriteLogFunction extends SqlFunction { - public WriteLogFunction() { - super("WRITELOG", - SqlKind.OTHER_FUNCTION, - ARG1, - null, - family(SqlTypeFamily.CHARACTER, SqlTypeFamily.ANY), - SqlFunctionCategory.USER_DEFINED_FUNCTION); - } - - @Override - public boolean isDeterministic() { - return false; - } - } - - static class RlikeFunction extends SqlFunction { - public RlikeFunction() { - super("RLIKE", - SqlKind.RLIKE, - ReturnTypes.BOOLEAN, - null, - OperandTypes.STRING_STRING, - SqlFunctionCategory.STRING); - } - - @Override - public boolean isDeterministic() { - // TODO: change this when we learn how to constant-fold in the RexToLixTranslator - return false; - } - } - public static final RelDataTypeSystem TYPE_SYSTEM = new RelDataTypeSystemImpl() { @Override public int getMaxNumericPrecision() { @@ -331,16 +289,15 @@ public CalciteCompiler(CompilerOptions options, IErrorReporter errorReporter) { rootSchema, Collections.singletonList(catalog.schemaName), this.typeFactory, connectionConfig); SqlOperatorTable operatorTable = SqlOperatorTables.chain( - // Libraries of user-defined functions supported. SqlLibraryOperatorTableFactory.INSTANCE.getOperatorTable( - // Standard SQL functions + // Libraries of functions supported. EnumSet.of(SqlLibrary.STANDARD, SqlLibrary.MYSQL, SqlLibrary.POSTGRESQL, SqlLibrary.BIG_QUERY, SqlLibrary.SPARK, SqlLibrary.SPATIAL)), - SqlOperatorTables.of(new SqlDivideFunction(), new RlikeFunction(), new WriteLogFunction()) + SqlOperatorTables.of(CustomFunctions.INSTANCE.getUDFs()) ); SqlValidator.Config validatorConfig = SqlValidator.Config.DEFAULT @@ -523,8 +480,7 @@ SqlParser createSqlParser(String sql) { String toParse = newlines + sql; SqlParser sqlParser = SqlParser.create(toParse, this.parserConfig); int lines = sql.split("\n").length; - for (int i = 0; i < lines; i++) - this.newlines.append("\n"); + this.newlines.append("\n".repeat(lines)); return sqlParser; } @@ -729,8 +685,8 @@ List createTableColumnsMetadata(SqlNodeList list) { return result; } - public List createColumnsMetadata( - CalciteObject node, SqlIdentifier objectName, boolean view, RelRoot relRoot, @Nullable SqlNodeList columnNames) { + public List createColumnsMetadata(CalciteObject node, + SqlIdentifier objectName, boolean view, RelRoot relRoot, @Nullable SqlNodeList columnNames) { List columns = new ArrayList<>(); RelDataType rowType = relRoot.rel.getRowType(); if (columnNames != null && columnNames.size() != relRoot.fields.size()) { @@ -786,13 +742,14 @@ public List createColumnsMetadata( } /** - * Compile a SQL statement. Return a description. + * Compile a SQL statement. Returns null if the statement does not require further processing. * @param node Compiled version of the SQL statement. * @param sqlStatement SQL statement as a string to compile. * @param comment Additional information about the compiled statement. * @param inputs If not null, add here a JSON description of the tables defined by the statement, if any. * @param outputs If not null, add here a JSON description of the views defined by the statement, if any. */ + @Nullable public FrontEndStatement compile( String sqlStatement, SqlNode node, @@ -829,16 +786,14 @@ public FrontEndStatement compile( .newline(); RelRoot relRoot = this.converter.convertQuery(ct.query, true, true); cols = this.createColumnsMetadata( - CalciteObject.create(ct), ct.name, false, relRoot, null); + ct.name, false, relRoot, null); */ } CreateTableStatement table = new CreateTableStatement(node, sqlStatement, tableName, comment, cols); this.catalog.addTable(tableName, table.getEmulatedTable()); inputs.add(new InputTableDescription(table)); return table; - } - - if (node.getKind().equals(SqlKind.CREATE_VIEW)) { + } else if (node.getKind().equals(SqlKind.CREATE_VIEW)) { SqlCreateView cv = (SqlCreateView) node; SqlNode query = cv.query; if (cv.getReplace()) @@ -851,8 +806,8 @@ public FrontEndStatement compile( .append(Objects.requireNonNull(query).toString()) .newline(); RelRoot relRoot = this.converter.convertQuery(query, true, true); - List columns = this.createColumnsMetadata( - CalciteObject.create(cv), cv.name, true, relRoot, cv.columnList); + List columns = this.createColumnsMetadata(CalciteObject.create(node), + cv.name, true, relRoot, cv.columnList); RelNode optimized = this.optimize(relRoot.rel); relRoot = relRoot.withRel(optimized); String viewName = Catalog.identifierToString(cv.name); @@ -864,6 +819,9 @@ public FrontEndStatement compile( if (this.generateOutputForNextView) outputs.add(new OutputViewDescription(view)); return view; + } else if (node.getKind().equals(SqlKind.CREATE_FUNCTION)) { + SqlCreateFunctionDeclaration decl = (SqlCreateFunctionDeclaration) node; + return null; } } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CustomFunctions.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CustomFunctions.java new file mode 100644 index 00000000000..4301642bf54 --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CustomFunctions.java @@ -0,0 +1,124 @@ +package org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperatorBinding; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeFamily; +import org.apache.calcite.sql.type.SqlTypeName; +import org.dbsp.sqlCompiler.compiler.errors.UnimplementedException; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.calcite.sql.type.OperandTypes.family; +import static org.apache.calcite.sql.type.ReturnTypes.ARG1; + +/** Several functions that we define and add to the existing ones. */ +public class CustomFunctions { + public static final CustomFunctions INSTANCE = new CustomFunctions(); + private final List functions = new ArrayList<>(); + + private CustomFunctions() { + this.functions.add(new SqlDivisionFunction()); + this.functions.add(new RlikeFunction()); + this.functions.add(new WriteLogFunction()); + } + + static class RlikeFunction extends SqlFunction { + public RlikeFunction() { + super("RLIKE", + SqlKind.RLIKE, + ReturnTypes.BOOLEAN, + null, + OperandTypes.STRING_STRING, + SqlFunctionCategory.STRING); + } + + @Override + public boolean isDeterministic() { + // TODO: change this when we learn how to constant-fold in the RexToLixTranslator + return false; + } + } + + /** + * WRITELOG(format, arg) returns its argument 'arg' unchanged but also logs + * its value to stdout. Used for debugging. In the format string + * each occurrence of %% is replaced with the arg */ + public static class WriteLogFunction extends SqlFunction { + public WriteLogFunction() { + super("WRITELOG", + SqlKind.OTHER_FUNCTION, + ARG1, + null, + family(SqlTypeFamily.CHARACTER, SqlTypeFamily.ANY), + SqlFunctionCategory.USER_DEFINED_FUNCTION); + } + + @Override + public boolean isDeterministic() { + return false; + } + } + + static class SqlDivisionFunction extends SqlFunction { + // Custom implementation of type inference DIVISION for our division operator. + static final SqlReturnTypeInference divResultInference = new SqlReturnTypeInference() { + @Override + public @org.checkerframework.checker.nullness.qual.Nullable + RelDataType inferReturnType(SqlOperatorBinding opBinding) { + // Default policy for division. + RelDataType result = ReturnTypes.QUOTIENT_NULLABLE.inferReturnType(opBinding); + List opTypes = opBinding.collectOperandTypes(); + // If all operands are integer or decimal, result is nullable + // otherwise it's not. + boolean nullable = true; + for (RelDataType type: opTypes) { + if (SqlTypeName.APPROX_TYPES.contains(type.getSqlTypeName())) { + nullable = false; + break; + } + } + if (nullable) + result = opBinding.getTypeFactory().createTypeWithNullability(result, true); + return result; + } + }; + + public SqlDivisionFunction() { + super("DIVISION", + SqlKind.OTHER_FUNCTION, + divResultInference, + null, + OperandTypes.NUMERIC_NUMERIC, + SqlFunctionCategory.NUMERIC); + } + + @Override + public boolean isDeterministic() { + // TODO: change this when we learn how to constant-fold in the RexToLixTranslator + // https://issues.apache.org/jira/browse/CALCITE-3394 may give a solution + return false; + } + } + + /** + * Create a new user-defined function with a specified signature. + * @param name Function name. + * @param signature Description of arguments as a struct. + * @param returnType Return type of function. + */ + public SqlFunction createUDF(String name, RelDataType signature, RelDataType returnType) { + throw new UnimplementedException(); + } + + /** Return the list of user-defined functions */ + public List getUDFs() { + return this.functions; + } +} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlCreateFunctionDeclaration.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlCreateFunctionDeclaration.java new file mode 100644 index 00000000000..f58e3792611 --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlCreateFunctionDeclaration.java @@ -0,0 +1,61 @@ +package org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler; + +import org.apache.calcite.sql.SqlCreate; +import org.apache.calcite.sql.SqlDataTypeSpec; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlSpecialOperator; +import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.parser.SqlParserPos; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +/** Our own version of CREATE FUNCTION, different from Calcite. + * We only expect a signature of the function. */ +public class SqlCreateFunctionDeclaration extends SqlCreate { + private final SqlIdentifier name; + private final SqlNodeList parameters; + private final SqlDataTypeSpec returnType; + + private static final SqlSpecialOperator OPERATOR = + new SqlSpecialOperator("CREATE FUNCTION", SqlKind.CREATE_FUNCTION); + + public SqlCreateFunctionDeclaration(SqlParserPos pos, boolean replace, + boolean ifNotExists, SqlIdentifier name, + SqlNodeList parameters, SqlDataTypeSpec returnType) { + super(OPERATOR, pos, replace, ifNotExists); + this.name = Objects.requireNonNull(name, "name"); + this.parameters = Objects.requireNonNull(parameters, "parameters"); + this.returnType = returnType; + } + + @Override public void unparse(SqlWriter writer, int leftPrec, + int rightPrec) { + writer.keyword(getReplace() ? "CREATE OR REPLACE" : "CREATE"); + writer.keyword("FUNCTION"); + if (ifNotExists) { + writer.keyword("IF NOT EXISTS"); + } + name.unparse(writer, 0, 0); + final SqlWriter.Frame frame = writer.startList(SqlWriter.FrameTypeEnum.SIMPLE); + for (SqlNode parameter : this.parameters) { + writer.sep(","); + parameter.unparse(writer, 0, 0); + } + writer.endList(frame); + this.returnType.unparse(writer, 0, 0); + } + + @Override public SqlOperator getOperator() { + return OPERATOR; + } + + @Override public List getOperandList() { + return Arrays.asList(this.name, this.parameters); + } +} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/simulator/SimulatorTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/simulator/SimulatorTests.java index e5665ef1ade..250fd5b2e66 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/simulator/SimulatorTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/simulator/SimulatorTests.java @@ -325,7 +325,6 @@ public void testMultiply() { ZSet input = getPersons(); ZSet address = getAddress(); ZSet product = input.multiply(address, (p, a) -> new NameAddress(p.name, a.city)); - System.out.println(product); Assert.assertEquals(input.entryCount() * address.entryCount(), product.entryCount()); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ParserTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ParserTests.java index 0cf3e432504..1ce9e40eca7 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ParserTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ParserTests.java @@ -60,6 +60,14 @@ public void DDLTest() throws SqlParseException { Assert.assertNotNull(node); } + @Test + public void createFunctionTest() throws SqlParseException { + CalciteCompiler calcite = this.getCompiler(); + String ddl = "CREATE FUNCTION json(data VARCHAR) VARBINARY"; + SqlNode node = calcite.parse(ddl); + Assert.assertNotNull(node); + } + @Test public void createTypeTest() throws SqlParseException { CalciteCompiler calcite = this.getCompiler(); From 537d28838eb447e3cc2e378c24e170ada44b5959 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 11 Dec 2023 00:11:18 -0800 Subject: [PATCH 2/6] [SQL] Document user-defined functions Signed-off-by: Mihai Budiu --- docs/contributors/compiler.md | 3 + docs/sidebars.js | 1 + docs/sql/grammar.md | 17 ++- docs/sql/udf.md | 124 ++++++++++++++++ .../SQL-compiler/src/main/codegen/config.fmpp | 11 +- .../src/main/codegen/includes/ddl.ftl | 3 +- .../src/main/codegen/includes/parserImpls.ftl | 9 -- .../org/dbsp/sqlCompiler/CompilerMain.java | 27 +++- .../sqlCompiler/compiler/CompilerOptions.java | 5 +- .../sqlCompiler/compiler/DBSPCompiler.java | 54 ++++++- .../compiler/backend/rust/RustFileWriter.java | 12 +- .../compiler/frontend/ExpressionCompiler.java | 114 +++++++++++++-- .../calciteCompiler/CalciteCompiler.java | 138 ++++++++++++------ .../frontend/calciteCompiler/Catalog.java | 12 ++ .../calciteCompiler/CustomFunctions.java | 133 +++++++++++++++-- .../SqlCreateFunctionDeclaration.java | 12 ++ .../compiler/visitors/inner/InnerVisitor.java | 8 + .../expression/DBSPConstructorExpression.java | 4 +- .../ir/expression/DBSPExpression.java | 7 + .../ir/expression/literal/DBSPU32Literal.java | 3 + .../sqlCompiler/ir/type/DBSPTypeCode.java | 1 + .../sqlCompiler/ir/type/DBSPTypeResult.java | 59 ++++++++ .../sqlCompiler/ir/type/DBSPTypeUser.java | 4 +- .../dbsp/sqlCompiler/ir/type/DBSPTypeVec.java | 4 +- .../compiler/sql/BaseSQLTests.java | 15 +- .../compiler/{ => sql}/DBSPCompilerTests.java | 4 +- .../{ => sql}/NegativeParserTests.java | 5 +- .../compiler/{ => sql}/OtherTests.java | 86 +++++++++-- .../compiler/{ => sql}/ParserTests.java | 17 ++- .../compiler/sql/functions/FunctionsTest.java | 6 + .../lib/sqllib/src/geopoint.rs | 7 + sql-to-dbsp-compiler/lib/sqllib/src/lib.rs | 19 ++- sql-to-dbsp-compiler/lib/sqllib/src/source.rs | 56 +++++++ sql-to-dbsp-compiler/lib/sqllib/src/types.rs | 9 ++ 34 files changed, 845 insertions(+), 144 deletions(-) create mode 100644 docs/sql/udf.md create mode 100644 sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/DBSPTypeResult.java rename sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/{ => sql}/DBSPCompilerTests.java (96%) rename sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/{ => sql}/NegativeParserTests.java (98%) rename sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/{ => sql}/OtherTests.java (88%) rename sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/{ => sql}/ParserTests.java (93%) create mode 100644 sql-to-dbsp-compiler/lib/sqllib/src/source.rs create mode 100644 sql-to-dbsp-compiler/lib/sqllib/src/types.rs diff --git a/docs/contributors/compiler.md b/docs/contributors/compiler.md index 2be7c88ede3..8907786bb17 100644 --- a/docs/contributors/compiler.md +++ b/docs/contributors/compiler.md @@ -56,6 +56,9 @@ Usage: sql-to-dbsp [options] Input file to compile --ignoreOrder Ignore ORDER BY clauses at the end Default: false + --udf + Specify a Rust file containing implementations of user-defined functions + Default: -O Optimization level (0, 1, or 2) Default: 2 diff --git a/docs/sidebars.js b/docs/sidebars.js index f9ef38848d2..39778dc5aff 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -108,6 +108,7 @@ const sidebars = { 'sql/binary', 'sql/array', 'sql/datetime' + 'sql/udf' ] }, { diff --git a/docs/sql/grammar.md b/docs/sql/grammar.md index 0789bef39c5..846e522aa3e 100644 --- a/docs/sql/grammar.md +++ b/docs/sql/grammar.md @@ -1,6 +1,14 @@ # SQL Grammar -This is a short formal description of the grammar supported: +This is a short formal description of the grammar supported in a BNF +form. + +- Constructs enclosed between `[]` are optional. +- `*` denotes zero or many repetitions. +- Uppercase words (`FUNCTION`) and single-quoted text (`')'`) indicate + grammar terminals. +- Parentheses `()` are used for grouping productions together. +- The vertical bar `|` indicates alternation. ``` statementList: @@ -15,7 +23,7 @@ generalType : type [NOT NULL] createFunctionStatement - : CREATE FUNCTION name '(' [ columnDecl [, columnDecl ]* ] ')' generalType + : CREATE FUNCTION name '(' [ columnDecl [, columnDecl ]* ] ')' RETURNS generalType createTableStatement : CREATE TABLE name @@ -112,7 +120,7 @@ tableExpression joinCondition : ON booleanExpression - | USING '(' column [, column ]* ')' + | USING '(' column [, column ]* ')' tableReference : tablePrimary [ pivot ] [ [ AS ] alias [ '(' columnAlias [, columnAlias ]* ')' ] ] @@ -149,6 +157,9 @@ and is currently ignored. In `orderItem`, if expression is a positive integer n, it denotes the nth item in the `SELECT` clause. +SQL `CREATE FUNCTION` can be used to declare [user-defined +functions](udf.md). + An aggregate query is a query that contains a `GROUP BY` or a `HAVING` clause, or aggregate functions in the `SELECT` clause. In the `SELECT`, `HAVING` and `ORDER` BY clauses of an aggregate query, all diff --git a/docs/sql/udf.md b/docs/sql/udf.md new file mode 100644 index 00000000000..8ceefb5b50c --- /dev/null +++ b/docs/sql/udf.md @@ -0,0 +1,124 @@ +# User-defined functions + +The SQL statement `CREATE FUNCTION` can be used to declare new +functions whose implementation is provided separately in Rust. + +The implementation of these functions is provided to the compiler in a +separate file, whose name is specified using the `--udf` flag. + +The following example shows a user-defined function: + +```sql +CREATE FUNCTION contains_number(str VARCHAR NOT NULL, value INTEGER) RETURNS BOOLEAN NOT NULL +CREATE VIEW V0 AS SELECT contains_number(CAST('YES: 10 NO:5' AS VARCHAR), 5) +``` + +The function name capitalization obeys the same rules as table and +view names: the names are converted by default to all-capitals, but if +the name is quoted capitalization is unchanged. This is important, +because the user must provide a rust implementation that matches the +canonical function name. Here is a possible implementation of the +function `contains_number` above: + +```rs +use sqllib::types::*; + +pub fn CONTAINS_NUMBER(pos: &SourcePositionRange, str: String, value: Option) -> + Result> { + match value { + None => Err(format!(\"{}: null value\", pos).into()), + Some(value) => Ok(str.contains(&format!(\"{}\", value).to_string())), + } +} +``` + +The `use sqllib::types::*` directive imports the definitions of the +standard Rust types that the compiler uses to implement SQL datatypes. +The next section explains what these types are. + +Notice the function implemented has an all-capitals name (which is not +a standard convention for Rust), dictated by the default SQL +capitalization rules. + +There is no type casting or type inference performed for the function +arguments in the SQL program. For example, a call such as +`CONTAINS_NUMBER('2010-10-20', '5')` will fail at SQL compilation time +because the first argument has type `CHAR(8)` instead of `VARCHAR`, +and the second argument has type `CHAR(1)` instead of `INTEGER`. + +Clearly, user-defined functions can pose security problems, since the +Rust implementation is only verified by the Rust compiler. Such +functions are expected to have no side-effects, to be deterministic, +and not to crash. + +## Type representation in Rust + +The following table shows the Rust representation of standard SQL data +types. A nullable SQL type is represented by the corresponding rust +`Option<>` type. Notice that some of these types are not standard +Rust types, but are defined in the DBSP runtime library. + +SQL | Rust +----------- +BOOLEAN | bool +TINYINT | i8 +SMALLINT | i16 +INT | i32 +BIGINT | i64 +DECIMAL(p, s) | Decimal +REAL | F32 +DOUBLE | F64 +CHAR(n) | String +VARCHAR, VARCHAR(n) | String +BINARY, BINARY(n) | ByteArray +NULL | () +INTERVAL | ShortInterval, LongInterval +TIME | Time +TIMESTAMP | Timestamp +DATE | Date +T ARRAY | Vec + +Multiple SQL types may be represented by the same Rust type. For +example, `CHAR`, `CHAR(n)`, `VARCHAR(n)`, and `VARCHAR` are all +represented by the standard Rust `String` type. + +The SQL family of `INTERVAL` types translates to one of two Rust +types: `ShortInterval` (representing intervals from days to seconds), +and `LongInterval` (representing intervals from years to months). +(Our dialect of SQL does allow mixing the two kinds of intervals in a +single expression.) + +In the Rust implementation the function always has to return the type +`Result>`, where `T` is the Rust +equivalent expected return type of the SQL function. The Rust +function should return an `Err` only when the function fails at +runtime; in this case the returned error can use the source position +information to indicate where the error has originated in the code. + +## Source position information + +The first argument passed to the Rust function is always `pos: +&SourcePositionRange`. This argument indicates where in the source +position is placed the code that generated the call to this +user-defined function. This information can be used to generate +better runtime error messages when the user-defined function +encounters an error. (Note: currently Calcite does not provide any +source position information, but we hope to remedy this state of +affairs soon.) + +## Limitations + +There can be only one function with each name. + +Functions cannot have names identical to standard SQL library function +names. + +Polymorphic functions are not supported. For example, in SQL the +addition operation operates on any numeric types; such an operation +cannot be implemented as a single user-defined function. + +The current type checker is very strict, and it requires the function +arguments to have exactly the specified types. No casts are inserted +by the compiler. That is why the previous example requires casting +the string `'YES: 10 NO: 5'` to `VARCHAR`. + diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/config.fmpp b/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/config.fmpp index cc1854f5ac8..a43c82799d2 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/config.fmpp +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/config.fmpp @@ -29,7 +29,7 @@ data: { "SEMI" "SEQUENCES" "TEMP" - "VOLATILE" + #"VOLATILE" ] # List of keywords from "keywords" section that are not reserved. @@ -884,13 +884,6 @@ data: { # List of methods for parsing custom SQL statements. # Return type of method implementation should be 'SqlNode'. statementParserMethods: [ - # The following are not yet released - # "PostgresqlSqlShow()", - # "PostgresqlSqlSetOption()", - # "PostgresqlSqlBegin()", - # "PostgresqlSqlDiscard()", - # "PostgresqlSqlCommit()", - # "PostgresqlSqlRollback()" ] # List of methods for parsing custom literals. @@ -919,7 +912,7 @@ data: { "SqlCreateView" "SqlCreateExtendedTable" "SqlCreateType" - "SqlFunction" + "SqlCreateFunction" ] truncateStatementParserMethods: [ diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/includes/ddl.ftl b/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/includes/ddl.ftl index d96e3a294d8..515cf6b5ab6 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/includes/ddl.ftl +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/includes/ddl.ftl @@ -152,7 +152,7 @@ void AttributeDef(List list) : } } -SqlCreateFunctionDeclaration SqlFunction(Span s, boolean replace) : +SqlCreateFunctionDeclaration SqlCreateFunction(Span s, boolean replace) : { final boolean ifNotExists; final SqlIdentifier id; @@ -164,6 +164,7 @@ SqlCreateFunctionDeclaration SqlFunction(Span s, boolean replace) : ifNotExists = IfNotExistsOpt() id = SimpleIdentifier() parameters = AttributeDefList() + type = DataType() nullable = NullableOptDefaultTrue() { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/includes/parserImpls.ftl b/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/includes/parserImpls.ftl index 5128bcb6522..2db30b6ec2a 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/includes/parserImpls.ftl +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/includes/parserImpls.ftl @@ -73,15 +73,6 @@ TableCollectionType TableCollectionTypeOpt() : { return TableCollectionType.UNSPECIFIED; } } -boolean VolatileOpt() : -{ -} -{ - { return true; } -| - { return false; } -} - /* Extra operators */ TOKEN : diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/CompilerMain.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/CompilerMain.java index 84594b89759..dbdfd33640d 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/CompilerMain.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/CompilerMain.java @@ -36,11 +36,13 @@ import org.dbsp.util.Logger; import javax.annotation.Nullable; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.util.Map; /** @@ -90,7 +92,7 @@ void parseOptions(String[] argv) { PrintStream getOutputStream() throws IOException { PrintStream outputStream; @Nullable String outputFile = this.options.ioOptions.outputFile; - if (outputFile == null) { + if (outputFile.isEmpty()) { outputStream = System.out; } else { outputStream = new PrintStream(Files.newOutputStream(Paths.get(outputFile))); @@ -145,7 +147,7 @@ CompilerMessages run() { : this.options.ioOptions.emitPng ? "png" : null); if (dotFormat != null) { - if (this.options.ioOptions.outputFile == null) { + if (this.options.ioOptions.outputFile.isEmpty()) { compiler.reportError(SourcePositionRange.INVALID, false, "Invalid output", "Must specify an output file when outputting jpeg or png"); return compiler.messages; @@ -166,6 +168,27 @@ CompilerMessages run() { false, "Error writing to file", e.getMessage()); return compiler.messages; } + + try { + if (!this.options.ioOptions.udfs.isEmpty()) { + String outputFileName = this.options.ioOptions.outputFile; + if (outputFileName.isEmpty()) { + compiler.reportError(SourcePositionRange.INVALID, false, + "No output file", "`-udf` option requires specifying an output file"); + return compiler.messages; + } + File outputFile = new File(outputFileName); + File outputDirectory = outputFile.getParentFile(); + File source = new File(this.options.ioOptions.udfs); + File destination = new File(outputDirectory, DBSPCompiler.UDF_FILE_NAME); + Files.copy(source.toPath(), destination.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + } catch (IOException e) { + compiler.reportError(SourcePositionRange.INVALID, + false, "Error copying UDF file", e.getMessage()); + return compiler.messages; + } + return compiler.messages; } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/CompilerOptions.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/CompilerOptions.java index cbf3ef0e984..ccf1bc8a590 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/CompilerOptions.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/CompilerOptions.java @@ -99,8 +99,9 @@ public static class IO { description = "Specify logging level for a class (can be repeated)") public Map loggingLevel = new HashMap<>(); @Parameter(names="-o", description = "Output file; stdout if null") - @Nullable - public String outputFile = null; + public String outputFile = ""; + @Parameter(names = "--udf", description = "Specify a Rust file containing implementations of user-defined functions") + public String udfs = ""; @Parameter(names = "-jpg", description = "Emit a jpg image of the circuit instead of Rust") public boolean emitJpeg = false; @Parameter(names = "-png", description = "Emit a png image of the circuit instead of Rust") diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java index 9b4a6249e5d..86e7fea439e 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java @@ -27,30 +27,35 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.calcite.runtime.CalciteContextException; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlOperatorTable; import org.apache.calcite.sql.parser.SqlParseException; -import org.dbsp.sqlCompiler.circuit.DBSPPartialCircuit; +import org.apache.calcite.sql.util.SqlOperatorTables; import org.dbsp.sqlCompiler.circuit.DBSPCircuit; +import org.dbsp.sqlCompiler.circuit.DBSPPartialCircuit; import org.dbsp.sqlCompiler.compiler.errors.BaseCompilerException; import org.dbsp.sqlCompiler.compiler.errors.CompilationError; import org.dbsp.sqlCompiler.compiler.errors.CompilerMessages; import org.dbsp.sqlCompiler.compiler.errors.SourceFileContents; import org.dbsp.sqlCompiler.compiler.errors.SourcePositionRange; +import org.dbsp.sqlCompiler.compiler.errors.UnsupportedException; import org.dbsp.sqlCompiler.compiler.frontend.CalciteObject; +import org.dbsp.sqlCompiler.compiler.frontend.CalciteToDBSPCompiler; +import org.dbsp.sqlCompiler.compiler.frontend.TableContents; import org.dbsp.sqlCompiler.compiler.frontend.TypeCompiler; -import org.dbsp.sqlCompiler.compiler.visitors.outer.CircuitOptimizer; import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.CalciteCompiler; +import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.CustomFunctions; import org.dbsp.sqlCompiler.compiler.frontend.statements.FrontEndStatement; -import org.dbsp.sqlCompiler.compiler.frontend.CalciteToDBSPCompiler; -import org.dbsp.sqlCompiler.compiler.frontend.TableContents; +import org.dbsp.sqlCompiler.compiler.visitors.outer.CircuitOptimizer; import org.dbsp.sqlCompiler.ir.expression.DBSPVariablePath; import org.dbsp.sqlCompiler.ir.type.DBSPType; import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeInteger; import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeWeight; import org.dbsp.util.IWritesLogs; import org.dbsp.util.Logger; -import org.dbsp.sqlCompiler.compiler.errors.UnsupportedException; import javax.annotation.Nullable; import java.io.IOException; @@ -73,6 +78,10 @@ * The contents after insertions can be obtained using getTableContents(). */ public class DBSPCompiler implements IWritesLogs, ICompilerComponent, IErrorReporter { + /** Name of the Rust file that will contain the user-defined functions. + * The definitions supplied by the user will be copied here. */ + public static final String UDF_FILE_NAME = "udf.rs"; + /** * Where does the compiled program come from? */ @@ -150,6 +159,10 @@ public boolean hasWarnings() { return this.hasWarnings; } + public CustomFunctions getCustomFunctions() { + return this.frontend.getCustomFunctions(); + } + @Override public DBSPCompiler getCompiler() { return this; @@ -203,6 +216,7 @@ private void compileInternal(String statements, boolean many, @Nullable String c } try { + // Parse using Calcite SqlNodeList parsed; if (many) { if (statements.isEmpty()) @@ -216,15 +230,39 @@ private void compileInternal(String statements, boolean many, @Nullable String c } if (this.hasErrors()) return; - for (SqlNode node : parsed) { + + // Compile first the statements that define functions. + List functions = new ArrayList<>(); + for (SqlNode node: parsed) { Logger.INSTANCE.belowLevel(this, 2) .append("Parsing result: ") .append(node.toString()) .newline(); + if (node.getKind().equals(SqlKind.CREATE_FUNCTION)) { + SqlFunction function = this.frontend.compileFunction(node); + functions.add(function); + } + } + + if (!functions.isEmpty()) { + // Reload the operator table to include all the newly defined functions + SqlOperatorTable newTable = SqlOperatorTables.of(functions); + this.frontend.addOperatorTable(newTable); + if (this.options.ioOptions.udfs.isEmpty()) { + this.getCompiler().reportError( + SourcePositionRange.INVALID, + true, "No UDFs", + "Program contains `CREATE FUNCTION` statements but the compiler" + + " was invoked without the `-udf` flag"); + } + } + + // Compile all statements which do not define functions + for (SqlNode node : parsed) { + if (node.getKind().equals(SqlKind.CREATE_FUNCTION)) + continue; FrontEndStatement fe = this.frontend.compile( node.toString(), node, comment, this.inputTables, this.outputViews); - if (fe == null) - continue; this.midend.compile(fe); } } catch (SqlParseException e) { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/RustFileWriter.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/RustFileWriter.java index 1d0df1aa0aa..1620e35bc22 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/RustFileWriter.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/RustFileWriter.java @@ -280,7 +280,7 @@ fn combine(left: &Tuple2, right: &Tuple2) -> Tuple2 { stream.decrease().append("}\n\n"); } - public String generatePreamble(StructuresUsed used) { + String generatePreamble(StructuresUsed used) { IndentStream stream = new IndentStream(new StringBuilder()); stream.append(commonPreamble); stream.append(rustPreamble) @@ -292,6 +292,16 @@ public String generatePreamble(StructuresUsed used) { .append(";") .newline(); this.generateStructures(used, stream); + + if (!this.compiler.options.ioOptions.udfs.isEmpty()) { + int dot = DBSPCompiler.UDF_FILE_NAME.lastIndexOf("."); + stream.append("mod ") + .append(DBSPCompiler.UDF_FILE_NAME.substring(0, dot)) + .append(";") + .newline() + .append("use crate::udf::*;") + .newline(); + } return stream.toString(); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java index a0b5a8a3413..6dc0ba316c1 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java @@ -24,22 +24,86 @@ package org.dbsp.sqlCompiler.compiler.frontend; import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.*; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.util.DateString; import org.apache.calcite.util.TimeString; import org.apache.calcite.util.TimestampString; -import org.dbsp.sqlCompiler.compiler.ICompilerComponent; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; +import org.dbsp.sqlCompiler.compiler.ICompilerComponent; import org.dbsp.sqlCompiler.compiler.backend.rust.RustSqlRuntimeLibrary; import org.dbsp.sqlCompiler.compiler.errors.BaseCompilerException; +import org.dbsp.sqlCompiler.compiler.errors.CompilationError; import org.dbsp.sqlCompiler.compiler.errors.InternalCompilerError; +import org.dbsp.sqlCompiler.compiler.errors.SourcePosition; +import org.dbsp.sqlCompiler.compiler.errors.SourcePositionRange; import org.dbsp.sqlCompiler.compiler.errors.UnimplementedException; -import org.dbsp.sqlCompiler.ir.expression.*; -import org.dbsp.sqlCompiler.ir.expression.literal.*; -import org.dbsp.sqlCompiler.ir.type.*; -import org.dbsp.sqlCompiler.ir.type.primitive.*; -import org.dbsp.util.*; +import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.CustomFunctions; +import org.dbsp.sqlCompiler.ir.expression.DBSPApplyExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPBinaryExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPConstructorExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPFieldExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPIfExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPOpcode; +import org.dbsp.sqlCompiler.ir.expression.DBSPUnaryExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPVariablePath; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPBinaryLiteral; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPBoolLiteral; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPDateLiteral; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPDecimalLiteral; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPDoubleLiteral; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPGeoPointLiteral; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPI16Literal; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPI32Literal; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPI64Literal; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPI8Literal; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPIntervalMillisLiteral; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPIntervalMonthsLiteral; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPKeywordLiteral; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPLiteral; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPNullLiteral; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPRealLiteral; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPStringLiteral; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPTimeLiteral; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPTimestampLiteral; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPU32Literal; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPVecLiteral; +import org.dbsp.sqlCompiler.ir.path.DBSPPath; +import org.dbsp.sqlCompiler.ir.type.DBSPType; +import org.dbsp.sqlCompiler.ir.type.DBSPTypeAny; +import org.dbsp.sqlCompiler.ir.type.DBSPTypeRef; +import org.dbsp.sqlCompiler.ir.type.DBSPTypeResult; +import org.dbsp.sqlCompiler.ir.type.DBSPTypeTuple; +import org.dbsp.sqlCompiler.ir.type.DBSPTypeVec; +import org.dbsp.sqlCompiler.ir.type.IsDateType; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeBinary; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeBool; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeDate; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeDecimal; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeDouble; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeFP; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeGeoPoint; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeInteger; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeKeyword; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeMillisInterval; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeMonthsInterval; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeNull; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeReal; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeString; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeTime; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeTimestamp; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeUSize; +import org.dbsp.util.IWritesLogs; +import org.dbsp.util.Linq; +import org.dbsp.util.Logger; +import org.dbsp.util.Utilities; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.Point; @@ -59,7 +123,8 @@ public class ExpressionCompiler extends RexVisitorImpl implement private final List constants; private final DBSPCompiler compiler; - public ExpressionCompiler(@Nullable DBSPVariablePath inputRow, DBSPCompiler compiler) { + public ExpressionCompiler( + @Nullable DBSPVariablePath inputRow, DBSPCompiler compiler) { this(inputRow, Linq.list(), compiler); } @@ -694,7 +759,7 @@ node, new DBSPTypeBool(CalciteObject.EMPTY, false), DBSPOpcode.EQ, case "array": return this.compileFunction(call, node, type, ops, 0); } - throw new UnimplementedException(node); + return this.compileUDF(node, call, type, ops); } case OTHER: String opName = call.op.getName().toLowerCase(); @@ -760,6 +825,37 @@ node, new DBSPTypeBool(CalciteObject.EMPTY, false), DBSPOpcode.EQ, } } + DBSPExpression toPosition(SourcePosition pos) { + return new DBSPConstructorExpression( + new DBSPPath("SourcePosition", "new").toExpression(), + DBSPTypeAny.getDefault(), + new DBSPU32Literal(pos.line), new DBSPU32Literal(pos.column)); + } + + DBSPExpression toPosition(SourcePositionRange range) { + return new DBSPConstructorExpression( + new DBSPPath("SourcePositionRange", "new").toExpression(), + DBSPTypeAny.getDefault(), + toPosition(range.start), toPosition(range.end)); + } + + private DBSPExpression compileUDF(CalciteObject node, RexCall call, DBSPType type, List ops) { + String function = call.op.getName(); // no lowercase applied + CustomFunctions.ExternalFunction ef = this.compiler.getCustomFunctions() + .getImplementation(function); + if (ef == null) + throw new CompilationError("Function " + Utilities.singleQuote(function) + " is unknown", node); + List operandTypes = Linq.map(ef.parameterList, + p -> this.typeCompiler.convertType(p.getType(), false)); + List converted = Linq.zip(ops, operandTypes, DBSPExpression::cast); + SourcePositionRange pos = node.getPositionRange(); + DBSPExpression[] arguments = new DBSPExpression[converted.size() + 1]; + arguments[0] = this.toPosition(pos).borrow(); + for (int i = 0; i < converted.size(); i++) + arguments[i+1] = converted.get(i); + return new DBSPApplyExpression(function, new DBSPTypeResult(type), arguments).unwrap(); + } + private DBSPExpression castTo(DBSPExpression expression, DBSPType type) { DBSPType originalType = expression.type; return expression.cast(type.setMayBeNull(originalType.mayBeNull)); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteCompiler.java index 425ac052b4a..5ff0cf8e56c 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteCompiler.java @@ -53,12 +53,15 @@ import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.runtime.MapEntry; +import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlBasicTypeNameSpec; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlDataTypeSpec; import org.apache.calcite.sql.SqlExplainFormat; import org.apache.calcite.sql.SqlExplainLevel; +import org.apache.calcite.sql.SqlFunction; import org.apache.calcite.sql.SqlFunctionCategory; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlInsert; @@ -69,6 +72,7 @@ import org.apache.calcite.sql.SqlOperatorTable; import org.apache.calcite.sql.SqlTypeNameSpec; import org.apache.calcite.sql.SqlUnresolvedFunction; +import org.apache.calcite.sql.ddl.SqlAttributeDefinition; import org.apache.calcite.sql.ddl.SqlColumnDeclaration; import org.apache.calcite.sql.ddl.SqlCreateTable; import org.apache.calcite.sql.ddl.SqlCreateView; @@ -141,18 +145,24 @@ public class CalciteCompiler implements IWritesLogs { private final CompilerOptions options; private final SqlParser.Config parserConfig; - private final SqlValidator validator; private final Catalog catalog; - private final SqlToRelConverter converter; public final RelOptCluster cluster; public final RelDataTypeFactory typeFactory; private final SqlToRelConverter.Config converterConfig; private final RewriteDivision astRewriter; /** Perform additional type validation in top of the Calcite rules. */ - private final ValidateTypes validateTypes; + @Nullable + private SqlValidator validator; + @Nullable + private SqlToRelConverter converter; + @Nullable + private ValidateTypes validateTypes; + private final CalciteConnectionConfig connectionConfig; private final IErrorReporter errorReporter; /** If true the next view will be an output, otherwise it's just an intermediate result */ boolean generateOutputForNextView = true; + private final SchemaPlus rootSchema; + private final CustomFunctions customFunctions; /** * This class rewrites instances of the division operator in the SQL AST @@ -182,6 +192,10 @@ public SqlNode visit(SqlCall call) { } } + public CustomFunctions getCustomFunctions() { + return this.customFunctions; + } + public void generateOutputForNextView(boolean generate) { this.generateOutputForNextView = generate; } @@ -247,6 +261,7 @@ public CalciteCompiler(CompilerOptions options, IErrorReporter errorReporter) { this.astRewriter = new RewriteDivision(); this.options = options; this.errorReporter = errorReporter; + this.customFunctions = new CustomFunctions(); final boolean preserveCasing = false; Properties connConfigProp = new Properties(); @@ -256,7 +271,7 @@ public CalciteCompiler(CompilerOptions options, IErrorReporter errorReporter) { connConfigProp.put(CalciteConnectionProperty.QUOTED_CASING.camelName(), Casing.UNCHANGED.toString()); connConfigProp.put(CalciteConnectionProperty.CONFORMANCE.camelName(), SqlConformanceEnum.BABEL.toString()); } - CalciteConnectionConfig connectionConfig = new CalciteConnectionConfigImpl(connConfigProp); + this.connectionConfig = new CalciteConnectionConfigImpl(connConfigProp); this.parserConfig = SqlParser.config() .withLex(options.languageOptions.lexicalRules) // Our own parser factory, which is a blend of DDL and BABEL @@ -267,26 +282,24 @@ public CalciteCompiler(CompilerOptions options, IErrorReporter errorReporter) { .withConformance(SqlConformanceEnum.LENIENT); this.typeFactory = new SqlTypeFactoryImpl(TYPE_SYSTEM); this.catalog = new Catalog("schema"); - CalciteSchema rootSchema = CalciteSchema.createRootSchema(false, false); - rootSchema.add(catalog.schemaName, this.catalog); + this.rootSchema = CalciteSchema.createRootSchema(false, false).plus(); + this.rootSchema.add(catalog.schemaName, this.catalog); // Register new types - rootSchema.add("BYTEA", factory -> factory.createSqlType(SqlTypeName.VARBINARY)); - rootSchema.add("DATETIME", factory -> factory.createSqlType(SqlTypeName.TIMESTAMP)); - rootSchema.add("INT2", factory -> factory.createSqlType(SqlTypeName.SMALLINT)); - rootSchema.add("INT8", factory -> factory.createSqlType(SqlTypeName.BIGINT)); - rootSchema.add("INT4", factory -> factory.createSqlType(SqlTypeName.INTEGER)); - rootSchema.add("SIGNED", factory -> factory.createSqlType(SqlTypeName.INTEGER)); - rootSchema.add("INT64", factory -> factory.createSqlType(SqlTypeName.BIGINT)); - rootSchema.add("FLOAT64", factory -> factory.createSqlType(SqlTypeName.DOUBLE)); - rootSchema.add("FLOAT32", factory -> factory.createSqlType(SqlTypeName.REAL)); - rootSchema.add("FLOAT4", factory -> factory.createSqlType(SqlTypeName.REAL)); - rootSchema.add("FLOAT8", factory -> factory.createSqlType(SqlTypeName.DOUBLE)); - rootSchema.add("STRING", factory -> factory.createSqlType(SqlTypeName.VARCHAR)); - rootSchema.add("NUMBER", factory -> factory.createSqlType(SqlTypeName.DECIMAL)); - rootSchema.add("TEXT", factory -> factory.createSqlType(SqlTypeName.VARCHAR)); - rootSchema.add("BOOL", factory -> factory.createSqlType(SqlTypeName.BOOLEAN)); - Prepare.CatalogReader catalogReader = new CalciteCatalogReader( - rootSchema, Collections.singletonList(catalog.schemaName), this.typeFactory, connectionConfig); + this.rootSchema.add("BYTEA", factory -> factory.createSqlType(SqlTypeName.VARBINARY)); + this.rootSchema.add("DATETIME", factory -> factory.createSqlType(SqlTypeName.TIMESTAMP)); + this.rootSchema.add("INT2", factory -> factory.createSqlType(SqlTypeName.SMALLINT)); + this.rootSchema.add("INT8", factory -> factory.createSqlType(SqlTypeName.BIGINT)); + this.rootSchema.add("INT4", factory -> factory.createSqlType(SqlTypeName.INTEGER)); + this.rootSchema.add("SIGNED", factory -> factory.createSqlType(SqlTypeName.INTEGER)); + this.rootSchema.add("INT64", factory -> factory.createSqlType(SqlTypeName.BIGINT)); + this.rootSchema.add("FLOAT64", factory -> factory.createSqlType(SqlTypeName.DOUBLE)); + this.rootSchema.add("FLOAT32", factory -> factory.createSqlType(SqlTypeName.REAL)); + this.rootSchema.add("FLOAT4", factory -> factory.createSqlType(SqlTypeName.REAL)); + this.rootSchema.add("FLOAT8", factory -> factory.createSqlType(SqlTypeName.DOUBLE)); + this.rootSchema.add("STRING", factory -> factory.createSqlType(SqlTypeName.VARCHAR)); + this.rootSchema.add("NUMBER", factory -> factory.createSqlType(SqlTypeName.DECIMAL)); + this.rootSchema.add("TEXT", factory -> factory.createSqlType(SqlTypeName.VARCHAR)); + this.rootSchema.add("BOOL", factory -> factory.createSqlType(SqlTypeName.BOOLEAN)); SqlOperatorTable operatorTable = SqlOperatorTables.chain( SqlLibraryOperatorTableFactory.INSTANCE.getOperatorTable( @@ -297,26 +310,43 @@ public CalciteCompiler(CompilerOptions options, IErrorReporter errorReporter) { SqlLibrary.BIG_QUERY, SqlLibrary.SPARK, SqlLibrary.SPATIAL)), - SqlOperatorTables.of(CustomFunctions.INSTANCE.getUDFs()) + SqlOperatorTables.of(this.customFunctions.getInitialFunctions()) ); + // This planner does not do anything. + // We use a series of planner stages later to perform the real optimizations. + RelOptPlanner planner = new HepPlanner(new HepProgramBuilder().build()); + planner.setExecutor(RexUtil.EXECUTOR); + this.cluster = RelOptCluster.create(planner, new RexBuilder(this.typeFactory)); + this.converterConfig = SqlToRelConverter.config() + .withExpand(true); + this.validator = null; + this.validateTypes = null; + this.converter = null; + this.addOperatorTable(operatorTable); + } + /** Add a new set of operators to the operator table. Creates a new validator, converter */ + public void addOperatorTable(SqlOperatorTable operatorTable) { + SqlOperatorTable newOperatorTable; + if (this.validator != null) { + newOperatorTable = SqlOperatorTables.chain( + this.validator.getOperatorTable(), + operatorTable); + } else { + newOperatorTable = operatorTable; + } SqlValidator.Config validatorConfig = SqlValidator.Config.DEFAULT .withIdentifierExpansion(true); + Prepare.CatalogReader catalogReader = new CalciteCatalogReader( + CalciteSchema.from(this.rootSchema), Collections.singletonList(catalog.schemaName), + this.typeFactory, connectionConfig); this.validator = SqlValidatorUtil.newValidator( - operatorTable, + newOperatorTable, catalogReader, this.typeFactory, validatorConfig ); this.validateTypes = new ValidateTypes(this.validator, errorReporter); - - // This planner does not do anything. - // We use a series of planner stages later to perform the real optimizations. - RelOptPlanner planner = new HepPlanner(new HepProgramBuilder().build()); - planner.setExecutor(RexUtil.EXECUTOR); - this.cluster = RelOptCluster.create(planner, new RexBuilder(this.typeFactory)); - this.converterConfig = SqlToRelConverter.config() - .withExpand(true); this.converter = new SqlToRelConverter( (type, query, schema, path) -> null, this.validator, @@ -560,6 +590,10 @@ public RelDataType convertType(SqlDataTypeSpec spec) { return result; } + public SqlToRelConverter getConverter() { + return Objects.requireNonNull(this.converter); + } + List createTableColumnsMetadata(SqlNodeList list) { List result = new ArrayList<>(); int index = 0; @@ -631,19 +665,20 @@ List createTableColumnsMetadata(SqlNodeList list) { isPrimaryKey = cd.primaryKey || declaredPrimary; if (declaredPrimary) primaryKeys.remove(name.getSimple()); + SqlToRelConverter converter = this.getConverter(); if (cd.lateness != null) - lateness = this.converter.convertExpression(cd.lateness); + lateness = converter.convertExpression(cd.lateness); if (cd.defaultValue != null) { // workaround for https://issues.apache.org/jira/browse/CALCITE-6129 if (cd.defaultValue instanceof SqlLiteral) { SqlLiteral literal = (SqlLiteral) cd.defaultValue; if (literal.getTypeName() == SqlTypeName.NULL) { - RelDataType type = literal.createSqlType(this.converter.getCluster().getTypeFactory()); - defaultValue = this.converter.getRexBuilder().makeLiteral(null, type); + RelDataType type = literal.createSqlType(converter.getCluster().getTypeFactory()); + defaultValue = converter.getRexBuilder().makeLiteral(null, type); } } if (defaultValue == null) - defaultValue = this.converter.convertExpression(cd.defaultValue); + defaultValue = converter.convertExpression(cd.defaultValue); } } else if (col instanceof SqlKeyConstraint) { continue; @@ -741,6 +776,23 @@ public List createColumnsMetadata(CalciteObject node, return columns; } + /** Compile a SQL statement which declares a user-defined function */ + public SqlFunction compileFunction(SqlNode node) { + SqlCreateFunctionDeclaration decl = (SqlCreateFunctionDeclaration) node; + List> parameters = Linq.map( + decl.getParameters(), param -> { + SqlAttributeDefinition attr = (SqlAttributeDefinition) param; + String name = attr.name.getSimple(); + RelDataType type = this.convertType(attr.dataType); + return new MapEntry<>(name, type); + }); + RelDataType structType = this.typeFactory.createStructType(parameters); + RelDataType returnType = this.convertType(decl.getReturnType()); + return this.customFunctions.createUDF( + CalciteObject.create(node), decl.getName(), structType, returnType + ); + } + /** * Compile a SQL statement. Returns null if the statement does not require further processing. * @param node Compiled version of the SQL statement. @@ -749,7 +801,6 @@ public List createColumnsMetadata(CalciteObject node, * @param inputs If not null, add here a JSON description of the tables defined by the statement, if any. * @param outputs If not null, add here a JSON description of the views defined by the statement, if any. */ - @Nullable public FrontEndStatement compile( String sqlStatement, SqlNode node, @@ -784,7 +835,7 @@ public FrontEndStatement compile( Logger.INSTANCE.belowLevel(this, 1) .append(ct.query.toString()) .newline(); - RelRoot relRoot = this.converter.convertQuery(ct.query, true, true); + RelRoot relRoot = converter.convertQuery(ct.query, true, true); cols = this.createColumnsMetadata( ct.name, false, relRoot, null); */ @@ -794,6 +845,7 @@ public FrontEndStatement compile( inputs.add(new InputTableDescription(table)); return table; } else if (node.getKind().equals(SqlKind.CREATE_VIEW)) { + SqlToRelConverter converter = this.getConverter(); SqlCreateView cv = (SqlCreateView) node; SqlNode query = cv.query; if (cv.getReplace()) @@ -805,7 +857,7 @@ public FrontEndStatement compile( Logger.INSTANCE.belowLevel(this, 2) .append(Objects.requireNonNull(query).toString()) .newline(); - RelRoot relRoot = this.converter.convertQuery(query, true, true); + RelRoot relRoot = converter.convertQuery(query, true, true); List columns = this.createColumnsMetadata(CalciteObject.create(node), cv.name, true, relRoot, cv.columnList); RelNode optimized = this.optimize(relRoot.rel); @@ -819,21 +871,19 @@ public FrontEndStatement compile( if (this.generateOutputForNextView) outputs.add(new OutputViewDescription(view)); return view; - } else if (node.getKind().equals(SqlKind.CREATE_FUNCTION)) { - SqlCreateFunctionDeclaration decl = (SqlCreateFunctionDeclaration) node; - return null; } } if (SqlKind.DML.contains(node.getKind())) { if (node instanceof SqlInsert) { + SqlToRelConverter converter = this.getConverter(); SqlInsert insert = (SqlInsert) node; SqlNode table = insert.getTargetTable(); if (!(table instanceof SqlIdentifier)) throw new UnimplementedException(CalciteObject.create(table)); SqlIdentifier id = (SqlIdentifier) table; TableModifyStatement stat = new TableModifyStatement(node, sqlStatement, id.toString(), insert.getSource(), comment); - RelRoot values = this.converter.convertQuery(stat.data, true, true); + RelRoot values = converter.convertQuery(stat.data, true, true); values = values.withRel(this.optimize(values.rel)); stat.setTranslation(values.rel); return stat; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/Catalog.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/Catalog.java index a102e721290..68489686d14 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/Catalog.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/Catalog.java @@ -23,6 +23,9 @@ package org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; +import org.apache.calcite.schema.Function; import org.apache.calcite.schema.Table; import org.apache.calcite.schema.impl.AbstractSchema; import org.apache.calcite.sql.SqlIdentifier; @@ -38,10 +41,12 @@ public class Catalog extends AbstractSchema { public final String schemaName; private final Map tableMap; + private final Multimap functionMap; public Catalog(String schemaName) { this.schemaName = schemaName; this.tableMap = new HashMap<>(); + this.functionMap = ArrayListMultimap.create(); } public static String identifierToString(SqlIdentifier identifier) { @@ -54,11 +59,18 @@ public void addTable(String name, Table table) { this.tableMap.put(name, table); } + public void addFunction(String name, Function function) { this.functionMap.put(name, function); } + @Override public Map getTableMap() { return this.tableMap; } + @Override + protected Multimap getFunctionMultimap() { + return this.functionMap; + } + public void dropTable(String tableName) { this.tableMap.remove(tableName); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CustomFunctions.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CustomFunctions.java index 4301642bf54..a4a2d476347 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CustomFunctions.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CustomFunctions.java @@ -1,32 +1,50 @@ package org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.sql.SqlCallBinding; import org.apache.calcite.sql.SqlFunction; import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOperandCountRange; +import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlOperatorBinding; +import org.apache.calcite.sql.type.InferTypes; import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlOperandCountRanges; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; +import org.apache.calcite.sql.type.SqlOperandTypeInference; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlSingleOperandTypeChecker; import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; -import org.dbsp.sqlCompiler.compiler.errors.UnimplementedException; +import org.dbsp.sqlCompiler.compiler.errors.CompilationError; +import org.dbsp.sqlCompiler.compiler.frontend.CalciteObject; +import org.dbsp.util.Linq; +import org.dbsp.util.Utilities; +import javax.annotation.Nullable; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import static java.util.Objects.requireNonNull; import static org.apache.calcite.sql.type.OperandTypes.family; import static org.apache.calcite.sql.type.ReturnTypes.ARG1; /** Several functions that we define and add to the existing ones. */ public class CustomFunctions { - public static final CustomFunctions INSTANCE = new CustomFunctions(); - private final List functions = new ArrayList<>(); + private final List initial = new ArrayList<>(); + private final HashMap udf; - private CustomFunctions() { - this.functions.add(new SqlDivisionFunction()); - this.functions.add(new RlikeFunction()); - this.functions.add(new WriteLogFunction()); + public CustomFunctions() { + this.initial.add(new SqlDivisionFunction()); + this.initial.add(new RlikeFunction()); + this.initial.add(new WriteLogFunction()); + this.udf = new HashMap<>(); } static class RlikeFunction extends SqlFunction { @@ -107,18 +125,103 @@ public boolean isDeterministic() { } } + /** A variant ot OperandTypes.TypeNameChecker which does not extend the interface + * ImplicitCastOperandTypeChecker. We do not want to allow implicit casts for these operands. */ + private static class ExactTypeNameChecker implements SqlSingleOperandTypeChecker { + final SqlTypeName typeName; + + ExactTypeNameChecker(SqlTypeName typeName) { + this.typeName = requireNonNull(typeName, "typeName"); + } + + @Override public boolean checkSingleOperandType(SqlCallBinding callBinding, + SqlNode operand, int iFormalOperand, boolean throwOnFailure) { + final RelDataType operandType = + callBinding.getValidator().getValidatedNodeType(operand); + return operandType.getSqlTypeName() == typeName; + } + + @Override public String getAllowedSignatures(SqlOperator op, String opName) { + return opName + "(" + typeName.getSpaceName() + ")"; + } + } + + public static class ExternalFunction extends SqlFunction { + public final List parameterList; + + static SqlOperandTypeChecker createTypeChecker(String function, List parameters) { + if (parameters.isEmpty()) + return OperandTypes.NILADIC; + SqlSingleOperandTypeChecker[] checkers = new SqlSingleOperandTypeChecker[parameters.size()]; + StringBuilder builder = new StringBuilder(); + builder.append(function).append("("); + for (int i = 0; i < parameters.size(); i++) { + RelDataTypeField field = parameters.get(i); + SqlTypeName typeName = field.getType().getSqlTypeName(); + // This type checker is a bit too strict. + checkers[i] = new ExactTypeNameChecker(typeName); + // This type checker allows implicit casts from any type in the type family, + // which is not great. + // checkers[i] = OperandTypes.typeName(typeName); + if (i > 0) + builder.append(", "); + builder.append("<") + .append(field.getType().toString()) + .append(">"); + } + builder.append(")"); + return OperandTypes.sequence(builder.toString(), checkers); + } + + static SqlOperandTypeInference createTypeinference(List parameters) { + return InferTypes.explicit(Linq.map(parameters, RelDataTypeField::getType)); + } + + public ExternalFunction(SqlIdentifier name, RelDataType returnType, + List parameters) { + super(name.getSimple(), SqlKind.OTHER_FUNCTION, ReturnTypes.explicit(returnType), + createTypeinference(parameters), createTypeChecker(name.getSimple(), parameters), + SqlFunctionCategory.USER_DEFINED_FUNCTION); + this.parameterList = parameters; + } + + @Override + public boolean isDeterministic() { + return false; + } + + @Override + public SqlOperandCountRange getOperandCountRange() { + return SqlOperandCountRanges.of(this.parameterList.size()); + } + } + /** * Create a new user-defined function with a specified signature. - * @param name Function name. - * @param signature Description of arguments as a struct. - * @param returnType Return type of function. + * + * @param name Function name. + * @param signature Description of arguments as a struct. + * @param returnType Return type of function. */ - public SqlFunction createUDF(String name, RelDataType signature, RelDataType returnType) { - throw new UnimplementedException(); + public SqlFunction createUDF(CalciteObject node, SqlIdentifier name, RelDataType signature, RelDataType returnType) { + List parameterList = signature.getFieldList(); + ExternalFunction result = new ExternalFunction(name, returnType, parameterList); + String functionName = result.getName(); + if (this.udf.containsKey(functionName)) { + throw new CompilationError("Function with name " + + Utilities.singleQuote(functionName) + " already exists", node); + } + Utilities.putNew(this.udf, functionName, result); + return result; + } + + @Nullable + public ExternalFunction getImplementation(String function) { + return this.udf.get(function); } - /** Return the list of user-defined functions */ - public List getUDFs() { - return this.functions; + /** Return the list custom functions we added to the library. */ + public List getInitialFunctions() { + return this.initial; } } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlCreateFunctionDeclaration.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlCreateFunctionDeclaration.java index f58e3792611..7904ef2a467 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlCreateFunctionDeclaration.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlCreateFunctionDeclaration.java @@ -55,6 +55,18 @@ public SqlCreateFunctionDeclaration(SqlParserPos pos, boolean replace, return OPERATOR; } + public SqlNodeList getParameters() { + return this.parameters; + } + + public SqlDataTypeSpec getReturnType() { + return this.returnType; + } + + public SqlIdentifier getName() { + return this.name; + } + @Override public List getOperandList() { return Arrays.asList(this.name, this.parameters); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/InnerVisitor.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/InnerVisitor.java index 467a2e098e6..2ffcb0ba5c0 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/InnerVisitor.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/InnerVisitor.java @@ -318,6 +318,10 @@ public VisitDecision preorder(DBSPTypeVec node) { return this.preorder((DBSPTypeUser) node); } + public VisitDecision preorder(DBSPTypeResult node) { + return this.preorder((DBSPTypeUser) node); + } + public VisitDecision preorder(DBSPTypeSemigroup node) { return this.preorder((DBSPTypeUser) node); } @@ -786,6 +790,10 @@ public void postorder(DBSPTypeVec node) { this.postorder((DBSPTypeUser) node); } + public void postorder(DBSPTypeResult node) { + this.postorder((DBSPTypeUser) node); + } + public void postorder(DBSPTypeSemigroup node) { this.postorder((DBSPTypeUser) node); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPConstructorExpression.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPConstructorExpression.java index 910c342c959..3e626d0131c 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPConstructorExpression.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPConstructorExpression.java @@ -30,9 +30,7 @@ import org.dbsp.util.IIndentStream; import org.dbsp.util.Linq; -/** - * Invocation of a Rust constructor with some arguments. - */ +/** Invocation of a Rust constructor with some arguments. */ public class DBSPConstructorExpression extends DBSPExpression { public final DBSPExpression function; public final DBSPExpression[] arguments; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPExpression.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPExpression.java index 718f3949af0..08d0d949091 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPExpression.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPExpression.java @@ -29,6 +29,7 @@ import org.dbsp.sqlCompiler.ir.IDBSPInnerNode; import org.dbsp.sqlCompiler.ir.expression.literal.DBSPBoolLiteral; import org.dbsp.sqlCompiler.ir.type.DBSPType; +import org.dbsp.sqlCompiler.ir.type.DBSPTypeResult; import org.dbsp.sqlCompiler.ir.type.IHasType; import javax.annotation.Nullable; @@ -65,6 +66,12 @@ public DBSPExpression borrow() { return new DBSPBorrowExpression(this); } + /** Unwrap a Rust 'Result' type */ + public DBSPExpression unwrap() { + DBSPTypeResult type = this.type.to(DBSPTypeResult.class); + return new DBSPApplyMethodExpression(this.getNode(), "unwrap", type.getTypeArg(0), this); + } + public DBSPExpression borrow(boolean mutable) { return new DBSPBorrowExpression(this, mutable); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/literal/DBSPU32Literal.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/literal/DBSPU32Literal.java index 2cbbb60af59..8e2c9ee4b6d 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/literal/DBSPU32Literal.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/literal/DBSPU32Literal.java @@ -23,6 +23,7 @@ package org.dbsp.sqlCompiler.ir.expression.literal; +import org.dbsp.sqlCompiler.compiler.errors.CompilationError; import org.dbsp.sqlCompiler.compiler.errors.InternalCompilerError; import org.dbsp.sqlCompiler.compiler.frontend.CalciteObject; import org.dbsp.sqlCompiler.compiler.visitors.VisitDecision; @@ -49,6 +50,8 @@ public boolean sameValue(@Nullable DBSPLiteral o) { public DBSPU32Literal(CalciteObject node, DBSPType type, @Nullable Integer value) { super(node, type, value == null); + if (value != null && value < 0) + throw new CompilationError("Negative value for u32 literal " + value); this.value = value; } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/DBSPTypeCode.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/DBSPTypeCode.java index 3ddd85879fd..e093fc4563c 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/DBSPTypeCode.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/DBSPTypeCode.java @@ -35,6 +35,7 @@ public enum DBSPTypeCode { INDEXED_ZSET("", ""), RAW_TUPLE("", ""), REF("", ""), + RESULT("", "Result"), SEMIGROUP("", ""), STREAM("", ""), STRUCT("", ""), diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/DBSPTypeResult.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/DBSPTypeResult.java new file mode 100644 index 00000000000..455128d3329 --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/DBSPTypeResult.java @@ -0,0 +1,59 @@ +/* + * Copyright 2022 VMware, Inc. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.dbsp.sqlCompiler.ir.type; + +import org.dbsp.sqlCompiler.compiler.frontend.CalciteObject; +import org.dbsp.sqlCompiler.compiler.visitors.VisitDecision; +import org.dbsp.sqlCompiler.compiler.visitors.inner.InnerVisitor; +import org.dbsp.sqlCompiler.ir.NonCoreIR; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeStr; + +import static org.dbsp.sqlCompiler.ir.type.DBSPTypeCode.RESULT; + +/** Represents the type of a Rust Result[T, 'str] type as a TypeUser. */ +@NonCoreIR +public class DBSPTypeResult extends DBSPTypeUser { + public DBSPTypeResult(DBSPType resultType) { + super(resultType.getNode(), RESULT, "Result", false, + resultType, new DBSPTypeStr(CalciteObject.EMPTY, false)); + } + + @Override + public void accept(InnerVisitor visitor) { + VisitDecision decision = visitor.preorder(this); + if (decision.stop()) return; + visitor.push(this); + for (DBSPType type: this.typeArgs) + type.accept(visitor); + visitor.pop(this); + visitor.postorder(this); + } + + @Override + public boolean hasCopy() { + return false; + } + + // sameType and hashCode inherited from TypeUser. +} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/DBSPTypeUser.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/DBSPTypeUser.java index c6f2c97736d..f5969d91e6c 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/DBSPTypeUser.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/DBSPTypeUser.java @@ -31,9 +31,7 @@ import java.util.Arrays; import java.util.Objects; -/** - * User-defined generic type with type arguments. - */ +/** User-defined generic type with type arguments. */ public class DBSPTypeUser extends DBSPType { public final String name; public final DBSPType[] typeArgs; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/DBSPTypeVec.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/DBSPTypeVec.java index 3b85ee03d47..40ba2841110 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/DBSPTypeVec.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/DBSPTypeVec.java @@ -28,9 +28,7 @@ import static org.dbsp.sqlCompiler.ir.type.DBSPTypeCode.VEC; -/** - * Represents the type of a Rust Vec as a TypeUser. - */ +/** Represents the type of a Rust Vec as a TypeUser. */ @SuppressWarnings("GrazieInspection") public class DBSPTypeVec extends DBSPTypeUser implements ICollectionType { public DBSPTypeVec(DBSPType vectorElementType, boolean mayBeNull) { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/BaseSQLTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/BaseSQLTests.java index 07d4cfd59bd..891c670fb42 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/BaseSQLTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/BaseSQLTests.java @@ -42,6 +42,7 @@ import java.io.IOException; import java.io.PrintStream; import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; @@ -83,13 +84,17 @@ public static void prepareTests() { testsToRun.clear(); } + public static File createInputFile(File file, String separator, String... contents) throws IOException { + file.deleteOnExit(); + PrintWriter script = new PrintWriter(file, StandardCharsets.UTF_8); + script.println(String.join(separator, contents)); + script.close(); + return file; + } + public static File createInputScript(String... contents) throws IOException { File result = File.createTempFile("script", ".sql", new File(rustDirectory)); - result.deleteOnExit(); - PrintWriter script = new PrintWriter(result, "UTF-8"); - script.println(String.join(";" + System.lineSeparator(), contents)); - script.close(); - return result; + return createInputFile(result, ";" + System.lineSeparator(), contents); } DBSPCompiler noThrowCompiler() { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/DBSPCompilerTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/DBSPCompilerTests.java similarity index 96% rename from sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/DBSPCompilerTests.java rename to sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/DBSPCompilerTests.java index d190925ac9b..4c5f6d5d437 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/DBSPCompilerTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/DBSPCompilerTests.java @@ -23,9 +23,11 @@ * */ -package org.dbsp.sqlCompiler.compiler; +package org.dbsp.sqlCompiler.compiler.sql; import org.dbsp.sqlCompiler.circuit.DBSPCircuit; +import org.dbsp.sqlCompiler.compiler.CompilerOptions; +import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.sqlCompiler.compiler.backend.rust.RustFileWriter; import org.dbsp.sqlCompiler.compiler.frontend.TableContents; import org.dbsp.sqlCompiler.ir.expression.literal.DBSPZSetLiteral; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/NegativeParserTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/NegativeParserTests.java similarity index 98% rename from sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/NegativeParserTests.java rename to sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/NegativeParserTests.java index a33ab21b4dd..f18a38f0330 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/NegativeParserTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/NegativeParserTests.java @@ -1,9 +1,10 @@ -package org.dbsp.sqlCompiler.compiler; +package org.dbsp.sqlCompiler.compiler.sql; import org.apache.calcite.sql.parser.SqlParseException; import org.dbsp.sqlCompiler.CompilerMain; +import org.dbsp.sqlCompiler.compiler.DBSPCompiler; +import org.dbsp.sqlCompiler.compiler.TestUtil; import org.dbsp.sqlCompiler.compiler.errors.CompilerMessages; -import org.dbsp.sqlCompiler.compiler.sql.BaseSQLTests; import org.junit.Assert; import org.junit.Test; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/OtherTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/OtherTests.java similarity index 88% rename from sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/OtherTests.java rename to sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/OtherTests.java index e1a660b1df4..d449e74c2e9 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/OtherTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/OtherTests.java @@ -21,28 +21,37 @@ * SOFTWARE. */ -package org.dbsp.sqlCompiler.compiler; +package org.dbsp.sqlCompiler.compiler.sql; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import org.dbsp.sqlCompiler.circuit.operator.DBSPStreamDistinctOperator; -import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; -import org.dbsp.sqlCompiler.compiler.backend.rust.RustFileWriter; -import org.dbsp.sqlCompiler.compiler.errors.CompilerMessages; import org.dbsp.sqlCompiler.CompilerMain; import org.dbsp.sqlCompiler.circuit.DBSPCircuit; +import org.dbsp.sqlCompiler.circuit.operator.DBSPOperator; +import org.dbsp.sqlCompiler.circuit.operator.DBSPStreamDistinctOperator; +import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.sqlCompiler.compiler.backend.ToCsvVisitor; +import org.dbsp.sqlCompiler.compiler.backend.rust.RustFileWriter; import org.dbsp.sqlCompiler.compiler.backend.rust.ToRustVisitor; +import org.dbsp.sqlCompiler.compiler.errors.CompilerMessages; import org.dbsp.sqlCompiler.compiler.frontend.CalciteObject; -import org.dbsp.sqlCompiler.compiler.visitors.inner.CollectIdentifiers; import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.CalciteCompiler; -import org.dbsp.sqlCompiler.compiler.sql.BaseSQLTests; import org.dbsp.sqlCompiler.compiler.sql.simple.EndToEndTests; +import org.dbsp.sqlCompiler.compiler.visitors.inner.CollectIdentifiers; import org.dbsp.sqlCompiler.compiler.visitors.outer.Passes; import org.dbsp.sqlCompiler.ir.DBSPFunction; import org.dbsp.sqlCompiler.ir.DBSPNode; -import org.dbsp.sqlCompiler.ir.expression.*; -import org.dbsp.sqlCompiler.ir.expression.literal.*; +import org.dbsp.sqlCompiler.ir.expression.DBSPApplyExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPApplyMethodExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPBlockExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPClosureExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPTupleExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPVariablePath; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPI32Literal; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPStrLiteral; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPUSizeLiteral; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPZSetLiteral; import org.dbsp.sqlCompiler.ir.statement.DBSPExpressionStatement; import org.dbsp.sqlCompiler.ir.statement.DBSPLetStatement; import org.dbsp.sqlCompiler.ir.statement.DBSPStatement; @@ -62,8 +71,15 @@ import org.junit.Test; import javax.imageio.ImageIO; -import java.io.*; -import java.sql.*; +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -324,6 +340,54 @@ public void testWith() throws IOException, InterruptedException { Utilities.compileAndTestRust(BaseSQLTests.rustDirectory, false); } + @Test + public void testUDFWarning() throws IOException { + File file = createInputScript("CREATE FUNCTION myfunction(d DATE, i INTEGER) RETURNS VARCHAR", + "CREATE VIEW V AS SELECT myfunction(DATE '2023-10-20', CAST(5 AS INTEGER))"); + CompilerMessages messages = CompilerMain.execute("-o", BaseSQLTests.testFilePath, file.getPath()); + Assert.assertEquals(1, messages.warningCount()); + Assert.assertTrue(messages.toString().contains("the compiler was invoked without the `-udf` flag")); + } + + @Test + public void testUDFTypeError() throws IOException { + File file = createInputScript("CREATE FUNCTION myfunction(d DATE, i INTEGER) RETURNS VARCHAR NOT NULL", + "CREATE VIEW V AS SELECT myfunction(DATE '2023-10-20', '5')"); + CompilerMessages messages = CompilerMain.execute("-o", BaseSQLTests.testFilePath, file.getPath()); + Assert.assertEquals(1, messages.errorCount()); + Assert.assertTrue(messages.toString().contains( + "Cannot apply 'MYFUNCTION' to arguments of type 'MYFUNCTION(, )'. " + + "Supported form(s): MYFUNCTION(, )")); + } + + @Test + public void testUDF() throws IOException, InterruptedException { + File file = createInputScript( + "CREATE FUNCTION contains_number(str VARCHAR NOT NULL, value INTEGER) RETURNS BOOLEAN NOT NULL", + "CREATE VIEW V0 AS SELECT contains_number(CAST('YES: 10 NO:5 MAYBE: 2' AS VARCHAR), 5)", + "CREATE FUNCTION \"empty\"() RETURNS VARCHAR", + "CREATE VIEW V1 AS SELECT \"empty\"()"); + File implementation = File.createTempFile("impl", ".rs", new File(rustDirectory)); + createInputFile(implementation, + System.lineSeparator(), + "use sqllib::types::*;", + "pub fn CONTAINS_NUMBER(pos: &SourcePositionRange, str: String, value: Option) -> " + + " Result> {", + " match value {", + " None => Err(format!(\"{}: null value\", pos).into()),", + " Some(value) => Ok(str.contains(&format!(\"{}\", value).to_string())),", + " }", + "}", + "pub fn empty(pos: &SourcePositionRange) -> Result, Box> {", + " Ok(Some(\"\".to_string()))", + "}"); + CompilerMessages messages = CompilerMain.execute("-o", BaseSQLTests.testFilePath, "--udf", + implementation.getPath(), file.getPath()); + if (messages.errorCount() > 0) + throw new RuntimeException(messages.toString()); + Utilities.compileAndTestRust(BaseSQLTests.rustDirectory, false); + } + @Test public void testProjectFiles() throws IOException, InterruptedException { // Compiles all the programs in the tests directory diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ParserTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/ParserTests.java similarity index 93% rename from sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ParserTests.java rename to sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/ParserTests.java index 1ce9e40eca7..f1cbe21d570 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ParserTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/ParserTests.java @@ -23,11 +23,14 @@ * */ -package org.dbsp.sqlCompiler.compiler; +package org.dbsp.sqlCompiler.compiler.sql; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.ddl.SqlCreateTable; import org.apache.calcite.sql.parser.SqlParseException; +import org.dbsp.sqlCompiler.compiler.CompilerOptions; +import org.dbsp.sqlCompiler.compiler.IErrorReporter; +import org.dbsp.sqlCompiler.compiler.StderrErrorReporter; import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.CalciteCompiler; import org.junit.Assert; import org.junit.Test; @@ -63,8 +66,9 @@ public void DDLTest() throws SqlParseException { @Test public void createFunctionTest() throws SqlParseException { CalciteCompiler calcite = this.getCompiler(); - String ddl = "CREATE FUNCTION json(data VARCHAR) VARBINARY"; - SqlNode node = calcite.parse(ddl); + String ddl = "CREATE FUNCTION to_json(data VARCHAR) RETURNS VARBINARY;\n" + + "CREATE FUNCTION from_json(data VARBINARY) RETURNS VARCHAR;\n"; + SqlNode node = calcite.parseStatements(ddl); Assert.assertNotNull(node); } @@ -75,8 +79,11 @@ public void createTypeTest() throws SqlParseException { " street VARCHAR(30),\n" + " city VARCHAR(30),\n" + " state CHAR(2),\n" + - " postal_code VARCHAR(6))"; - SqlNode node = calcite.parse(ddl); + " postal_code VARCHAR(6));\n" + + "CREATE TYPE person_type AS (\n" + + " firstname VARCHAR(30),\n" + + " lastname VARCHAR(30));"; + SqlNode node = calcite.parseStatements(ddl); Assert.assertNotNull(node); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/functions/FunctionsTest.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/functions/FunctionsTest.java index 2a13e1b80d5..1873a1a7f0b 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/functions/FunctionsTest.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/functions/FunctionsTest.java @@ -1,9 +1,15 @@ package org.dbsp.sqlCompiler.compiler.sql.functions; +import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.sqlCompiler.compiler.sql.SqlIoTest; import org.junit.Test; public class FunctionsTest extends SqlIoTest { + @Override + public void prepareData(DBSPCompiler compiler) { + compiler.compileStatements(""); + } + @Test public void testLeft() { this.q("SELECT LEFT('string', 1);\n" + diff --git a/sql-to-dbsp-compiler/lib/sqllib/src/geopoint.rs b/sql-to-dbsp-compiler/lib/sqllib/src/geopoint.rs index 7a7b1227914..28bafba0da0 100644 --- a/sql-to-dbsp-compiler/lib/sqllib/src/geopoint.rs +++ b/sql-to-dbsp-compiler/lib/sqllib/src/geopoint.rs @@ -1,5 +1,6 @@ // I cannot use the standard geopoint object because it doesn't implement Ord +use crate::some_polymorphic_function2; use ::serde::{Deserialize, Serialize}; use dbsp::algebra::F64; use dbsp::num_entries_scalar; @@ -72,3 +73,9 @@ pub fn make_geopointN_dN_dN(left: Option, right: Option) -> Option make_geopointN_d_d(x, y), } } + +pub fn st_distance_geopoint_geopoint(left: GeoPoint, right: GeoPoint) -> F64 { + left.distance(&right) +} + +some_polymorphic_function2!(st_distance, geopoint, GeoPoint, geopoint, GeoPoint, F64); diff --git a/sql-to-dbsp-compiler/lib/sqllib/src/lib.rs b/sql-to-dbsp-compiler/lib/sqllib/src/lib.rs index 0b92b1ea39b..58b8a3b905c 100644 --- a/sql-to-dbsp-compiler/lib/sqllib/src/lib.rs +++ b/sql-to-dbsp-compiler/lib/sqllib/src/lib.rs @@ -6,14 +6,23 @@ pub mod casts; pub mod geopoint; pub mod interval; pub mod operators; +pub mod source; pub mod string; pub mod timestamp; +pub mod types; + +// Types re-exported by types.rs +pub use geopoint::GeoPoint; +pub use interval::LongInterval; +pub use interval::ShortInterval; +pub use source::{SourcePosition, SourcePositionRange}; +pub use timestamp::Date; +pub use timestamp::Time; +pub use timestamp::Timestamp; -use crate::interval::ShortInterval; use dbsp::algebra::{Semigroup, SemigroupValue, ZRingValue, F32, F64}; use dbsp::trace::{Batch, BatchReader, Builder, Cursor}; use dbsp::{DBData, DBWeight, OrdIndexedZSet, OrdZSet}; -use geopoint::GeoPoint; use num::{Signed, ToPrimitive}; use rust_decimal::{Decimal, MathematicalOps}; use std::fmt::Debug; @@ -662,12 +671,6 @@ where } } -pub fn st_distance_geopoint_geopoint(left: GeoPoint, right: GeoPoint) -> F64 { - left.distance(&right) -} - -some_polymorphic_function2!(st_distance, geopoint, GeoPoint, geopoint, GeoPoint, F64); - pub fn times_ShortInterval_i64(left: ShortInterval, right: i64) -> ShortInterval { left * right } diff --git a/sql-to-dbsp-compiler/lib/sqllib/src/source.rs b/sql-to-dbsp-compiler/lib/sqllib/src/source.rs new file mode 100644 index 00000000000..a7af788dea0 --- /dev/null +++ b/sql-to-dbsp-compiler/lib/sqllib/src/source.rs @@ -0,0 +1,56 @@ +//! Source position representation. +//! Used for reporting run-time errors. + +// Currently Calcite does not provide source-position, but we have +// opened submitted patches. Once they are accepted this will be much +// more useful. + +use std::fmt; + +#[derive(Default, Debug)] +pub struct SourcePosition { + // Source lines are counted from 1, so a value a 0 for row + // indicates "unknown". + pub line: u32, + pub column: u32, +} + +#[derive(Default, Debug)] +pub struct SourcePositionRange { + pub start: SourcePosition, + pub end: SourcePosition, +} + +impl SourcePosition { + pub fn new(line: u32, column: u32) -> Self { + Self { line, column } + } + + pub fn isValid(&self) -> bool { + self.line > 0 + } +} + +impl fmt::Display for SourcePosition { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if self.isValid() { write!(f, "{}:{}", self.line, self.column) } + else { write!(f, "") } + } +} + +impl SourcePositionRange { + pub fn new(start: SourcePosition, end: SourcePosition) -> Self { + Self { start, end } + } + + pub fn isValid(&self) -> bool { + self.start.isValid() + } +} + +impl fmt::Display for SourcePositionRange { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if self.isValid() { write!(f, "{}-{}", self.start, self.end) } + else { write!(f, "") } + } +} diff --git a/sql-to-dbsp-compiler/lib/sqllib/src/types.rs b/sql-to-dbsp-compiler/lib/sqllib/src/types.rs new file mode 100644 index 00000000000..f3a319d2f23 --- /dev/null +++ b/sql-to-dbsp-compiler/lib/sqllib/src/types.rs @@ -0,0 +1,9 @@ +//! Crate which just re-exports all the types that are used to implement +//! the types that appear in SQL. + +pub use dbsp::algebra::{F32, F64}; + +pub use crate::{ + Date, GeoPoint, LongInterval, ShortInterval, SourcePosition, SourcePositionRange, Time, + Timestamp +}; From 195f354edbcae547bf89126a27466aafab923688 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 11 Dec 2023 00:14:46 -0800 Subject: [PATCH 3/6] [SQL] Changelog for UDFs Signed-off-by: Mihai Budiu --- CHANGELOG.md | 2 ++ sql-to-dbsp-compiler/lib/sqllib/src/source.rs | 14 ++++++++++---- sql-to-dbsp-compiler/lib/sqllib/src/types.rs | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4063b7b17cd..1496a87d18d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - SQL: support for trigonometric functions `sin` and `cos` ([#1118](https://github.com/feldera/feldera/pull/1118)) - SQL: support for mathematical constant `PI` ([#1123](https://github.com/feldera/feldera/pull/1123)) - WebConsole: 'Inspect connector' button in the connector list in Pipeline Builder that opens a non-editable popup +- SQL: Support for user-defined functions, declared in SQL and implemented + in Rust ([#1129](https://github.com/feldera/feldera/pull/1129)) ## [0.5.0] - 2023-12-05 diff --git a/sql-to-dbsp-compiler/lib/sqllib/src/source.rs b/sql-to-dbsp-compiler/lib/sqllib/src/source.rs index a7af788dea0..225b2bd8eb4 100644 --- a/sql-to-dbsp-compiler/lib/sqllib/src/source.rs +++ b/sql-to-dbsp-compiler/lib/sqllib/src/source.rs @@ -33,8 +33,11 @@ impl SourcePosition { impl fmt::Display for SourcePosition { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - if self.isValid() { write!(f, "{}:{}", self.line, self.column) } - else { write!(f, "") } + if self.isValid() { + write!(f, "{}:{}", self.line, self.column) + } else { + write!(f, "") + } } } @@ -50,7 +53,10 @@ impl SourcePositionRange { impl fmt::Display for SourcePositionRange { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - if self.isValid() { write!(f, "{}-{}", self.start, self.end) } - else { write!(f, "") } + if self.isValid() { + write!(f, "{}-{}", self.start, self.end) + } else { + write!(f, "") + } } } diff --git a/sql-to-dbsp-compiler/lib/sqllib/src/types.rs b/sql-to-dbsp-compiler/lib/sqllib/src/types.rs index f3a319d2f23..3316c16a4bc 100644 --- a/sql-to-dbsp-compiler/lib/sqllib/src/types.rs +++ b/sql-to-dbsp-compiler/lib/sqllib/src/types.rs @@ -5,5 +5,5 @@ pub use dbsp::algebra::{F32, F64}; pub use crate::{ Date, GeoPoint, LongInterval, ShortInterval, SourcePosition, SourcePositionRange, Time, - Timestamp + Timestamp, }; From 528e8ae75c0b34148f8e6903e111d883cd339534 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 11 Dec 2023 00:22:27 -0800 Subject: [PATCH 4/6] [SQL] Minor fixes from self-review Signed-off-by: Mihai Budiu --- .../calciteCompiler/CalciteCompiler.java | 2 +- .../sqlCompiler/ir/type/DBSPTypeResult.java | 23 ------------------- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteCompiler.java index 5ff0cf8e56c..a4bbfb36060 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteCompiler.java @@ -794,7 +794,7 @@ public SqlFunction compileFunction(SqlNode node) { } /** - * Compile a SQL statement. Returns null if the statement does not require further processing. + * Compile a SQL statement. * @param node Compiled version of the SQL statement. * @param sqlStatement SQL statement as a string to compile. * @param comment Additional information about the compiled statement. diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/DBSPTypeResult.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/DBSPTypeResult.java index 455128d3329..a7fb8f69f4e 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/DBSPTypeResult.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/DBSPTypeResult.java @@ -1,26 +1,3 @@ -/* - * Copyright 2022 VMware, Inc. - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - package org.dbsp.sqlCompiler.ir.type; import org.dbsp.sqlCompiler.compiler.frontend.CalciteObject; From 80c89d55989d63000dbf9260ae7585468d622a01 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Mon, 11 Dec 2023 05:31:43 -0800 Subject: [PATCH 5/6] Address review comments Signed-off-by: Mihai Budiu --- docs/sql/udf.md | 62 ++++++++++--------- .../compiler/frontend/ExpressionCompiler.java | 9 +-- .../calciteCompiler/CalciteCompiler.java | 2 +- .../sqlCompiler/compiler/sql/OtherTests.java | 2 +- .../lib/sqllib/src/geopoint.rs | 22 +------ sql-to-dbsp-compiler/lib/sqllib/src/lib.rs | 2 - sql-to-dbsp-compiler/lib/sqllib/src/types.rs | 9 --- 7 files changed, 37 insertions(+), 71 deletions(-) delete mode 100644 sql-to-dbsp-compiler/lib/sqllib/src/types.rs diff --git a/docs/sql/udf.md b/docs/sql/udf.md index 8ceefb5b50c..eb6c6a4697e 100644 --- a/docs/sql/udf.md +++ b/docs/sql/udf.md @@ -21,7 +21,7 @@ canonical function name. Here is a possible implementation of the function `contains_number` above: ```rs -use sqllib::types::*; +use sqllib::*; pub fn CONTAINS_NUMBER(pos: &SourcePositionRange, str: String, value: Option) -> Result> { @@ -40,8 +40,8 @@ Notice the function implemented has an all-capitals name (which is not a standard convention for Rust), dictated by the default SQL capitalization rules. -There is no type casting or type inference performed for the function -arguments in the SQL program. For example, a call such as +Currently there is no type casting or type inference performed for the +function arguments in the SQL program. For example, a call such as `CONTAINS_NUMBER('2010-10-20', '5')` will fail at SQL compilation time because the first argument has type `CHAR(8)` instead of `VARCHAR`, and the second argument has type `CHAR(1)` instead of `INTEGER`. @@ -60,23 +60,23 @@ Rust types, but are defined in the DBSP runtime library. SQL | Rust ----------- -BOOLEAN | bool -TINYINT | i8 -SMALLINT | i16 -INT | i32 -BIGINT | i64 -DECIMAL(p, s) | Decimal -REAL | F32 -DOUBLE | F64 -CHAR(n) | String -VARCHAR, VARCHAR(n) | String -BINARY, BINARY(n) | ByteArray -NULL | () -INTERVAL | ShortInterval, LongInterval -TIME | Time -TIMESTAMP | Timestamp -DATE | Date -T ARRAY | Vec +`BOOLEAN` | `bool` +`TINYINT` | `i8` +`SMALLINT` | `i16` +`INT` | `i32` +`BIGINT` | `i64` +`DECIMAL`(p, s) | `Decimal` +`REAL` | `F32` +`DOUBLE` | `F64` +`CHAR`(n) | `String` +`VARCHAR`, `VARCHAR`(n) | `String` +`BINARY`, `BINARY`(n) | `ByteArray` +`NULL` | `()` +`INTERVAL` | `ShortInterval`, `LongInterval` +`TIME` | `Time` +`TIMESTAMP` | `Timestamp` +`DATE` | `Date` +`ARRAY T` | `Vec` Multiple SQL types may be represented by the same Rust type. For example, `CHAR`, `CHAR(n)`, `VARCHAR(n)`, and `VARCHAR` are all @@ -85,26 +85,28 @@ represented by the standard Rust `String` type. The SQL family of `INTERVAL` types translates to one of two Rust types: `ShortInterval` (representing intervals from days to seconds), and `LongInterval` (representing intervals from years to months). -(Our dialect of SQL does allow mixing the two kinds of intervals in a -single expression.) +(Our dialect of SQL does not allow mixing the two kinds of intervals +in a single expression.) In the Rust implementation the function always has to return the type `Result>`, where `T` is the Rust -equivalent expected return type of the SQL function. The Rust +equivalent of the expected return type of the SQL function. The Rust function should return an `Err` only when the function fails at runtime; in this case the returned error can use the source position information to indicate where the error has originated in the code. +The function should return an error only for fatal conditions, similar +to other SQL functions (e.g., array index out of bounds, arithmetic +overflows, etc.). ## Source position information The first argument passed to the Rust function is always `pos: -&SourcePositionRange`. This argument indicates where in the source -position is placed the code that generated the call to this -user-defined function. This information can be used to generate -better runtime error messages when the user-defined function -encounters an error. (Note: currently Calcite does not provide any -source position information, but we hope to remedy this state of -affairs soon.) +&SourcePositionRange`. This argument indicates the position in the +SQL source code of the call to this user-defined function. This +information can be used to generate better runtime error messages when +the user-defined function encounters an error. (Note: currently +Calcite does not provide any source position information, but we hope +to remedy this state of affairs soon.) ## Limitations diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java index 6dc0ba316c1..2e2f38d0f4c 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java @@ -646,14 +646,7 @@ node, new DBSPTypeBool(CalciteObject.EMPTY, false), DBSPOpcode.EQ, return result; } case ST_POINT: { - if (ops.size() != 2) - throw new UnimplementedException("Expected only 2 operands", node); - DBSPExpression left = ops.get(0); - DBSPExpression right = ops.get(1); - String functionName = "make_geopoint" + type.nullableSuffix() + - "_" + left.getType().baseTypeWithSuffix() + - "_" + right.getType().baseTypeWithSuffix(); - return new DBSPApplyExpression(node, functionName, type, left, right); + return this.compilePolymorphicFunction(call, node, type, ops, 2); } case OTHER_FUNCTION: { String opName = call.op.getName().toLowerCase(); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteCompiler.java index a4bbfb36060..50184d26e79 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteCompiler.java @@ -794,7 +794,7 @@ public SqlFunction compileFunction(SqlNode node) { } /** - * Compile a SQL statement. + * Compile a SQL statement. * @param node Compiled version of the SQL statement. * @param sqlStatement SQL statement as a string to compile. * @param comment Additional information about the compiled statement. diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/OtherTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/OtherTests.java index d449e74c2e9..c1c785e8614 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/OtherTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/OtherTests.java @@ -370,7 +370,7 @@ public void testUDF() throws IOException, InterruptedException { File implementation = File.createTempFile("impl", ".rs", new File(rustDirectory)); createInputFile(implementation, System.lineSeparator(), - "use sqllib::types::*;", + "use sqllib::*;", "pub fn CONTAINS_NUMBER(pos: &SourcePositionRange, str: String, value: Option) -> " + " Result> {", " match value {", diff --git a/sql-to-dbsp-compiler/lib/sqllib/src/geopoint.rs b/sql-to-dbsp-compiler/lib/sqllib/src/geopoint.rs index 28bafba0da0..bc4cc5a8555 100644 --- a/sql-to-dbsp-compiler/lib/sqllib/src/geopoint.rs +++ b/sql-to-dbsp-compiler/lib/sqllib/src/geopoint.rs @@ -50,29 +50,11 @@ impl GeoPoint { } } -pub fn make_geopoint_d_d(left: F64, right: F64) -> GeoPoint { +pub fn st_point_d_d(left: F64, right: F64) -> GeoPoint { GeoPoint::new(left, right) } -pub fn make_geopointN_d_d(left: F64, right: F64) -> Option { - Some(make_geopoint_d_d(left, right)) -} - -pub fn make_geopointN_dN_d(left: Option, right: F64) -> Option { - left.map(|x| make_geopoint_d_d(x, right)) -} - -pub fn make_geopointN_d_dN(left: F64, right: Option) -> Option { - right.map(|x| make_geopoint_d_d(left, x)) -} - -pub fn make_geopointN_dN_dN(left: Option, right: Option) -> Option { - match (left, right) { - (None, _) => None, - (_, None) => None, - (Some(x), Some(y)) => make_geopointN_d_d(x, y), - } -} +some_polymorphic_function2!(st_point, d, F64, d, F64, GeoPoint); pub fn st_distance_geopoint_geopoint(left: GeoPoint, right: GeoPoint) -> F64 { left.distance(&right) diff --git a/sql-to-dbsp-compiler/lib/sqllib/src/lib.rs b/sql-to-dbsp-compiler/lib/sqllib/src/lib.rs index 58b8a3b905c..a1fab0dc85c 100644 --- a/sql-to-dbsp-compiler/lib/sqllib/src/lib.rs +++ b/sql-to-dbsp-compiler/lib/sqllib/src/lib.rs @@ -9,9 +9,7 @@ pub mod operators; pub mod source; pub mod string; pub mod timestamp; -pub mod types; -// Types re-exported by types.rs pub use geopoint::GeoPoint; pub use interval::LongInterval; pub use interval::ShortInterval; diff --git a/sql-to-dbsp-compiler/lib/sqllib/src/types.rs b/sql-to-dbsp-compiler/lib/sqllib/src/types.rs deleted file mode 100644 index 3316c16a4bc..00000000000 --- a/sql-to-dbsp-compiler/lib/sqllib/src/types.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! Crate which just re-exports all the types that are used to implement -//! the types that appear in SQL. - -pub use dbsp::algebra::{F32, F64}; - -pub use crate::{ - Date, GeoPoint, LongInterval, ShortInterval, SourcePosition, SourcePositionRange, Time, - Timestamp, -}; From dec67ba8a41b81940283975a53246a7e46d1f5a3 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Tue, 12 Dec 2023 14:24:05 -0800 Subject: [PATCH 6/6] [SQL] Revert incorrect change to st_point function Signed-off-by: Mihai Budiu --- CONTRIBUTING.md | 3 ++- .../SQL-compiler/out7670973118205954433.tmp | 0 .../compiler/frontend/ExpressionCompiler.java | 16 ++++++++++++-- .../compiler/sql/ComplexQueriesTest.java | 10 +++++---- .../compiler/sql/simple/EndToEndTests.java | 2 +- .../lib/sqllib/src/geopoint.rs | 22 +++++++++++++++++-- 6 files changed, 43 insertions(+), 10 deletions(-) create mode 100644 sql-to-dbsp-compiler/SQL-compiler/out7670973118205954433.tmp diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 79b089dba22..8f01dd1c333 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,7 +13,7 @@ with `git commit -s`. Our team develops and tests using Linux and MacOS. Windows Subsystem for Linux works fine too. The Feldera container images and CI workflows use Linux. You can see our setup in -our [Dockerfile](deploy/Dockerfile) and [Earthfile](Earthfile) . +our [Dockerfile](deploy/Dockerfile) and [Earthfile](Earthfile). Our known dependencies are: - Runtime @@ -29,6 +29,7 @@ Our known dependencies are: - Python 3 - typescript - Redpanda or Kafka + - Earthly (https://earthly.dev/get-earthly) Additional dependencies are automatically installed by the Rust, maven, Python, and typescript build tools. diff --git a/sql-to-dbsp-compiler/SQL-compiler/out7670973118205954433.tmp b/sql-to-dbsp-compiler/SQL-compiler/out7670973118205954433.tmp new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java index 2e2f38d0f4c..da69ea7594f 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java @@ -394,7 +394,9 @@ String getCallName(RexCall call) { * depending on the argument types. * @param call Operation that is compiled. * @param node CalciteObject holding the call. - * @param resultType Type of result produced by call. + * @param resultType Type of result produced by call. We assume that + * the typechecker is right, and this is the correct + * result produced by this function. No cast needed. * @param ops Translated operands for the call. * @param expectedArgCount A list containing all known possible argument counts. */ @@ -646,7 +648,17 @@ node, new DBSPTypeBool(CalciteObject.EMPTY, false), DBSPOpcode.EQ, return result; } case ST_POINT: { - return this.compilePolymorphicFunction(call, node, type, ops, 2); + // Sometimes the Calcite type for ST_POINT is nullable + // even if all arguments are not nullable. So we can't + // just use compilePolymorphicFunction. + if (ops.size() != 2) + throw new UnimplementedException("Expected only 2 operands", node); + DBSPExpression left = ops.get(0); + DBSPExpression right = ops.get(1); + String functionName = "make_geopoint" + type.nullableSuffix() + + "_" + left.getType().baseTypeWithSuffix() + + "_" + right.getType().baseTypeWithSuffix(); + return new DBSPApplyExpression(node, functionName, type, left, right); } case OTHER_FUNCTION: { String opName = call.op.getName().toLowerCase(); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/ComplexQueriesTest.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/ComplexQueriesTest.java index 11f3e7a4a12..5843d72cae5 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/ComplexQueriesTest.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/ComplexQueriesTest.java @@ -80,6 +80,7 @@ public void testDateDiff() { this.addRustTestCase("ComplexQueriesTest.testDateDiff", compiler, getCircuit(compiler)); } + /* @Test @Ignore("Not yet tested") public void testCrossApply() { String query = " select d.DocumentID, ds.Status, ds.DateCreated \n" + @@ -90,6 +91,7 @@ public void testCrossApply() { " where DocumentID = d.DocumentId\n" + " order by DateCreated desc) as ds"; } + */ @Test public void smallTaxiTest() { @@ -468,8 +470,8 @@ public void demographicsTest() { " amt FLOAT64,\n" + " trans_num STRING,\n" + " unix_time INTEGER NOT NULL,\n" + - " merch_lat FLOAT64,\n" + - " merch_long FLOAT64,\n" + + " merch_lat FLOAT64 NOT NULL,\n" + + " merch_long FLOAT64 NOT NULL,\n" + " is_fraud INTEGER\n" + ");\n" + "\n" + @@ -538,8 +540,8 @@ public void demographicsTest() { new DBSPDoubleLiteral(10.0, true), new DBSPStringLiteral("Transnum", true), new DBSPI32Literal(1000), - new DBSPDoubleLiteral(128.0, true), - new DBSPDoubleLiteral(128.0, true), + new DBSPDoubleLiteral(128.0), + new DBSPDoubleLiteral(128.0), new DBSPI32Literal(0, true) )) }; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/EndToEndTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/EndToEndTests.java index c1580c8e270..690b85d6d95 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/EndToEndTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/EndToEndTests.java @@ -559,7 +559,7 @@ public void geoPointTest() { public void geoDistanceTest() { String query = "SELECT ST_DISTANCE(ST_POINT(0, 0), ST_POINT(0,1))"; this.testConstantOutput(query, new DBSPZSetLiteral.Contents( - new DBSPTupleExpression(new DBSPDoubleLiteral(1).some()))); + new DBSPTupleExpression(new DBSPDoubleLiteral(1.0, true)))); } @Test diff --git a/sql-to-dbsp-compiler/lib/sqllib/src/geopoint.rs b/sql-to-dbsp-compiler/lib/sqllib/src/geopoint.rs index bc4cc5a8555..28bafba0da0 100644 --- a/sql-to-dbsp-compiler/lib/sqllib/src/geopoint.rs +++ b/sql-to-dbsp-compiler/lib/sqllib/src/geopoint.rs @@ -50,11 +50,29 @@ impl GeoPoint { } } -pub fn st_point_d_d(left: F64, right: F64) -> GeoPoint { +pub fn make_geopoint_d_d(left: F64, right: F64) -> GeoPoint { GeoPoint::new(left, right) } -some_polymorphic_function2!(st_point, d, F64, d, F64, GeoPoint); +pub fn make_geopointN_d_d(left: F64, right: F64) -> Option { + Some(make_geopoint_d_d(left, right)) +} + +pub fn make_geopointN_dN_d(left: Option, right: F64) -> Option { + left.map(|x| make_geopoint_d_d(x, right)) +} + +pub fn make_geopointN_d_dN(left: F64, right: Option) -> Option { + right.map(|x| make_geopoint_d_d(left, x)) +} + +pub fn make_geopointN_dN_dN(left: Option, right: Option) -> Option { + match (left, right) { + (None, _) => None, + (_, None) => None, + (Some(x), Some(y)) => make_geopointN_d_d(x, y), + } +} pub fn st_distance_geopoint_geopoint(left: GeoPoint, right: GeoPoint) -> F64 { left.distance(&right)