From 5bf7e947dc19d9342bbe340bccb0b981d7b0be46 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Sun, 28 Jun 2026 17:54:17 -0700 Subject: [PATCH] [SQL] Compiler option --reformat to pretty-print SQL Signed-off-by: Mihai Budiu --- crates/sqllib/src/timestamp.rs | 40 +- .../src/main/codegen/includes/parserImpls.ftl | 2 +- .../org/dbsp/sqlCompiler/CompilerMain.java | 2 +- .../sqlCompiler/compiler/CompilerOptions.java | 4 + .../sqlCompiler/compiler/DBSPCompiler.java | 67 +- .../compiler/frontend/ExpressionCompiler.java | 32 + .../compiler/frontend/SqlComment.java | 32 + .../compiler/frontend/SqlCommentParser.java | 194 ++++++ .../compiler/frontend/SqlPrettyPrinter.java | 579 ++++++++++++++++++ .../calciteCompiler/SqlToRelCompiler.java | 12 + .../frontend/parser/SqlCreateAggregate.java | 3 + .../parser/SqlCreateFunctionDeclaration.java | 3 + .../compiler/frontend/parser/SqlLateness.java | 1 + .../literal/DBSPKeywordLiteral.java | 26 +- .../java/org/dbsp/util/IIndentStream.java | 3 + .../main/java/org/dbsp/util/IndentStream.java | 3 + .../java/org/dbsp/util/NullIndentStream.java | 3 + .../main/java/org/dbsp/util/Utilities.java | 20 + .../compiler/sql/MetadataTests.java | 269 ++++++++ .../sqlCompiler/compiler/sql/QATests.java | 2 +- .../sql/mysql/TimestampDiffTests.java | 2 + sql-to-dbsp-compiler/calcite_version.env | 2 +- sql-to-dbsp-compiler/using.md | 3 + 23 files changed, 1266 insertions(+), 38 deletions(-) create mode 100644 sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/SqlComment.java create mode 100644 sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/SqlCommentParser.java create mode 100644 sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/SqlPrettyPrinter.java diff --git a/crates/sqllib/src/timestamp.rs b/crates/sqllib/src/timestamp.rs index 2777bee5b78..3b7e3d8554b 100644 --- a/crates/sqllib/src/timestamp.rs +++ b/crates/sqllib/src/timestamp.rs @@ -2866,13 +2866,51 @@ some_polymorphic_function1!(extract_second, Date, Date, i64); some_polymorphic_function1!(extract_minute, Date, Date, i64); some_polymorphic_function1!(extract_hour, Date, Date, i64); +#[doc(hidden)] +pub fn datediff_hour_Date_Date(left: Date, right: Date) -> i32 { + (right.days() - left.days()) * 24 +} + +some_polymorphic_function2!(datediff_hour, Date, Date, Date, Date, i32); + #[doc(hidden)] pub fn datediff_day_Date_Date(left: Date, right: Date) -> i32 { - left.days() - right.days() + right.days() - left.days() } some_polymorphic_function2!(datediff_day, Date, Date, Date, Date, i32); +#[doc(hidden)] +pub fn datediff_week_Date_Date(left: Date, right: Date) -> i32 { + (right.days() - left.days()) / 7 +} + +some_polymorphic_function2!(datediff_week, Date, Date, Date, Date, i32); + +#[doc(hidden)] +pub fn datediff_month_Date_Date(left: Date, right: Date) -> i32 { + let interval = minus_LongInterval_Date_Date__(right, left); + interval.months() +} + +some_polymorphic_function2!(datediff_month, Date, Date, Date, Date, i32); + +#[doc(hidden)] +pub fn datediff_quarter_Date_Date(left: Date, right: Date) -> i32 { + datediff_year_Date_Date(left, right) * 4 + + (extract_quarter_Date(right) - extract_quarter_Date(left)) as i32 +} + +some_polymorphic_function2!(datediff_quarter, Date, Date, Date, Date, i32); + +#[doc(hidden)] +pub fn datediff_year_Date_Date(left: Date, right: Date) -> i32 { + let interval = minus_LongInterval_Date_Date__(right, left); + interval.years() +} + +some_polymorphic_function2!(datediff_year, Date, Date, Date, Date, i32); + #[doc(hidden)] pub fn format_date__(format: SqlString, date: Date) -> SqlString { SqlString::from(date.to_dateTime().format(format.str()).to_string()) diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/includes/parserImpls.ftl b/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/includes/parserImpls.ftl index e24962d33f0..90ab6f19830 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/includes/parserImpls.ftl +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/codegen/includes/parserImpls.ftl @@ -36,7 +36,7 @@ SqlNode DateaddFunctionCall() : { ( { op = SqlLibraryOperators.DATE_PART; } | { op = SqlLibraryOperators.DATEADD; } - | { op = SqlStdOperatorTable.TIMESTAMP_DIFF; } + | { op = SqlLibraryOperators.DATEDIFF; } | { op = SqlLibraryOperators.DATEPART; } ) { s = span(); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/CompilerMain.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/CompilerMain.java index 8f67060514c..c49ef2abc89 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/CompilerMain.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/CompilerMain.java @@ -175,7 +175,7 @@ CompilerMessages run() { return compiler.messages; // The following runs all compilation stages DBSPCircuit circuit = compiler.getFinalCircuit(false); - if (options.ioOptions.anonymize) + if (options.ioOptions.anonymize || options.ioOptions.format) // The front-end has already emitted the output, we are done. return compiler.messages; if (compiler.hasErrors()) diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/CompilerOptions.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/CompilerOptions.java index 55cbe5246e8..59c1c4f3d97 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/CompilerOptions.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/CompilerOptions.java @@ -159,6 +159,9 @@ public String diff(CompilerOptions other) { public static class IO implements IDiff, IValidate { @Parameter(names = "--anonymize", description = "Produce in the output file an anonymized version of the input program") public boolean anonymize = false; + @Parameter(names = "--format", + description = "Output the SQL program reformatted") + public boolean format = false; @DynamicParameter(names = "-T", description = "Specify logging level for a class (can be repeated)") public Map loggingLevel = new HashMap<>(); @@ -278,6 +281,7 @@ public String toString() { ",\n\tnoRust=" + this.noRust + ",\n\toutputFile=" + Utilities.singleQuote(this.outputFile) + ",\n\tquiet=" + this.quiet + + ",\n\tformat=" + this.format + ",\n\truntime=" + Utilities.singleQuote(this.runtimePath) + ",\n\ttrimInputs=" + this.trimInputs + ",\n\tverbosity=" + this.verbosity + diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java index 6d9bd146b2d..1a24181368f 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java @@ -39,6 +39,9 @@ import org.apache.calcite.sql.dialect.CalciteSqlDialect; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.pretty.SqlPrettyWriter; +import org.dbsp.sqlCompiler.compiler.frontend.SqlComment; +import org.dbsp.sqlCompiler.compiler.frontend.SqlCommentParser; +import org.dbsp.sqlCompiler.compiler.frontend.SqlPrettyPrinter; import org.apache.calcite.sql.util.SqlOperatorTables; import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction; import org.dbsp.sqlCompiler.circuit.DBSPCircuit; @@ -98,7 +101,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.List; -import java.util.Locale; +import java.util.Objects; import java.util.Map; import java.util.regex.Pattern; @@ -532,7 +535,7 @@ boolean validateCreateIndex(CreateIndexStatement statement) { return true; } - void renameIdentifiers(List statements) { + void emitSql(List statements) { final PrintStream outputStream; try { @Nullable String outputFile = this.options.ioOptions.outputFile; @@ -547,20 +550,36 @@ void renameIdentifiers(List statements) { return; } + List comments = this.options.ioOptions.format + ? SqlCommentParser.parse(this.sources.getWholeProgram()) + : new ArrayList<>(); + IndentStream indentStream = new IndentStream(outputStream); + SqlPrettyPrinter printer = new SqlPrettyPrinter(indentStream, comments); + RenameIdentifiers renamer = new RenameIdentifiers(); - for (ParsedStatement stat: statements) { + for (ParsedStatement stat : statements) { if (stat.visible()) { - SqlNode renamed = renamer.visitNode(stat.statement()); - Utilities.enforce(renamed != null); - final SqlWriter sqlWriter = new SqlPrettyWriter( - SqlWriterConfig.of() - .withDialect(CalciteSqlDialect.DEFAULT) - .withQuoteAllIdentifiers(false) - ); - renamed.unparse(sqlWriter, 0, 0); - outputStream.println(sqlWriter + ";"); + SqlNode node = stat.statement(); + if (this.options.ioOptions.anonymize) + node = Objects.requireNonNull(renamer.visitNode(node)); + Utilities.enforce(node != null); + if (this.options.ioOptions.format) { + printer.print(node); + indentStream.append(";").newline(); + } else { + final SqlWriter sqlWriter = new SqlPrettyWriter( + SqlWriterConfig.of() + .withDialect(CalciteSqlDialect.DEFAULT) + .withQuoteAllIdentifiers(false) + ); + node.unparse(sqlWriter, 0, 0); + outputStream.println(sqlWriter + ";"); + } } } + if (this.options.ioOptions.format) { + printer.flushRemainingComments(); + } outputStream.close(); } @@ -569,8 +588,8 @@ void renameIdentifiers(List statements) { if (this.hasErrors()) return null; - if (this.options.ioOptions.anonymize) { - this.renameIdentifiers(parsed); + if (this.options.ioOptions.anonymize || this.options.ioOptions.format) { + this.emitSql(parsed); return null; } @@ -749,7 +768,8 @@ void renameIdentifiers(List statements) { return null; } - static final Pattern ITEM_ERROR = Pattern.compile("Cannot apply 'ITEM' to arguments of type 'ITEM\\(([^,]+), ([^']+)\\)'(.*)", Pattern.DOTALL); + static final Pattern ITEM_ERROR = Pattern.compile( + "Cannot apply 'ITEM' to arguments of type 'ITEM\\(([^,]+), ([^']+)\\)'(.*)", Pattern.DOTALL); static SourcePositionRange getRange(CalciteContextException e) { return new SourcePositionRange( @@ -769,23 +789,6 @@ private CompilationError improveErrorMessage(CalciteContextException e) { String newMessage = "Cannot apply indexing to arguments of type " + source + "[" + index + "]" + tail; return new CompilationError(newMessage, getRange(e)); } - if (message.contains("'TIMESTAMPDIFF'.")) { - // The Calcite parser replaces DATEDIFF with TIMESTAMPDIFF - // Try to figure out whether this is what happened and rewrite it. - // This heuristic may fail... but will work in general. - SourcePositionRange range = getRange(e); - String fragment = this.sources.getFragment(range, false); - String lower = fragment.toLowerCase(Locale.ENGLISH); - if (lower.contains("datediff") && !lower.contains("timestampdiff")) { - message = message.replace("TIMESTAMPDIFF", "DATEDIFF"); - return new CompilationError(message, range); - } - } - String newMessage = message.replace(":PEEK_NO_EXPAND", ""); - newMessage = newMessage.replace("RECORDTYPE", "ROW"); - if (!newMessage.equals(message)) { - return new CompilationError(newMessage, getRange(e)); - } } return new CompilationError(e); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java index ef59edbf0bc..39b08a50299 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java @@ -107,6 +107,7 @@ import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeBaseType; import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeDate; import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeFP; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeKeyword; import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeTime; import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeTimestampTz; import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeVariant; @@ -142,6 +143,7 @@ import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Objects; import java.util.UUID; @@ -818,6 +820,18 @@ void ensureInteger(CalciteObject node, List ops, int argument) { ops.set(argument, arg.cast(node, expected, DBSPCastExpression.CastType.SqlUnsafe)); } + /** Ensure that a specific operand is interpreted as a KEYWORD */ + void ensureKeyword(CalciteObject node, List ops, int argument) { + DBSPExpression arg = ops.get(argument); + if (arg.getType().sameType(DBSPTypeKeyword.INSTANCE)) + return; + if (arg.is(DBSPStringLiteral.class)) { + ops.set(argument, new DBSPKeywordLiteral(node, Objects.requireNonNull(arg.to(DBSPStringLiteral.class).value))); + return; + } + throw new CompilationError("Unknown keyword", node); + } + @SuppressWarnings("SameParameterValue") void nullLiteralToNullArray(List ops, int arg) { if (ops.get(arg).is(DBSPNullLiteral.class)) { @@ -1934,6 +1948,24 @@ else if (arg0Type.is(DBSPTypeMap.class)) this.ensureString(ops, 2); return compileKeywordFunction(call, node, null, type, ops, 0, 3); case TIMESTAMP_DIFF: + // Also called for DATEDIFF + this.ensureKeyword(node, ops, 0); + DBSPKeywordLiteral lit = ops.get(0).to(DBSPKeywordLiteral.class); + if (call.op.getName().toLowerCase(Locale.ENGLISH).equals("datediff")) { + switch (lit.keyword) { + // These units would overflow in 32 bits + case "minute": + case "second": + case "millisecond": + case "microsecond": + case "nanosecond": + throw new CompilationError("Use TIMESTAMPDIFF instead of DATEDIFF for " + + lit.keyword.toUpperCase(Locale.ENGLISH) + + "\n(TIMESTAMPDIFF result is BIGINT)", node); + default: + break; + } + } return compileKeywordFunction(call, node, null, type, ops, 0, 3); case TUMBLE: { validateArgCount(node, operationName, ops.size(), 2, 3); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/SqlComment.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/SqlComment.java new file mode 100644 index 00000000000..40eefd0d16d --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/SqlComment.java @@ -0,0 +1,32 @@ +package org.dbsp.sqlCompiler.compiler.frontend; + +import org.apache.calcite.sql.parser.SqlParserPos; + +/** A SQL comment extracted from the original source text, with its source position. */ +public class SqlComment { + public enum Kind { + /** {@code -- ...} to end of line (SQL standard). */ + LINE_DASH, + /** {@code // ...} to end of line (C++ style). */ + LINE_SLASH, + /** {@code /* ... *\/} possibly spanning multiple lines. */ + BLOCK + } + + public final Kind kind; + /** Full comment text, including delimiters (e.g. {@code -- foo\n} or {@code /* bar *\/}). */ + public final String text; + /** Position of the first character of the comment in the original source. */ + public final SqlParserPos pos; + + public SqlComment(Kind kind, String text, SqlParserPos pos) { + this.kind = kind; + this.text = text; + this.pos = pos; + } + + @Override + public String toString() { + return pos + ": " + text.trim(); + } +} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/SqlCommentParser.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/SqlCommentParser.java new file mode 100644 index 00000000000..67a82237eae --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/SqlCommentParser.java @@ -0,0 +1,194 @@ +package org.dbsp.sqlCompiler.compiler.frontend; + +import org.apache.calcite.sql.parser.SqlParserPos; + +import java.util.ArrayList; +import java.util.List; + +import static java.util.Objects.requireNonNull; + +/** + * Scans SQL source text for comments, recording each comment's text and + * position in the original source. + * + *

Correctly skips string literals ({@code '...'}), double-quoted + * identifiers ({@code "..."}), and backtick-quoted identifiers so that + * comment-like sequences inside them are not mistaken for real comments. + * + *

Recognised comment forms: + *

    + *
  • {@code --} to end of line (SQL standard single-line comment)
    • + *
  • {@code //} to end of line (C++-style; also accepted by Calcite)
    • + *
  • {@code /* ... *\/} (block comment, may span multiple lines)
    • + *
+ * + *

Calcite query hints ({@code /*+ ... *\/}) are not included in the + * output: they are already represented in the {@code SqlNode} tree and handled + * separately by {@link SqlPrettyPrinter}. + * + *

The input is assumed to be syntactically valid SQL (as accepted by + * Calcite). Unterminated block comments or string literals are consumed + * silently to end-of-input; the resulting output may be malformed. + */ +public class SqlCommentParser { + private final String sql; // source text being scanned + private final int n; // cached length of sql + private int i; // scan cursor (index into sql) + private int line; // current line number, 1-based + private int col; // current column number, 1-based + + private SqlCommentParser(final String sql) { + this.sql = sql; + this.n = sql.length(); + this.i = 0; + this.line = 1; + this.col = 1; + } + + /** Extract all comments from {@code sql}, in source order. */ + public static List parse(final String sql) { + return new SqlCommentParser(sql).scan(); + } + + private List scan() { + final List result = new ArrayList<>(); + + while (this.i < this.n) { + final char c = this.current(); + + // single-line comment: -- or // + if ((c == '-' && this.peekNext() == '-') + || (c == '/' && this.peekNext() == '/')) { + final SqlComment.Kind kind = (c == '-') ? SqlComment.Kind.LINE_DASH : SqlComment.Kind.LINE_SLASH; + final int start = this.i; + final SqlParserPos pos = new SqlParserPos(this.line, this.col); + this.nextChar(); // consume first marker char + this.nextChar(); // consume second marker char + while (this.i < this.n && this.current() != '\n' && this.current() != '\r') { + this.nextChar(); + } + if (this.i < this.n) { + this.nextChar(); // consume line terminator; \r\n handled as one unit + } + result.add(new SqlComment(kind, requireNonNull(this.sql.substring(start, this.i)), pos)); + continue; + } + + // block comment: /* ... */ + // Calcite query hints /*+ ... */ are not collected: they are already + // represented in the SqlNode tree and emitted by SqlPrettyPrinter separately. + if (c == '/' && this.peekNext() == '*') { + final int start = this.i; + final SqlParserPos pos = new SqlParserPos(this.line, this.col); + this.nextChar(); // consume '/' + this.nextChar(); // consume '*' + final boolean isHint = this.i < this.n && this.current() == '+'; + while (this.i < this.n) { + if (this.current() == '*' && this.peekNext() == '/') { + this.nextChar(); // consume '*' + this.nextChar(); // consume '/' + break; + } + this.nextChar(); + } + if (!isHint) { + result.add(new SqlComment(SqlComment.Kind.BLOCK, requireNonNull(this.sql.substring(start, this.i)), pos)); + } + continue; + } + + // single-quoted string literal: '...' + if (c == '\'') { + this.nextChar(); // consume opening ' + while (this.i < this.n) { + if (this.current() == '\'') { + this.nextChar(); // consume ' + if (this.i < this.n && this.current() == '\'') { + this.nextChar(); // '' is an escaped single quote; keep scanning + } else { + break; + } + } else { + this.nextChar(); + } + } + continue; + } + + // double-quoted identifier: "..." + if (c == '"') { + this.nextChar(); // consume opening " + while (this.i < this.n) { + if (this.current() == '"') { + this.nextChar(); // consume " + if (this.i < this.n && this.current() == '"') { + this.nextChar(); // "" is an escaped double-quote; keep scanning + } else { + break; + } + } else { + this.nextChar(); + } + } + continue; + } + + // backtick-quoted identifier: `...` + if (c == '`') { + this.nextChar(); // consume opening ` + while (this.i < this.n) { + if (this.current() == '`') { + this.nextChar(); // consume ` + if (this.i < this.n && this.current() == '`') { + this.nextChar(); // `` is an escaped backtick; keep scanning + } else { + break; + } + } else { + this.nextChar(); + } + } + continue; + } + + // ordinary character + this.nextChar(); + } + + return result; + } + + /** Returns the character at the current scan position; caller must ensure {@code i < n}. */ + private char current() { + return this.sql.charAt(this.i); + } + + /** Returns the character one position ahead of the cursor, or {@code 0} if out of bounds. */ + private char peekNext() { + final int next = this.i + 1; + return next < this.n ? this.sql.charAt(next) : 0; + } + + /** + * Consumes and returns the current character, advancing the cursor and + * updating line/column tracking. A {@code \r\n} pair counts as one line + * ending: this method skips the {@code \n} when the current character is + * {@code \r} and the next is {@code \n}. + */ + private char nextChar() { + final char c = this.sql.charAt(this.i++); + if (c == '\n') { + this.line++; + this.col = 1; + } else if (c == '\r') { + this.line++; + this.col = 1; + if (this.i < this.n && this.sql.charAt(this.i) == '\n') { + this.i++; // skip the \n of a \r\n pair + } + } else { + this.col++; + } + return c; + } +} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/SqlPrettyPrinter.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/SqlPrettyPrinter.java new file mode 100644 index 00000000000..628c952ce01 --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/SqlPrettyPrinter.java @@ -0,0 +1,579 @@ +package org.dbsp.sqlCompiler.compiler.frontend; + +import org.apache.calcite.sql.JoinConditionType; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlJoin; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlOrderBy; +import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.SqlSelectKeyword; +import org.apache.calcite.sql.SqlWith; +import org.apache.calcite.sql.SqlWithItem; +import org.apache.calcite.sql.SqlWriterConfig; +import org.apache.calcite.sql.dialect.CalciteSqlDialect; +import org.apache.calcite.sql.fun.SqlCase; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.dbsp.sqlCompiler.compiler.frontend.parser.SqlCreateTable; +import org.dbsp.sqlCompiler.compiler.frontend.parser.SqlCreateView; +import org.dbsp.util.IIndentStream; +import org.dbsp.util.IndentStream; +import org.dbsp.util.Utilities; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.function.UnaryOperator; + +import static java.util.Objects.requireNonNull; + +/** Formats a {@link SqlNode} tree as SQL text using an {@link IIndentStream}. */ +// Calcite has an unparse method, but it has lots of options and bugs. This printer has no options. +public class SqlPrettyPrinter { + private static final UnaryOperator UNQUOTED_CONFIG = + c -> c.withDialect(CalciteSqlDialect.DEFAULT).withQuoteAllIdentifiers(false); + + /** + * Wraps a sorted list of {@link SqlComment}s with a cursor. + * Given a position, {@link #before(SqlParserPos)} returns every comment + * whose source position strictly precedes that position, then advances + * the cursor past those comments. + */ + private static class CommentStream { + private final List comments; + private int cursor; + + CommentStream(List comments) { + this.comments = comments; + this.cursor = 0; + } + + /** Returns {@code true} if {@code a} appears strictly before {@code b} in the source. */ + private static boolean isBefore(SqlParserPos a, SqlParserPos b) { + int aLine = a.getLineNum(); + int bLine = b.getLineNum(); + if (aLine != bLine) return aLine < bLine; + return a.getColumnNum() < b.getColumnNum(); + } + + /** + * Returns all comments whose source position is strictly before {@code pos}, + * advancing the cursor past them. + */ + List before(SqlParserPos pos) { + List result = new ArrayList<>(); + while (this.cursor < this.comments.size()) { + SqlComment c = requireNonNull(this.comments.get(this.cursor)); + if (!isBefore(requireNonNull(c.pos), pos)) break; + result.add(c); + this.cursor++; + } + return result; + } + + /** Returns all remaining comments and advances the cursor to the end. */ + List remaining() { + List result = new ArrayList<>(this.comments.subList(this.cursor, this.comments.size())); + this.cursor = this.comments.size(); + return result; + } + } + + private final IIndentStream stream; + private final CommentStream commentStream; + + public SqlPrettyPrinter(IIndentStream stream, List comments) { + this.stream = stream; + this.commentStream = new CommentStream(comments); + } + + public SqlPrettyPrinter(IIndentStream stream) { + this(stream, new ArrayList<>()); + } + + /** + * Emit all comments whose source line precedes {@code node}'s start position, + * then advance the comment cursor past them. + */ + private void flushBefore(SqlNode node) { + for (SqlComment comment : this.commentStream.before(requireNonNull(node.getParserPosition()))) { + this.stream.append(requireNonNull(comment.text.stripTrailing())).newline(); + } + } + + /** Emit all remaining comments that follow every printed node. */ + public void flushRemainingComments() { + for (SqlComment comment : this.commentStream.remaining()) { + this.stream.append(requireNonNull(comment.text.stripTrailing())).newline(); + } + } + + /** Format {@code node} as a SQL string. */ + public static String toString(SqlNode node) { + StringBuilder sb = new StringBuilder(); + IndentStream stream = new IndentStream(sb); + new SqlPrettyPrinter(stream).print(node); + return sb.toString(); + } + + public static String toString(SqlNodeList nodes) { + return toString(nodes, List.of()); + } + + /** + * Format a list of SQL statements, interleaving comments from the original source + * at the correct positions — both between statements and inside them (e.g., between + * columns of a CREATE TABLE). + */ + public static String toString(SqlNodeList nodes, List comments) { + StringBuilder sb = new StringBuilder(); + IndentStream stream = new IndentStream(sb); + SqlPrettyPrinter printer = new SqlPrettyPrinter(stream, comments); + for (SqlNode node : nodes) { + printer.print(requireNonNull(node)); + stream.append(";").newline(); + } + printer.flushRemainingComments(); + return sb.toString(); + } + + /** Get the SQL using the Calcite implementation */ + private String fallbackSql(SqlNode node) { + return node.toSqlString(UNQUOTED_CONFIG).getSql(); + } + + /** Append the SQL text for {@code node} to the this.stream. */ + public void print(@Nullable SqlNode node) { + if (node == null) return; + this.flushBefore(node); + + // Feldera-specific DDL nodes (check before SqlCall) + if (node instanceof SqlCreateView) { this.printCreateView((SqlCreateView) node); return; } + if (node instanceof SqlCreateTable) { this.printCreateTable((SqlCreateTable) node); return; } + // Standard Calcite query nodes + if (node instanceof SqlSelect) { this.printSelect((SqlSelect) node); return; } + if (node instanceof SqlOrderBy) { this.printOrderBy((SqlOrderBy) node); return; } + if (node instanceof SqlWith) { this.printWith((SqlWith) node); return; } + if (node instanceof SqlJoin) { this.printJoin((SqlJoin) node); return; } + + if (node instanceof SqlCase) { this.printCase((SqlCase) node); return; } + if (node instanceof SqlBasicCall) { this.printBasicCall((SqlBasicCall) node); return; } + + // Fallback: use Calcite's own unparse without forcing quotes on all identifiers + this.stream.append(this.fallbackSql(node)); + } + + /** + * Print a node that appears inside an expression context. + * A SELECT sub-query is wrapped in parentheses and indented. + */ + private void printExpr(@Nullable SqlNode node) { + if (node == null) return; + SqlKind kind = node.getKind(); + if (node instanceof SqlSelect + || node instanceof SqlOrderBy + || node instanceof SqlWith + || kind == SqlKind.UNION + || kind == SqlKind.INTERSECT + || kind == SqlKind.EXCEPT + || kind == SqlKind.VALUES) { + this.printQueryParens(node); + } else { + this.print(node); + } + } + + private void printQueryParens(SqlNode query) { + this.stream.append("(").increase(); + this.print(query); + this.stream.decrease().newline().append(")"); + } + + /** Print the RHS of an IN / NOT IN expression. + * A value list {@code (1, 2, 3)} is a {@link SqlNodeList} and needs explicit parens. + * A subquery is handled by {@link #printExpr}. */ + private void printInList(SqlNode rhs) { + if (rhs instanceof SqlNodeList list) { + this.stream.append("("); + for (int i = 0; i < list.size(); i++) { + if (i > 0) this.stream.append(", "); + this.printExpr(requireNonNull(list.get(i))); + } + this.stream.append(")"); + } else { + this.printExpr(rhs); + } + } + + private void printSelect(SqlSelect select) { + this.stream.append("SELECT"); + + if (select.hasHints()) { + this.stream.append(" /*+"); + SqlNodeList hints = requireNonNull(select.getHints()); + for (int i = 0; i < hints.size(); i++) { + if (i > 0) this.stream.append(","); + this.stream.append(" ").append(this.fallbackSql(requireNonNull(hints.get(i)))); + } + this.stream.append(" */"); + } + + if (select.isDistinctOn()) { + this.stream.append(" DISTINCT ON ("); + SqlNodeList distinctOn = requireNonNull(select.getDistinctOn()); + for (int i = 0; i < distinctOn.size(); i++) { + if (i > 0) this.stream.append(", "); + this.printExpr(requireNonNull(distinctOn.get(i))); + } + this.stream.append(")"); + } else if (select.isDistinct()) { + this.stream.append(" DISTINCT"); + } else if (select.getModifierNode(SqlSelectKeyword.ALL) != null) { + this.stream.append(" ALL"); + } + + // SELECT list — one item per line, indented + SqlNodeList selectList = select.getSelectList(); + if (selectList != null && !selectList.isEmpty()) { + this.stream.increase(); + for (int i = 0; i < selectList.size(); i++) { + if (i > 0) this.stream.append(",").newline(); + this.printExpr(requireNonNull(selectList.get(i))); + } + this.stream.decrease(); + } + + SqlNode from = select.getFrom(); + if (from != null) { + this.stream.newline().append("FROM "); + this.printFrom(from); + } + + SqlNode where = select.getWhere(); + if (where != null) { + this.stream.newline().append("WHERE "); + this.printExpr(where); + } + + SqlNodeList groupBy = select.getGroup(); + if (groupBy != null && !groupBy.isEmpty()) { + this.stream.newline().append("GROUP BY "); + for (int i = 0; i < groupBy.size(); i++) { + if (i > 0) this.stream.append(", "); + this.printExpr(groupBy.get(i)); + } + } + + SqlNode having = select.getHaving(); + if (having != null) { + this.stream.newline().append("HAVING "); + this.printExpr(having); + } + + SqlNodeList windowDecls = select.getWindowList(); + if (windowDecls != null && !windowDecls.isEmpty()) { + this.stream.newline().append("WINDOW "); + for (int i = 0; i < windowDecls.size(); i++) { + if (i > 0) this.stream.append(", "); + this.print(windowDecls.get(i)); + } + } + + SqlNode qualify = select.getQualify(); + if (qualify != null) { + this.stream.newline().append("QUALIFY "); + this.printExpr(qualify); + } + + SqlNodeList orderBy = select.getOrderList(); + if (orderBy != null && !orderBy.isEmpty()) { + this.stream.newline().append("ORDER BY "); + for (int i = 0; i < orderBy.size(); i++) { + if (i > 0) this.stream.append(", "); + this.printExpr(orderBy.get(i)); + } + } + + SqlNode offset = select.getOffset(); + if (offset != null) { + this.stream.newline().append("OFFSET "); + this.printExpr(offset); + } + + SqlNode fetch = select.getFetch(); + if (fetch != null) { + this.stream.newline().append("LIMIT "); + this.printExpr(fetch); + } + } + + private void printFrom(SqlNode from) { + if (from instanceof SqlJoin) { + this.printJoin((SqlJoin) from); + } else { + this.printExpr(from); + } + } + + private void printJoin(SqlJoin join) { + this.printFrom(join.getLeft()); + this.stream.newline(); + boolean natural = join.isNatural(); + switch (join.getJoinType()) { + case INNER: this.stream.append(natural ? "NATURAL INNER JOIN " : "INNER JOIN "); break; + case LEFT: this.stream.append(natural ? "NATURAL LEFT JOIN " : "LEFT JOIN "); break; + case RIGHT: this.stream.append(natural ? "NATURAL RIGHT JOIN " : "RIGHT JOIN "); break; + case FULL: this.stream.append(natural ? "NATURAL FULL JOIN " : "FULL JOIN "); break; + case CROSS: this.stream.append(natural ? "NATURAL CROSS JOIN " : "CROSS JOIN "); break; + case COMMA: this.stream.append(", "); break; + default: this.stream.append("JOIN "); break; + } + SqlNode rightSide = join.getRight(); + if (rightSide instanceof SqlJoin) { + // A nested join on the right must be parenthesized; otherwise + // "JOIN (B JOIN C ON c1) ON c2" would print as "JOIN B JOIN C ON c1 ON c2" + // which the parser rejects (second ON appears where a join type is expected). + this.stream.append("(").increase(); + this.printJoin((SqlJoin) rightSide); + this.stream.decrease().newline().append(")"); + } else { + this.printExpr(rightSide); + } + JoinConditionType condType = join.getConditionType(); + SqlNode cond = join.getCondition(); + if (condType == JoinConditionType.ON && cond != null) { + this.stream.append(" ON "); + this.printExpr(cond); + } else if (condType == JoinConditionType.USING && cond != null) { + this.stream.append(" USING ("); + this.printExpr(cond); + this.stream.append(")"); + } + } + + private void printOrderBy(SqlOrderBy orderBy) { + this.print(requireNonNull(orderBy.query)); + if (!orderBy.orderList.isEmpty()) { + this.stream.newline().append("ORDER BY "); + for (int i = 0; i < orderBy.orderList.size(); i++) { + if (i > 0) this.stream.append(", "); + this.printExpr(orderBy.orderList.get(i)); + } + } + if (orderBy.offset != null) { + this.stream.newline().append("OFFSET "); + this.printExpr(orderBy.offset); + } + if (orderBy.fetch != null) { + this.stream.newline().append("LIMIT "); + this.printExpr(orderBy.fetch); + } + } + + private void printWith(SqlWith with) { + this.stream.append("WITH").increase(); + for (int i = 0; i < with.withList.size(); i++) { + if (i > 0) this.stream.append(",").newline(); + SqlWithItem item = (SqlWithItem) with.withList.get(i); + this.stream.append(this.fallbackSql(requireNonNull(item.name))); + this.stream.append(" AS "); + this.printQueryParens(item.query); + } + this.stream.decrease().newline(); + this.print(with.body); + } + + private void printCase(SqlCase caseExpr) { + SqlNode value = caseExpr.getValueOperand(); + if (value != null) { + this.stream.append("CASE "); + this.printExpr(value); + } else { + this.stream.append("CASE"); + } + SqlNodeList whenList = caseExpr.getWhenOperands(); + SqlNodeList thenList = caseExpr.getThenOperands(); + for (int i = 0; i < whenList.size(); i++) { + this.stream.append(" WHEN "); + this.printExpr(requireNonNull(whenList.get(i))); + this.stream.append(" THEN "); + this.printExpr(requireNonNull(thenList.get(i))); + } + SqlNode elseExpr = caseExpr.getElseOperand(); + if (elseExpr != null) { + this.stream.append(" ELSE "); + this.printExpr(elseExpr); + } + this.stream.append(" END"); + } + + private void printBasicCall(SqlBasicCall call) { + SqlOperator op = call.getOperator(); + List<@Nullable SqlNode> operands = call.getOperandList(); + SqlKind kind = op.getKind(); + + switch (kind) { + case AS: + this.printExpr(requireNonNull(operands.get(0))); + this.stream.append(" AS "); + // Alias is always a simple identifier — no parens needed + this.print(operands.get(1)); + return; + case CAST: { + this.stream.append("CAST("); + this.printExpr(requireNonNull(operands.get(0))); + this.stream.append(" AS "); + SqlNode castType = requireNonNull(operands.get(1)); + if (castType instanceof org.apache.calcite.sql.SqlIntervalQualifier) { + this.stream.append("INTERVAL "); + } + this.print(castType); + this.stream.append(")"); + return; + } + case DESCENDING: + this.printExpr(requireNonNull(operands.get(0))); + this.stream.append(" DESC"); + return; + case NULLS_FIRST: + this.printExpr(requireNonNull(operands.get(0))); + this.stream.append(" NULLS FIRST"); + return; + case NULLS_LAST: + this.printExpr(requireNonNull(operands.get(0))); + this.stream.append(" NULLS LAST"); + return; + case IN: + this.printExpr(requireNonNull(operands.get(0))); + this.stream.append(" IN "); + this.printInList(requireNonNull(operands.get(1))); + return; + case NOT_IN: + this.printExpr(requireNonNull(operands.get(0))); + this.stream.append(" NOT IN "); + this.printInList(requireNonNull(operands.get(1))); + return; + case LATERAL: + this.stream.append("LATERAL "); + this.printExpr(requireNonNull(operands.get(0))); + return; + case FILTER: + this.printExpr(requireNonNull(operands.get(0))); + this.stream.append(" FILTER (WHERE "); + this.printExpr(requireNonNull(operands.get(1))); + this.stream.append(")"); + return; + case WITHIN_GROUP: { + this.printExpr(requireNonNull(operands.get(0))); + this.stream.append(" WITHIN GROUP (ORDER BY "); + SqlNodeList orderList = (SqlNodeList) requireNonNull(operands.get(1)); + for (int i = 0; i < orderList.size(); i++) { + if (i > 0) this.stream.append(", "); + this.printExpr(requireNonNull(orderList.get(i))); + } + this.stream.append(")"); + return; + } + default: + break; + } + + if (operands.size() == 2 && op.getSyntax() == org.apache.calcite.sql.SqlSyntax.BINARY) { + this.printExpr(operands.get(0)); + this.stream.append(" ").append(op.getName()).append(" "); + this.printExpr(operands.get(1)); + return; + } else if (operands.size() == 1 && op.getSyntax() == org.apache.calcite.sql.SqlSyntax.POSTFIX) { + this.printExpr(operands.get(0)); + this.stream.append(" ").append(op.getName()); + return; + } else if (operands.size() == 1 && op.getSyntax() == org.apache.calcite.sql.SqlSyntax.PREFIX) { + this.stream.append(op.getName()).append(" "); + this.printExpr(operands.get(0)); + return; + } + + // Fallback: let Calcite's own unparse() handle everything else + this.stream.append(this.fallbackSql(call)); + } + + private void printCreateView(SqlCreateView view) { + if (view.getReplace()) { + this.stream.append("CREATE OR REPLACE"); + } else { + this.stream.append("CREATE"); + } + switch (view.viewKind) { + case LOCAL: this.stream.append(" LOCAL"); break; + case MATERIALIZED: this.stream.append(" MATERIALIZED"); break; + default: break; + } + this.stream.append(" VIEW "); + this.stream.append(this.fallbackSql(requireNonNull(view.name))); + + SqlNodeList cols = view.columnList; + if (cols != null && !cols.isEmpty()) { + this.stream.append("("); + for (int i = 0; i < cols.size(); i++) { + if (i > 0) this.stream.append(", "); + this.print(cols.get(i)); + } + this.stream.append(")"); + } + + this.printProperties(view.viewProperties); + this.stream.append(" AS").newline(); + this.print(view.query); + } + + private void printCreateTable(SqlCreateTable table) { + this.stream.append("CREATE TABLE "); + if (table.ifNotExists) this.stream.append("IF NOT EXISTS "); + this.stream.append(this.fallbackSql(table.name)); + this.stream.append(" (").increase(); + for (int i = 0; i < table.columnsOrForeignKeys.size(); i++) { + if (i > 0) this.stream.append(",").newline(); + this.print(requireNonNull(table.columnsOrForeignKeys.get(i))); + } + this.stream.decrease().newline().append(")"); + this.printProperties(table.tableProperties); + } + + /** Format a WITH (key = value, ...) property list. */ + private void printProperties(@Nullable SqlNodeList properties) { + if (properties == null || properties.isEmpty()) return; + this.stream.append(" WITH (").increase(); + for (int i = 0; i < properties.size(); i += 2) { + if (i > 0) this.stream.append(",").newline(); + SqlNode keyNode = requireNonNull(properties.get(i)); + SqlNode valueNode = requireNonNull(properties.get(i + 1)); + this.print(keyNode); + this.stream.append(" = "); + + // Pretty-print the 'connectors' value as formatted JSON + boolean emitted = false; + if (keyNode instanceof SqlLiteral key + && valueNode instanceof SqlLiteral value + && Objects.equals(key.toValue(), "connectors")) { + String raw = value.toValue(); + if (raw == null) { + this.stream.append("null"); + } else { + String formatted = Utilities.tryFormatAsJson(raw, this.stream.getIndentAmount()); + if (!formatted.equals(raw)) { + this.stream.append("'" + formatted.replace("'", "''") + "'"); + emitted = true; + } + } + } + if (!emitted) { + this.print(valueNode); + } + } + this.stream.decrease().newline().append(")"); + } +} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlToRelCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlToRelCompiler.java index 77cbf11c38a..57782723dfb 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlToRelCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlToRelCompiler.java @@ -141,6 +141,9 @@ import org.dbsp.sqlCompiler.compiler.errors.UnimplementedException; import org.dbsp.sqlCompiler.compiler.errors.UnsupportedException; import org.dbsp.sqlCompiler.compiler.frontend.ExtendedSqlParserPos; +import org.dbsp.sqlCompiler.compiler.frontend.SqlComment; +import org.dbsp.sqlCompiler.compiler.frontend.SqlCommentParser; +import org.dbsp.sqlCompiler.compiler.frontend.SqlPrettyPrinter; import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.optimizer.CalciteOptimizer; import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteObject; import org.dbsp.sqlCompiler.compiler.frontend.parser.PropertyList; @@ -847,9 +850,18 @@ public SqlNode parse(String sql) throws SqlParseException { /** Given a list of statements separated by semicolons, parse all of them. * @param saveLines True if the lines are from the user program; false if they are internally generated. */ public List parseStatements(String statements, boolean saveLines) throws SqlParseException { + // Debugging flag for (partially) testing SqlPrettyPrinter + final boolean ROUND_TRIP_CHECK = false; + SqlParser sqlParser = this.createSqlParser(statements, saveLines); List result = new ArrayList<>(); SqlNodeList sqlNodes = sqlParser.parseStmtList(); + if (saveLines && ROUND_TRIP_CHECK) { + List comments = SqlCommentParser.parse(statements); + String sql = SqlPrettyPrinter.toString(sqlNodes, comments); + // System.out.println(sql); + this.createSqlParser(sql, false).parseStmtList(); + } for (SqlNode node: sqlNodes) { SqlNode sqlNode = this.postParsingProcess(node, saveLines); ParsedStatement stat = new ParsedStatement(sqlNode, saveLines); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/parser/SqlCreateAggregate.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/parser/SqlCreateAggregate.java index 6fd559e5f99..d478a0e9fef 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/parser/SqlCreateAggregate.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/parser/SqlCreateAggregate.java @@ -76,6 +76,9 @@ public boolean isLinear() { writer.endList(frame); writer.keyword("RETURNS"); this.returnType.unparse(writer, 0, 0); + if (Boolean.FALSE.equals(this.returnType.getNullable())) { + writer.keyword("NOT NULL"); + } } @Override public SqlOperator getOperator() { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/parser/SqlCreateFunctionDeclaration.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/parser/SqlCreateFunctionDeclaration.java index df28c40dc07..82f4f480ded 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/parser/SqlCreateFunctionDeclaration.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/parser/SqlCreateFunctionDeclaration.java @@ -71,6 +71,9 @@ public SqlCreateFunctionDeclaration(SqlParserPos pos, boolean replace, writer.endList(frame); writer.keyword("RETURNS"); this.returnType.unparse(writer, 0, 0); + if (Boolean.FALSE.equals(this.returnType.getNullable())) { + writer.keyword("NOT NULL"); + } if (this.body != null) { writer.keyword("AS"); this.body.unparse(writer, 0, 0); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/parser/SqlLateness.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/parser/SqlLateness.java index 8dfd137bb98..d269b2f9ff0 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/parser/SqlLateness.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/parser/SqlLateness.java @@ -101,6 +101,7 @@ public SqlNode getLateness() { final int opLeft = getOperator().getLeftPrec(); final int opRight = getOperator().getRightPrec(); this.view.unparse(writer, opLeft, opRight); + writer.print("."); this.column.unparse(writer, opLeft, opRight); this.expression.unparse(writer, 0, 0); writer.newlineAndIndent(); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/literal/DBSPKeywordLiteral.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/literal/DBSPKeywordLiteral.java index 9dd8fea9a78..90dc7152eb3 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/literal/DBSPKeywordLiteral.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/literal/DBSPKeywordLiteral.java @@ -48,8 +48,31 @@ public boolean sameValue(@Nullable ISameValue o) { public DBSPKeywordLiteral(CalciteObject node, String keyword) { super(node, DBSPTypeKeyword.INSTANCE, false); - this.keyword = keyword.toLowerCase(); switch (keyword.toLowerCase()) { + case "sql_tsi_year": + keyword = "year"; + break; + case "sql_tsi_quarter": + keyword = "quarter"; + break; + case "sql_tsi_month": + keyword = "month"; + break; + case "sql_tsi_week": + keyword = "week"; + break; + case "sql_tsi_day": + keyword = "day"; + break; + case "sql_tsi_hour": + keyword = "hour"; + break; + case "sql_tsi_second": + keyword = "second"; + break; + case "sql_tsi_minute": + keyword = "minute"; + break; case "dow": case "epoch": case "isodow": @@ -77,6 +100,7 @@ public DBSPKeywordLiteral(CalciteObject node, String keyword) { default: throw new UnimplementedException("Not yet implemented support for Calcite construct " + keyword, node); } + this.keyword = keyword.toLowerCase(); } @Override diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/IIndentStream.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/IIndentStream.java index 23c1402fa5a..11bedfa460c 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/IIndentStream.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/IIndentStream.java @@ -224,6 +224,9 @@ default IIndentStream newline() { IIndentStream increase(); IIndentStream decrease(); + /** Return the number of spaces added per indentation level. */ + int getIndentAmount(); + default IIndentStream appendJsonLabelAndColon(String label) { return this.append("\"").append(Utilities.escapeDoubleQuotes(label)).append("\": "); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/IndentStream.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/IndentStream.java index 486af0b037b..339d12fc124 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/IndentStream.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/IndentStream.java @@ -51,6 +51,9 @@ public Appendable setOutputStream(Appendable appendable) { return result; } + @Override + public int getIndentAmount() { return this.amount; } + /** Set the indent amount. If less or equal to 0, newline will have no effect. */ public IIndentStream setIndentAmount(int amount) { Utilities.enforce(amount >= 0); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/NullIndentStream.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/NullIndentStream.java index a2e461fd8b1..2088215e5b8 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/NullIndentStream.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/NullIndentStream.java @@ -151,6 +151,9 @@ public IIndentStream newline() { return this; } + @Override + public int getIndentAmount() { return 0; } + @Override public IIndentStream increase() { return this; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/Utilities.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/Utilities.java index 57f9119cb88..fbf2c209021 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/Utilities.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/Utilities.java @@ -29,6 +29,8 @@ import com.fasterxml.jackson.core.JsonLocation; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.util.DefaultIndenter; +import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; @@ -226,6 +228,24 @@ public static ObjectMapper deterministicObjectMapper() { .build(); } + /** + * If {@code value} is valid JSON, returns it pretty-printed using {@code indent} + * spaces per indentation level. + * Returns the original string unchanged when the value is not valid JSON. + */ + public static String tryFormatAsJson(String value, int indent) { + try { + Object parsed = deterministicObjectMapper().readValue(value, Object.class); + DefaultPrettyPrinter printer = new DefaultPrettyPrinter(); + // Arrays use NopIndenter so "[" and "{" appear on the same line as the key. + printer.indentArraysWith(DefaultPrettyPrinter.NopIndenter.instance); + printer.indentObjectsWith(new DefaultIndenter(" ".repeat(indent), "\n")); + return deterministicObjectMapper().writer(printer).writeValueAsString(parsed); + } catch (Exception e) { + return value; + } + } + /** Add double quotes around string and escape symbols that need it. * @param unicodeBraces If true use \\u{} for Unicode, otherwise use \\u. */ public static String doubleQuote(String value, boolean unicodeBraces) { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MetadataTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MetadataTests.java index 81ce87bbc41..bc1434c6e23 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MetadataTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MetadataTests.java @@ -12,6 +12,8 @@ import org.dbsp.sqlCompiler.compiler.TestUtil; import org.dbsp.sqlCompiler.compiler.backend.JsonDecoder; import org.dbsp.sqlCompiler.compiler.errors.CompilerMessages; +import org.dbsp.sqlCompiler.compiler.frontend.SqlComment; +import org.dbsp.sqlCompiler.compiler.frontend.SqlCommentParser; import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.ProgramIdentifier; import org.dbsp.sqlCompiler.compiler.frontend.statements.CreateTableStatement; import org.dbsp.sqlCompiler.compiler.frontend.statements.DeclareViewStatement; @@ -1201,6 +1203,9 @@ public void testHelpMessage() throws SQLException { --errors Error output file; stderr if not specified Default: + --format + Output the SQL program reformatted + Default: false --handles Use handles (true) or Catalog (false) in the emitted Rust code Default: false @@ -2064,6 +2069,270 @@ SELECT jsonstring_as_ID_0(ID_7), ID_6, SUM(ID_4.ID_5) AS ID_9, COUNT(ID_4.ID_1[1 Utilities.deleteFile(input, false); } + // Helpers for SqlCommentParser unit tests + static void assertComment(SqlComment c, SqlComment.Kind kind, String text, int line, int col) { + Assert.assertEquals(kind, c.kind); + Assert.assertEquals(text, c.text); + Assert.assertEquals(line, c.pos.getLineNum()); + Assert.assertEquals(col, c.pos.getColumnNum()); + } + + /** Verify that {@code sql} is accepted by the SQL compiler without errors. */ + private void assertCompilesWithoutError(String sql) { + DBSPCompiler compiler = this.testCompiler(); + compiler.submitStatementsForCompilation(sql); + getCircuit(compiler); + Assert.assertEquals(0, compiler.messages.exitCode); + } + + @Test + public void testCommentParserLineDash() { + List cs = SqlCommentParser.parse("-- hello\n"); + Assert.assertEquals(1, cs.size()); + assertComment(cs.get(0), SqlComment.Kind.LINE_DASH, "-- hello\n", 1, 1); + } + + @Test + public void testCommentParserLineSlash() { + // "CREATE VIEW V AS SELECT 1; " is 27 chars, so "//" is at col 28. + String sql = "CREATE VIEW V AS SELECT 1; // end\n"; + List cs = SqlCommentParser.parse(sql); + Assert.assertEquals(1, cs.size()); + assertComment(cs.get(0), SqlComment.Kind.LINE_SLASH, "// end\n", 1, 28); + assertCompilesWithoutError(sql); + } + + @Test + public void testCommentParserBlock() { + List cs = SqlCommentParser.parse("/* hello */"); + Assert.assertEquals(1, cs.size()); + assertComment(cs.get(0), SqlComment.Kind.BLOCK, "/* hello */", 1, 1); + } + + @Test + public void testCommentParserNoTrailingNewline() { + // Single-line comment at EOF with no newline: text has no trailing newline. + String sql = "CREATE VIEW V AS SELECT 1; -- end"; + List cs = SqlCommentParser.parse(sql); + Assert.assertEquals(1, cs.size()); + Assert.assertEquals("-- end", cs.get(0).text); + assertCompilesWithoutError(sql); + } + + @Test + public void testCommentParserMultipleComments() { + // Line 2: "CREATE VIEW V AS SELECT 1; " is 27 chars, so "--" is at col 28. + String sql = "-- first\nCREATE VIEW V AS SELECT 1; -- second\n/* third */\n"; + List cs = SqlCommentParser.parse(sql); + Assert.assertEquals(3, cs.size()); + assertComment(cs.get(0), SqlComment.Kind.LINE_DASH, "-- first\n", 1, 1); + assertComment(cs.get(1), SqlComment.Kind.LINE_DASH, "-- second\n", 2, 28); + assertComment(cs.get(2), SqlComment.Kind.BLOCK, "/* third */", 3, 1); + assertCompilesWithoutError(sql); + } + + @Test + public void testCommentParserBlockMultiline() { + // Block comment spanning two lines; position tracks correctly. + String sql = "/* line 1\n line 2 */"; + List cs = SqlCommentParser.parse(sql); + Assert.assertEquals(1, cs.size()); + assertComment(cs.get(0), SqlComment.Kind.BLOCK, sql, 1, 1); + } + + @Test + public void testCommentParserInsideSingleQuote() { + // Comment markers inside a single-quoted string are not comments. + String sql = "CREATE VIEW V AS SELECT '-- not a comment /* also not */'"; + List cs = SqlCommentParser.parse(sql); + Assert.assertEquals(0, cs.size()); + assertCompilesWithoutError(sql); + } + + @Test + public void testCommentParserInsideDoubleQuote() { + // Comment markers inside a double-quoted identifier are not comments. + String sql = + "CREATE TABLE t (\"-- not a comment\" INT);\n" + + "CREATE VIEW V AS SELECT \"-- not a comment\" FROM t;"; + List cs = SqlCommentParser.parse(sql); + Assert.assertEquals(0, cs.size()); + assertCompilesWithoutError(sql); + } + + @Test + public void testCommentParserInsideBacktick() { + // Comment markers inside a backtick-quoted identifier are not comments. + List cs = SqlCommentParser.parse("SELECT `-- not a comment` FROM t"); + Assert.assertEquals(0, cs.size()); + } + + @Test + public void testCommentParserHintNotComment() { + // Calcite query hints /*+ ... */ are not collected as comments: + // they are already in the SqlNode tree and emitted by SqlPrettyPrinter. + String sql = + "CREATE TABLE t (x INT);\n" + + "CREATE TABLE s (x INT);\n" + + "CREATE VIEW V AS SELECT /*+ broadcast(t), shard(s) */ * FROM t JOIN s USING (x);"; + List cs = SqlCommentParser.parse(sql); + Assert.assertEquals(0, cs.size()); + assertCompilesWithoutError(sql); + } + + @Test + public void testCommentParserEscapedSingleQuote() { + // '' is an escaped quote; the string continues past it, so -- is not a comment. + String sql = "CREATE VIEW V AS SELECT 'it''s -- not a comment'"; + List cs = SqlCommentParser.parse(sql); + Assert.assertEquals(0, cs.size()); + assertCompilesWithoutError(sql); + } + + @Test + public void testCommentParserEscapedDoubleQuote() { + // "" is an escaped double-quote inside a double-quoted identifier. + String sql = + "CREATE TABLE t (\"my\"\"-- not a comment\" INT);\n" + + "CREATE VIEW V AS SELECT \"my\"\"-- not a comment\" FROM t;"; + List cs = SqlCommentParser.parse(sql); + Assert.assertEquals(0, cs.size()); + assertCompilesWithoutError(sql); + } + + @Test + public void testCommentParserEscapedBacktick() { + // `` is an escaped backtick inside a backtick-quoted identifier. + List cs = SqlCommentParser.parse("SELECT `my``-- not a comment`"); + Assert.assertEquals(0, cs.size()); + } + + @Test + public void testCommentParserStringWithNewline() { + // A string literal containing a newline: the -- on line 2 is still inside the string. + List cs = SqlCommentParser.parse("SELECT 'line1\n-- not a comment\n'"); + Assert.assertEquals(0, cs.size()); + } + + @Test + public void testCommentParserCommentAfterString() { + // Real comment immediately following a closed string. + // "CREATE VIEW V AS SELECT 'value' " is 32 chars, so "--" is at col 33. + String sql = "CREATE VIEW V AS SELECT 'value' -- real comment\n"; + List cs = SqlCommentParser.parse(sql); + Assert.assertEquals(1, cs.size()); + assertComment(cs.get(0), SqlComment.Kind.LINE_DASH, "-- real comment\n", 1, 33); + assertCompilesWithoutError(sql); + } + + @Test + public void testCommentParserInsideTableDef() { + // Comments between columns of a CREATE TABLE, verifying line/col positions. + String sql = + "CREATE TABLE t (\n" + + " -- col a\n" + + " a INT,\n" + + " -- col b\n" + + " b VARCHAR\n" + + ");\n"; + List cs = SqlCommentParser.parse(sql); + Assert.assertEquals(2, cs.size()); + assertComment(cs.get(0), SqlComment.Kind.LINE_DASH, "-- col a\n", 2, 5); + assertComment(cs.get(1), SqlComment.Kind.LINE_DASH, "-- col b\n", 4, 5); + assertCompilesWithoutError(sql); + } + + @Test + public void testFormat() throws IOException, SQLException { + File input = createInputScript(""" + -- Custom type for JSON payloads + CREATE TYPE TYP AS ("Z" INT); + CREATE FUNCTION jsonstring_as_typ(l VARCHAR) RETURNS TYP; + -- Main fact table + CREATE TABLE T( + -- primary key + x INT, + y INT LATENESS 0, + "Z" TYP ARRAY, + W STRING + ) WITH ('connectors' = '[{"name": "kafka", "url": "localhost"}]'); + CREATE TABLE orders(id INT NOT NULL, product VARCHAR, amount BIGINT, customer_x INT); + /* Aggregation view */ + CREATE VIEW V WITH ('emit_final' = 'y', 'connectors' = '[]') AS + SELECT jsonstring_as_typ(W), y, SUM(T.x) as sum, COUNT(T."Z"[1]) FROM T GROUP BY T.y, W; + CREATE VIEW v2 WITH ('connectors' = '[{"name": "http", "url": "localhost:8080"}]') AS + SELECT t.y, o.product, SUM(o.amount) AS total, + CASE WHEN SUM(o.amount) > 100 THEN 'high' ELSE 'low' END AS tier + FROM t JOIN orders AS o ON t.x = o.customer_x + WHERE o.amount > (SELECT AVG(amount) FROM orders) + GROUP BY t.y, o.product + HAVING COUNT(*) > 2;"""); + File out = File.createTempFile("out", ".sql", new File(".")); + out.deleteOnExit(); + CompilerMessages execute = CompilerMain.execute("--format", "-o", out.getPath(), input.getPath()); + Assert.assertEquals(0, execute.exitCode); + String result = Utilities.readFile(out.getAbsolutePath()); + Assert.assertEquals(""" + -- Custom type for JSON payloads + CREATE TYPE typ AS ("Z" INTEGER); + CREATE FUNCTION jsonstring_as_typ (l VARCHAR) RETURNS typ; + -- Main fact table + CREATE TABLE t ( + -- primary key + x INTEGER, + y INTEGER LATENESS 0, + "Z" typ ARRAY, + w string + ) WITH ( + 'connectors' = '[{ + "name" : "kafka", + "url" : "localhost" + }]' + ); + CREATE TABLE orders ( + id INTEGER NOT NULL, + product VARCHAR, + amount BIGINT, + customer_x INTEGER + ); + /* Aggregation view */ + CREATE VIEW v WITH ( + 'emit_final' = 'y', + 'connectors' = '[ ]' + ) AS + SELECT + jsonstring_as_typ(w), + y, + SUM(t.x) AS sum, + COUNT(t."Z"[1]) + FROM t + GROUP BY t.y, w; + CREATE VIEW v2 WITH ( + 'connectors' = '[{ + "name" : "http", + "url" : "localhost:8080" + }]' + ) AS + SELECT + t.y, + o.product, + SUM(o.amount) AS total, + CASE WHEN SUM(o.amount) > 100 THEN 'high' ELSE 'low' END AS tier + FROM t + INNER JOIN orders AS o ON t.x = o.customer_x + WHERE o.amount > ( + SELECT + AVG(amount) + FROM orders + ) + GROUP BY t.y, o.product + HAVING COUNT(*) > 2;""", result); + // Reformatted program is valid + execute = CompilerMain.execute("--noRust", "-i", out.getPath()); + Assert.assertEquals(0, execute.exitCode); + Utilities.deleteFile(input, false); + } + @Test public void testWarningsAreErrors() { this.statementsFailingInCompilation(""" diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/QATests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/QATests.java index 4a2a95ddd64..9ba3adc942f 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/QATests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/QATests.java @@ -91,7 +91,7 @@ static List getQATests() { } @Test - public void qaTests() throws SQLException, IOException { + public void qaTests() throws SQLException { // BaseSQLTests.showPlan(); for (File c : getQATests()) { // This program cannot be compiled because it contains a udf diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/mysql/TimestampDiffTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/mysql/TimestampDiffTests.java index 6423ba0a4ef..6f95535d7d1 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/mysql/TimestampDiffTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/mysql/TimestampDiffTests.java @@ -150,6 +150,8 @@ public void testDateDiff() { ----- 89 (1 row)"""); + this.statementsFailingInCompilation("CREATE VIEW V AS SELECT DATEDIFF(MINUTE, '', '');", + "Use TIMESTAMPDIFF instead of DATEDIFF for MINUTE"); } @Test diff --git a/sql-to-dbsp-compiler/calcite_version.env b/sql-to-dbsp-compiler/calcite_version.env index cac5a3695d6..062f571c5a4 100644 --- a/sql-to-dbsp-compiler/calcite_version.env +++ b/sql-to-dbsp-compiler/calcite_version.env @@ -3,5 +3,5 @@ CALCITE_REPO="https://github.com/apache/calcite.git" #CALCITE_REPO="https://github.com/mihaibudiu/calcite" CALCITE_BRANCH="main" CALCITE_CURRENT="1.42.0" -CALCITE_NEXT_COMMIT="1317946ea1c318ca7cc895031ce8ca48777fcf2d" +CALCITE_NEXT_COMMIT="7311760481af747175d0119ebdb83d58c14fc684" CALCITE_NEXT="1.43.0" diff --git a/sql-to-dbsp-compiler/using.md b/sql-to-dbsp-compiler/using.md index 3f2b2ea4014..bca7b7ca8cb 100644 --- a/sql-to-dbsp-compiler/using.md +++ b/sql-to-dbsp-compiler/using.md @@ -64,6 +64,9 @@ Usage: sql-to-dbsp [options] Input file to compile --errors Error output file; stderr if not specified Default: + --format + Output the SQL program reformatted + Default: false --handles Use handles (true) or Catalog (false) in the emitted Rust code Default: false