From f537be9a487592f9097bd242b34a750a08f7012a Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Wed, 22 Oct 2025 14:45:29 -0700 Subject: [PATCH 1/2] [SQL] Advance Calcite version fixing two more issues Signed-off-by: Mihai Budiu --- .../compiler/sql/simple/Regression1Tests.java | 30 +++++++++++++++++++ sql-to-dbsp-compiler/calcite_version.env | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/Regression1Tests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/Regression1Tests.java index 0326a7a4840..e76f9338af8 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/Regression1Tests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/Regression1Tests.java @@ -1169,5 +1169,35 @@ SELECT cast(contacts as MAP) contacts d| null|1 e| {"f":1}|1"""); } + + @Test + public void issue4890() { + this.getCC(""" + CREATE TABLE tbl( + roww ROW(i1 INT, v1 VARCHAR NULL)); + + CREATE MATERIALIZED VIEW v1 AS SELECT + roww IN (ROW(4,'cat')) AS roww + FROM tbl; + + CREATE MATERIALIZED VIEW v2 AS SELECT + roww IN (ROW(4,'cat')) AS roww + FROM tbl;"""); + } + + @Test + public void issue4891() { + this.getCC(""" + CREATE TABLE tbl( + roww ROW(i1 INT, v1 VARCHAR NULL)); + + CREATE MATERIALIZED VIEW v1 AS SELECT + roww IN (roww) AS roww + FROM tbl; + + CREATE MATERIALIZED VIEW v2 AS SELECT + roww NOT IN (roww) AS roww + FROM tbl;"""); + } } \ No newline at end of file diff --git a/sql-to-dbsp-compiler/calcite_version.env b/sql-to-dbsp-compiler/calcite_version.env index ff0a2003d38..f26c103e7da 100644 --- a/sql-to-dbsp-compiler/calcite_version.env +++ b/sql-to-dbsp-compiler/calcite_version.env @@ -3,5 +3,5 @@ CALCITE_REPO="https://github.com/apache/calcite.git" #CALCITE_REPO="https://github.com/mihaibudiu/calcite" CALCITE_BRANCH="main" CALCITE_CURRENT="1.40.0" -CALCITE_NEXT_COMMIT="0af5284d9f004fd9e38e2d62abd8c04de004be5c" +CALCITE_NEXT_COMMIT="5b3cf922657dcdd72eed7d45093db82cc80d0efc" CALCITE_NEXT="1.41.0" From a0cda87d458a13dec1ebad03c7ef9304bb30beee Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Thu, 23 Oct 2025 18:01:41 -0700 Subject: [PATCH 2/2] [SQL] Implement MAP_KEYS function Signed-off-by: Mihai Budiu --- crates/sqllib/src/map.rs | 18 +++++++- docs.feldera.com/docs/sql/function-index.md | 1 + docs.feldera.com/docs/sql/map.md | 1 + .../sqlCompiler/compiler/DBSPCompiler.java | 4 +- .../backend/rust/ToRustInnerVisitor.java | 6 ++- .../compiler/frontend/ExpressionCompiler.java | 5 +++ .../compiler/frontend/TypeCompiler.java | 3 ++ .../calciteCompiler/CalciteFunctions.java | 5 ++- .../calciteCompiler/SqlToRelCompiler.java | 4 ++ .../compiler/sql/simple/MapTests.java | 44 +++++++++++++++++++ 10 files changed, 86 insertions(+), 5 deletions(-) diff --git a/crates/sqllib/src/map.rs b/crates/sqllib/src/map.rs index a97a0a856be..457ec217ac1 100644 --- a/crates/sqllib/src/map.rs +++ b/crates/sqllib/src/map.rs @@ -1,6 +1,6 @@ //! Functions for manipulating maps -use crate::{ConcatSemigroup, Semigroup, Weight}; +use crate::{Array, ConcatSemigroup, Semigroup, Weight}; use dbsp::utils::Tup2; use std::collections::BTreeMap; use std::sync::Arc; @@ -274,3 +274,19 @@ where let key = key?; Some(map_contains_key__(value, key)) } + +#[doc(hidden)] +pub fn map_keys_(value: Map) -> Array +where + I: Ord + Clone, +{ + Arc::new(value.keys().cloned().collect()) +} + +#[doc(hidden)] +pub fn map_keysN(value: Option>) -> Option> +where + I: Ord + Clone, +{ + value.map(|value| map_keys_(value)) +} diff --git a/docs.feldera.com/docs/sql/function-index.md b/docs.feldera.com/docs/sql/function-index.md index 131eed7eef8..0d02981b9b4 100644 --- a/docs.feldera.com/docs/sql/function-index.md +++ b/docs.feldera.com/docs/sql/function-index.md @@ -150,6 +150,7 @@ * `MAP` (aggregate): [aggregates](aggregates.md#map) * `MAP`: [map](map.md#map-literals) * `MAP_CONTAINS_KEY`: [map](map.md#map_contains_key) +* `MAP_KEYS`: [map](map.md#map_keys) * `MAX` (aggregate): [aggregates](aggregates.md#max), [aggregates](aggregates.md#window-max) * `MD5`: [string](string.md#md5), [binary](binary.md#md5) * `MIN` (aggregate): [aggregates](aggregates.md#min), [aggregates](aggregates.md#window-min) diff --git a/docs.feldera.com/docs/sql/map.md b/docs.feldera.com/docs/sql/map.md index d0f57d6056f..90d844d71b9 100644 --- a/docs.feldera.com/docs/sql/map.md +++ b/docs.feldera.com/docs/sql/map.md @@ -48,6 +48,7 @@ Comparison operations (`=`, `<>`, `!=`, `>`, `<`, `>=`, `<=`) can be applied to | _map_`[`_key_`]` | Returns the element in the map with the specified key. If there is no such key, the result is `NULL`. | `MAP['x', 4, 'y', 3]['x']` => 4 | | `CARDINALITY`(map) | Returns the number of key-value pairs in the map. | `CARDINALITY(MAP['x', 4])` => 1 | | `MAP_CONTAINS_KEY`(map, key) | Returns true when the map has an item with the specified key; `NULL` if any argument is `NULL`. | `MAP_CONTAINS_KEY(MAP['x', 4], 'x')` => `true` | +| `MAP_KEYS`(map) | Returns a sorted ARRAY of the appropriate type with all the keys of the MAP. | `MAP_KEYS(MAP['x', 4, 'y', 5])` => `['x', 'y']` | ## The `UNNEST` Operator diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java index 4f03b9ab2d6..60e67f59b0f 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java @@ -444,8 +444,10 @@ List runParser() { for (SqlStatements stat : this.toCompile) { try { if (stat.many) { - if (stat.statement.isEmpty()) + if (stat.statement.isEmpty()) { + this.sqlToRelCompiler.emptyStatement(); continue; + } parsed.addAll(this.sqlToRelCompiler.parseStatements(stat.statement, stat.visible)); } else { SqlNode node = this.sqlToRelCompiler.parse(stat.statement, stat.visible); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/ToRustInnerVisitor.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/ToRustInnerVisitor.java index 3b22736f13b..6293dafd9f0 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/ToRustInnerVisitor.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/backend/rust/ToRustInnerVisitor.java @@ -1295,7 +1295,11 @@ public VisitDecision preorder(DBSPCastExpression expression) { if (sourceType.is(DBSPTypeVariant.class)) { // Cast variant to map functionName = "cast_to_map" + destType.nullableSuffix() + "_" + sourceType.baseTypeWithSuffix(); - this.builder.append(functionName).append("(").increase(); + this.builder.append(functionName).append("::<"); + destMap.getKeyType().accept(this); + this.builder.append(", "); + destMap.getValueType().accept(this); + this.builder.append(">(").increase(); expression.source.accept(this); this.builder.decrease().append(")"); this.pop(expression); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java index 180b00f73fd..3ac0bf201b7 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java @@ -1887,6 +1887,11 @@ else if (arg0Type.is(DBSPTypeMap.class)) } return next; } + case MAP_KEYS: + validateArgCount(node, operationName, ops.size(), 1); + DBSPExpression arg0 = ops.get(0); + String method = getArrayOrMapCallName(call, arg0); + return new DBSPApplyExpression(node, method, type, arg0); case DOT: default: throw new UnimplementedException("Function " + Utilities.singleQuote(call.getOperator().toString()) diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/TypeCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/TypeCompiler.java index a6c4a33ec5e..41003fe4bea 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/TypeCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/TypeCompiler.java @@ -425,6 +425,9 @@ public DBSPType convertType(SourcePositionRange context, RelDataType dt, boolean case MAP: { RelDataType kt = Objects.requireNonNull(dt.getKeyType()); DBSPType keyType = this.convertType(context, kt, asStruct); + if (keyType.code == DBSPTypeCode.NULL) { + throw new CompilationError("MAP key type cannot be NULL"); + } RelDataType vt = Objects.requireNonNull(dt.getValueType()); DBSPType valueType = this.convertType(context, vt, asStruct); return new DBSPTypeMap(keyType, valueType, dt.isNullable()); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteFunctions.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteFunctions.java index 4ce559f8907..84d0bd4e9d8 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteFunctions.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteFunctions.java @@ -320,8 +320,9 @@ record Func(SqlOperator function, String functionName, SqlLibrary library, new Func(SqlLibraryOperators.MD5, "MD5", SqlLibrary.SPARK, "string#md5,binary#md5", false), new Func(SqlLibraryOperators.IFNULL, "IFNULL", SqlLibrary.BIG_QUERY, - "comparisons#ifnull", false) - + "comparisons#ifnull", false), + new Func(SqlLibraryOperators.MAP_KEYS, "MAP_KEYS", SqlLibrary.SPARK, + "map#map_keys", false) // new Func(SqlLibraryOperators.SAFE_ORDINAL, "SAFE_ORDINAL", SqlLibrary.BIG_QUERY, "array", false), }; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlToRelCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlToRelCompiler.java index 88bffe31969..a86ebe0cd15 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlToRelCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlToRelCompiler.java @@ -376,6 +376,10 @@ public int getDefaultPrecision(SqlTypeName typeName) { public boolean shouldConvertRaggedUnionTypesToVarying() { return true; } }; + public void emptyStatement() { + this.newlines.append("\n"); + } + /** A TypeFactory that knows about RelStruct, our representation of user-defined types */ public static class CustomTypeFactory extends SqlTypeFactoryImpl { static int currentId = 0; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/MapTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/MapTests.java index e97f511cf31..78dcd41c900 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/MapTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/MapTests.java @@ -6,6 +6,7 @@ import org.dbsp.sqlCompiler.compiler.sql.tools.Change; import org.dbsp.sqlCompiler.compiler.sql.tools.InputOutputChange; import org.dbsp.sqlCompiler.compiler.sql.tools.InputOutputChangeStream; +import org.dbsp.sqlCompiler.ir.expression.DBSPArrayExpression; import org.dbsp.sqlCompiler.ir.expression.DBSPTupleExpression; import org.dbsp.sqlCompiler.ir.expression.literal.DBSPI32Literal; import org.dbsp.sqlCompiler.ir.expression.DBSPMapExpression; @@ -163,4 +164,47 @@ public void testUnnestMapFields() { this.testQuery("", sql, new InputOutputChangeStream() .addPair(new Change(), new Change("V", result))); } + + @Test + public void nullMapKey() { + this.statementsFailingInCompilation("CREATE VIEW V AS SELECT MAP[NULL, NULL]", + "MAP key type cannot be NULL"); + } + + @Test + public void testMapKeys() { + String sql = "SELECT map_keys(map['foo', 1, 'bar', 2])"; + DBSPZSetExpression result = new DBSPZSetExpression( + new DBSPTupleExpression(new DBSPArrayExpression( + false, + new DBSPStringLiteral("bar"), + new DBSPStringLiteral("foo")))); + this.testQuery("", sql, new InputOutputChangeStream() + .addPair(new Change(), new Change("V", result))); + } + + @Test + public void mapVariant() { + var ccs = this.getCCS(""" + create table j(j VARCHAR); + + create LOCAL view user_props AS + SELECT PARSE_JSON(j) AS contacts FROM j; + + create view abc as + WITH ref_profile AS ( + SELECT cast(contacts as MAP) contacts + FROM user_props + ) SELECT key + FROM ref_profile profile_0, UNNEST(MAP_KEYS(profile_0.contacts)) AS t(key)"""); + ccs.step(""" + INSERT INTO j VALUES('{ "a": "1", "b": 2, "c": [1, 2, 3], "d": null, "e": { "f": 1 } }');""", """ + key | weight + ------------------------ + a| 1 + b| 1 + c| 1 + d| 1 + e| 1"""); + } }