diff --git a/docs.feldera.com/docs/sql/array.md b/docs.feldera.com/docs/sql/array.md index 4938db2dffa..e617a31e91c 100644 --- a/docs.feldera.com/docs/sql/array.md +++ b/docs.feldera.com/docs/sql/array.md @@ -49,7 +49,7 @@ CREATE VIEW V AS SELECT city, data.country FROM data CROSS JOIN UNNEST(data.cities) AS city; ``` -`UNNEST` applied to a `NULL` value returns an empty array. +`UNNEST` applied to a `NULL` value returns an empty table. Note that applying `UNNEST` to an `ARRAY` of structure-typed objects will produce a collection whose columns are the fields of the diff --git a/docs.feldera.com/docs/sql/function-index.md b/docs.feldera.com/docs/sql/function-index.md index c718a83572a..adcd0bdec54 100644 --- a/docs.feldera.com/docs/sql/function-index.md +++ b/docs.feldera.com/docs/sql/function-index.md @@ -222,7 +222,7 @@ * `UNION`: [grammar](grammar.md#setop) * `UNION ALL`: [grammar](grammar.md#setop) * `UNIQUE`: [comparisons](comparisons.md#unique) -* `UNNEST`: [array](array.md#the-unnest-sql-operator) +* `UNNEST`: [array](array.md#the-unnest-sql-operator), [map](map.md#the-unnest-operator) * `UPPER`: [string](string.md#upper) * `VALUES`: [grammar](grammar.md#values) * `WEEK`: [datetime](datetime.md#week) diff --git a/docs.feldera.com/docs/sql/map.md b/docs.feldera.com/docs/sql/map.md index 2d6b46ce603..d0f57d6056f 100644 --- a/docs.feldera.com/docs/sql/map.md +++ b/docs.feldera.com/docs/sql/map.md @@ -49,3 +49,21 @@ Comparison operations (`=`, `<>`, `!=`, `>`, `<`, `>=`, `<=`) can be applied to | `CARDINALITY`(map) | Returns the number of key-value pairs in the map. | `CARDINALITY(MAP['x', 4])` => 1 | | `MAP_CONTAINS_KEY`(map, key) | Returns true when the map has an item with the specified key; `NULL` if any argument is `NULL`. | `MAP_CONTAINS_KEY(MAP['x', 4], 'x')` => `true` | +## The `UNNEST` Operator + +The `UNNEST` operator takes a `MAP` and returns a table with a row for +each key-value pair in the `MAP`. If the input is a +map with 5 key-value pairs, the output is a table with 5 rows, each +row holding one key-value pair of the map. + +When `UNNEST` operator is used in self-joins as follows, an alias needs +to be used to name the key and value fields (`zips(city, zip)` in the example): + +```sql +CREATE TABLE data(zipcodes MAP, COUNTRY VARCHAR); + +CREATE VIEW V AS SELECT data.country, city, zip +FROM data CROSS JOIN UNNEST(data.zipcodes) AS zips(city, zip); +``` + +`UNNEST` applied to a `NULL` value returns an empty table. diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java index e4e6d48322e..c9d24082f8a 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java @@ -188,6 +188,7 @@ import org.dbsp.sqlCompiler.ir.statement.DBSPStructItem; import org.dbsp.sqlCompiler.ir.type.DBSPType; import org.dbsp.sqlCompiler.ir.type.DBSPTypeCode; +import org.dbsp.sqlCompiler.ir.type.ICollectionType; import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeFunction; import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeAny; import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeRawTuple; @@ -501,8 +502,8 @@ void visitCorrelate(LogicalCorrelate correlate) { ExpressionCompiler eComp = new ExpressionCompiler(correlate, dataVar, this.compiler); DBSPClosureExpression arrayExpression = eComp.compile(projection).closure(dataVar); DBSPTypeTuple uncollectElementType = this.convertType(uncollect.getRowType(), false).to(DBSPTypeTuple.class); - DBSPType arrayElementType = arrayExpression.getResultType().to(DBSPTypeArray.class).getElementType(); - if (arrayElementType.mayBeNull) + DBSPType collectionElementType = arrayExpression.getResultType().to(ICollectionType.class).getElementType(); + if (collectionElementType.mayBeNull) // This seems to be a bug in Calcite, we should not need to do this adjustment uncollectElementType = uncollectElementType.withMayBeNull(true).to(DBSPTypeTuple.class); @@ -529,8 +530,8 @@ void visitCorrelate(LogicalCorrelate correlate) { else shuffleSize += 1; } - if (arrayElementType.is(DBSPTypeTupleBase.class)) - shuffleSize += arrayElementType.to(DBSPTypeTupleBase.class).size(); + if (collectionElementType.is(DBSPTypeTupleBase.class)) + shuffleSize += collectionElementType.to(DBSPTypeTupleBase.class).size(); else shuffleSize += 1; DBSPTypeFunction functionType = new DBSPTypeFunction(type, leftElementType.ref()); @@ -706,7 +707,8 @@ void visitUncollect(Uncollect uncollect) { throw new UnimplementedException("UNNEST with multiple vectors", node); } DBSPVariablePath data = new DBSPVariablePath(inputRowType.ref()); - DBSPType arrayElementType = data.deref().field(0).getType(); + DBSPType arrayType = data.deref().field(0).getType(); + DBSPType collectionElementType = arrayType.to(ICollectionType.class).getElementType(); DBSPClosureExpression getField0 = data.deref().field(0).closure(data); DBSPTypeFunction functionType = new DBSPTypeFunction(type, inputRowType.ref()); @@ -717,10 +719,11 @@ void visitUncollect(Uncollect uncollect) { else shuffleSize += 1; } - if (arrayElementType.is(DBSPTypeTupleBase.class)) - shuffleSize += arrayElementType.to(DBSPTypeTupleBase.class).size(); - else + if (collectionElementType.is(DBSPTypeTupleBase.class)) { + shuffleSize += collectionElementType.to(DBSPTypeTupleBase.class).size(); + } else { shuffleSize += 1; + } DBSPFlatmap function = new DBSPFlatmap(node, functionType, inputRowType, getField0, Linq.list(), null, indexType, new IdShuffle(shuffleSize)); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteFunctions.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteFunctions.java index 2b98bae47e0..83a4f16179a 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteFunctions.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteFunctions.java @@ -129,7 +129,7 @@ record Func(SqlOperator function, String functionName, SqlLibrary library, new Func(SqlStdOperatorTable.IGNORE_NULLS, "IGNORE NULLS", SqlLibrary.STANDARD, "grammar#window-aggregates", false), new Func(SqlStdOperatorTable.RESPECT_NULLS, "RESPECT NULLS", SqlLibrary.STANDARD, "grammar#window-aggregates", false), new Func(SqlStdOperatorTable.MINUS_DATE, "-", SqlLibrary.STANDARD, "datetime", false), - new Func(SqlStdOperatorTable.UNNEST, "UNNEST", SqlLibrary.STANDARD, "array#the-unnest-sql-operator", false), + new Func(SqlStdOperatorTable.UNNEST, "UNNEST", SqlLibrary.STANDARD, "array#the-unnest-sql-operator,map#the-unnest-operator", false), new Func(SqlStdOperatorTable.UNNEST_WITH_ORDINALITY, "UNNEST WITH ORDINALITY", SqlLibrary.STANDARD, "", false), new Func(SqlStdOperatorTable.LATERAL, "LATERAL", SqlLibrary.STANDARD, "grammar#lateral", false), new Func(SqlStdOperatorTable.COLLECTION_TABLE, "TABLE", SqlLibrary.STANDARD, "grammar", false), diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/LowerCircuitVisitor.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/LowerCircuitVisitor.java index 7a8cdd5f6fa..85d47c345db 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/LowerCircuitVisitor.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/LowerCircuitVisitor.java @@ -16,6 +16,7 @@ import org.dbsp.sqlCompiler.circuit.operator.DBSPStreamAggregateOperator; import org.dbsp.sqlCompiler.circuit.OutputPort; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; +import org.dbsp.sqlCompiler.compiler.errors.InternalCompilerError; import org.dbsp.sqlCompiler.compiler.frontend.TypeCompiler; import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteRelNode; import org.dbsp.sqlCompiler.ir.aggregate.DBSPFold; @@ -42,12 +43,14 @@ import org.dbsp.sqlCompiler.ir.statement.DBSPLetStatement; import org.dbsp.sqlCompiler.ir.statement.DBSPStatement; import org.dbsp.sqlCompiler.ir.type.DBSPType; +import org.dbsp.sqlCompiler.ir.type.DBSPTypeCode; import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTuple; import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeAny; import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeRawTuple; import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTupleBase; import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeUSize; import org.dbsp.sqlCompiler.ir.type.user.DBSPComparatorType; +import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeMap; import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeVec; import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeWithCustomOrd; import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeZSet; @@ -72,8 +75,8 @@ public static DBSPExpression rewriteFlatmap( // move |x: &Tuple2, Option>, | -> _ { // let x0: Array = (*x.0).clone(); // let x1: x.1.clone(); - // let array_clone = (*x).0.clone(); - // array_clone.into_iter().map({ + // let collection_clone = (*x).0.clone(); + // collection_clone.into_iter().map({ // move |e: i32, | -> Tuple3, Option, i32> { // Tuple3::new(x0.clone(), x1.clone(), e) // } @@ -159,30 +162,40 @@ public static DBSPExpression rewriteFlatmap( } resultColumns = flatmap.shuffle.shuffle(resultColumns); - // let array = if (*x).0.is_none() { + // let collection = if (*x).0.is_none() { // vec!() // } else { // (*x).0.clone().unwrap() // }; // or - // let array = (*x).0.clone(); - DBSPExpression extractArray = flatmap.collectionExpression.call(rowVar).applyClone(); + // let collection = (*x).0.clone(); + DBSPExpression extractCollection = flatmap.collectionExpression.call(rowVar).applyClone(); if (compiler != null) - extractArray = extractArray.reduce(compiler); - DBSPLetStatement statement = new DBSPLetStatement("array", extractArray); + extractCollection = extractCollection.reduce(compiler); + DBSPLetStatement statement = new DBSPLetStatement("collection", extractCollection); statements.add(statement); - DBSPType arrayType = extractArray.getType(); + DBSPType collectionType = extractCollection.getType(); - DBSPExpression arrayExpression; - if (arrayType.mayBeNull) { + DBSPExpression collectionExpression; + if (collectionType.mayBeNull) { DBSPExpression condition = statement.getVarReference().is_null(); - DBSPExpression empty = new DBSPTypeVec(collectionElementType, false).emptyVector(); + DBSPExpression empty; + if (flatmap.collectionKind == DBSPTypeCode.ARRAY) + empty = new DBSPTypeVec(collectionElementType, false).emptyVector(); + else if (flatmap.collectionKind == DBSPTypeCode.MAP) { + DBSPTypeTuple tup = collectionElementType.to(DBSPTypeTuple.class); + Utilities.enforce(tup.size() == 2); + empty = new DBSPTypeMap(tup.tupFields[0], tup.tupFields[1], false).defaultValue(); + } else { + throw new InternalCompilerError("Unexpected collection type in flatmap " + flatmap.collectionKind); + } + DBSPExpression contents = statement.getVarReference().unwrap().deref().applyClone(); - arrayExpression = new DBSPIfExpression(flatmap.getNode(), condition, empty, contents); + collectionExpression = new DBSPIfExpression(flatmap.getNode(), condition, empty, contents); } else { - arrayExpression = statement.getVarReference().deref().applyClone(); + collectionExpression = statement.getVarReference().deref().applyClone(); } - statement = new DBSPLetStatement("array_clone", arrayExpression); + statement = new DBSPLetStatement("collection_clone", collectionExpression); statements.add(statement); // move |e: i32, | -> Tuple3, Option, i32> { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPFlatmap.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPFlatmap.java index 964094cb24a..92d4a514156 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPFlatmap.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPFlatmap.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.JsonNode; import org.dbsp.sqlCompiler.compiler.backend.JsonDecoder; +import org.dbsp.sqlCompiler.compiler.errors.UnimplementedException; import org.dbsp.sqlCompiler.compiler.visitors.outer.LowerCircuitVisitor; import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteObject; import org.dbsp.sqlCompiler.compiler.visitors.VisitDecision; @@ -15,6 +16,7 @@ import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTupleBase; import org.dbsp.sqlCompiler.ir.type.ICollectionType; import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeBaseType; +import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeArray; import org.dbsp.util.IIndentStream; import org.dbsp.util.Linq; import org.dbsp.util.Shuffle; @@ -55,6 +57,12 @@ * The result type is the output element type. */ public final class DBSPFlatmap extends DBSPExpression { + // The kind of collection that we are iterating on + enum CollectionKind { + Array, + Map, + } + /** Type of the input row. */ public final DBSPTypeTuple inputElementType; /** A closure which, applied to 'data', produces the @@ -77,6 +85,7 @@ public final class DBSPFlatmap extends DBSPExpression { public final DBSPType ordinalityIndexType; /** Shuffle to apply to elements in the produced tuple */ public final Shuffle shuffle; + public final DBSPTypeCode collectionKind; public DBSPFlatmap(CalciteObject node, DBSPTypeFunction resultElementType, @@ -91,7 +100,13 @@ public DBSPFlatmap(CalciteObject node, this.inputElementType = inputElementType; this.rightProjections = rightProjections; this.collectionExpression = collectionExpression; + this.collectionKind = collectionExpression.getResultType().code; this.leftInputIndexes = leftInputIndexes; + Utilities.enforce(this.collectionKind == DBSPTypeCode.ARRAY || + this.collectionKind == DBSPTypeCode.MAP); + if (ordinalityIndexType != null && this.collectionKind == DBSPTypeCode.MAP) { + throw new UnimplementedException("UNNEST with ORDINALITY not supported for MAP values", node); + } Utilities.enforce(collectionExpression.parameters.length == 1); Utilities.enforce(collectionExpression.parameters[0].type.sameType(this.inputElementType.ref()), "Collection expression expects " + collectionExpression.parameters[0].type diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/user/DBSPTypeMap.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/user/DBSPTypeMap.java index 34bfee362ba..51e9616d8f0 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/user/DBSPTypeMap.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/type/user/DBSPTypeMap.java @@ -7,6 +7,9 @@ import org.dbsp.sqlCompiler.ir.expression.DBSPExpression; import org.dbsp.sqlCompiler.ir.expression.DBSPMapExpression; import org.dbsp.sqlCompiler.ir.type.DBSPType; +import org.dbsp.sqlCompiler.ir.type.ICollectionType; +import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeRawTuple; +import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTuple; import org.dbsp.util.Linq; import org.dbsp.util.Utilities; @@ -14,10 +17,14 @@ import static org.dbsp.sqlCompiler.ir.type.DBSPTypeCode.MAP; -/** Represents the type of a Rust Map as a TypeUser. */ -public class DBSPTypeMap extends DBSPTypeUser { +/** Represents the type of a Rust Map as a TypeUser. + * As collection, it contains key-value pair tuples. */ +public class DBSPTypeMap extends DBSPTypeUser implements ICollectionType { + public final DBSPTypeRawTuple collectionElementType; + public DBSPTypeMap(DBSPType mapKeyType, DBSPType mapValueType, boolean mayBeNull) { super(mapKeyType.getNode(), MAP, "Map", mayBeNull, mapKeyType, mapValueType); + this.collectionElementType = new DBSPTypeRawTuple(this.getKeyType(), this.getValueType()); } public DBSPType getKeyType() { @@ -78,4 +85,9 @@ public static DBSPTypeMap fromJson(JsonNode node, JsonDecoder decoder) { boolean mayBeNull = fromJsonMayBeNull(node); return new DBSPTypeMap(typeArgs.get(0), typeArgs.get(1), mayBeNull); } + + @Override + public DBSPType getElementType() { + return this.collectionElementType; + } } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MultiCrateTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MultiCrateTests.java index b8a1c53eb85..af759574f3c 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MultiCrateTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/MultiCrateTests.java @@ -29,7 +29,7 @@ void compileToMultiCrate(String file, boolean check) throws SQLException, IOExce messages.print(); throw new RuntimeException("Error during compilation"); } - if (check) + if (check && !BaseSQLTests.skipRust) Utilities.compileAndCheckRust(BaseSQLTests.RUST_CRATES_DIRECTORY, true); } @@ -182,7 +182,8 @@ public void testPackagedDemos() throws SQLException, IOException, InterruptedExc this.appendCargoDependencies(cargoContents); } } - Utilities.compileAndCheckRust(BaseSQLTests.RUST_CRATES_DIRECTORY, true); + if (!BaseSQLTests.skipRust) + Utilities.compileAndCheckRust(BaseSQLTests.RUST_CRATES_DIRECTORY, true); if (udf != null) { //noinspection ResultOfMethodCallIgnored udf.delete(); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/OtherTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/OtherTests.java index cbab96ed67e..ba127712f39 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/OtherTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/OtherTests.java @@ -290,7 +290,8 @@ WITH LOW_PRICE_CTE AS (select part, MIN(price) as price from PRICE group by part CompilerMessages messages = CompilerMain.execute("-o", BaseSQLTests.TEST_FILE_PATH, file.getPath()); if (messages.errorCount() > 0) throw new RuntimeException(messages.toString()); - Utilities.compileAndCheckRust(BaseSQLTests.RUST_DIRECTORY, true); + if (!BaseSQLTests.skipRust) + Utilities.compileAndCheckRust(BaseSQLTests.RUST_DIRECTORY, true); } void compileFile(String file, boolean run) throws SQLException, IOException, InterruptedException { @@ -298,7 +299,7 @@ void compileFile(String file, boolean run) throws SQLException, IOException, Int "-i", "--alltables", "-q", "--ignoreOrder", "-o", BaseSQLTests.TEST_FILE_PATH, file); messages.print(); Assert.assertEquals(0, messages.errorCount()); - if (run) + if (run && !BaseSQLTests.skipRust) Utilities.compileAndCheckRust(BaseSQLTests.RUST_DIRECTORY, true); // cleanup after ourselves createEmptyStubs(); @@ -418,7 +419,8 @@ public void testRustCompiler() throws IOException, InterruptedException, SQLExce CompilerMessages messages = CompilerMain.execute("-q", "-o", BaseSQLTests.TEST_FILE_PATH, file.getPath()); messages.print(); Assert.assertEquals(0, messages.exitCode); - Utilities.compileAndCheckRust(BaseSQLTests.RUST_DIRECTORY, true); + if (!BaseSQLTests.skipRust) + Utilities.compileAndCheckRust(BaseSQLTests.RUST_DIRECTORY, true); } @Test @@ -501,7 +503,8 @@ pub fn test() { try (FileWriter fr = new FileWriter(rust, true)) { // append fr.write(rustHandlesTest); } - Utilities.compileAndCheckRust(BaseSQLTests.RUST_DIRECTORY, true); + if (!BaseSQLTests.skipRust) + Utilities.compileAndCheckRust(BaseSQLTests.RUST_DIRECTORY, true); // Second test message = CompilerMain.execute( @@ -513,7 +516,8 @@ pub fn test() { try (FileWriter fr = new FileWriter(rust, true)) { // append fr.write(rustCatalogTest); } - Utilities.compileAndCheckRust(BaseSQLTests.RUST_DIRECTORY, true); + if (!BaseSQLTests.skipRust) + Utilities.compileAndCheckRust(BaseSQLTests.RUST_DIRECTORY, true); } @Test @@ -635,7 +639,7 @@ public void serializationTest() throws JsonProcessingException { public void generateFunctionIndex() throws IOException { // When invoked it generates documentation for the supported functions and operators // in the specified file. - String file = "../../docs/sql/function-index.md"; + String file = "../../docs.feldera.com/docs/sql/function-index.md"; FunctionDocumentation.generateIndex(file); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/ArrayTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/ArrayTests.java index f338a94e0cf..87699809812 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/ArrayTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/ArrayTests.java @@ -115,6 +115,38 @@ public void testUnnest() { new Change(), new Change(result)).toStream()); } + @Test + public void testUnnestRow() { + String query = "select * from UNNEST(array[ROW(3, 5), ROW(4, 6)]) as T2(y, z)"; + DBSPTupleExpression tuple0 = new DBSPTupleExpression( + new DBSPI32Literal(3, false), + new DBSPI32Literal(5, false)); + DBSPTupleExpression tuple1 = new DBSPTupleExpression( + new DBSPI32Literal(4, false), + new DBSPI32Literal(6, false)); + DBSPZSetExpression result = new DBSPZSetExpression(tuple0, tuple1); + this.testQuery("", query, new InputOutputChange( + new Change(), new Change(result)).toStream()); + } + + @Test + public void testUnnestRowRow() { + String query = "select * from UNNEST(array[ROW(1, ROW(5, 10)), ROW(2, ROW(6, 12))]) as T2(y, z)"; + DBSPTupleExpression tuple0 = new DBSPTupleExpression( + new DBSPI32Literal(1, false), + new DBSPTupleExpression( + new DBSPI32Literal(5, false), + new DBSPI32Literal(10, false))); + DBSPTupleExpression tuple1 = new DBSPTupleExpression( + new DBSPI32Literal(2, false), + new DBSPTupleExpression( + new DBSPI32Literal(6, false), + new DBSPI32Literal(12, false))); + DBSPZSetExpression result = new DBSPZSetExpression(tuple0, tuple1); + this.testQuery("", query, new InputOutputChange( + new Change(), new Change(result)).toStream()); + } + @Test public void testUnnestDuplicate() { String query = "SELECT * FROM UNNEST(ARRAY [1, 1, 1])"; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/MapTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/MapTests.java index cbcbbc47b53..faa55c5549c 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/MapTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/MapTests.java @@ -113,4 +113,54 @@ public void testMapSubquery() { this.testQuery(ddl, query, new InputOutputChangeStream().addPair(new Change(input), new Change(result))); } + + @Test + public void testUnnestMap() { + String sql = "select * from UNNEST(map['a', 12])"; + DBSPZSetExpression result = new DBSPZSetExpression( + new DBSPTupleExpression( + new DBSPStringLiteral("a", false), + new DBSPI32Literal(12, false))); + this.testQuery("", sql, + new InputOutputChangeStream().addPair(new Change(), new Change(result))); + } + + @Test + public void testUnnestMap2() { + String sql = "select * from UNNEST(map['a', 12, 'b', 15, 'c', NULL])"; + DBSPZSetExpression result = new DBSPZSetExpression( + new DBSPTupleExpression( + new DBSPStringLiteral("a", false), + new DBSPI32Literal(12, true)), + new DBSPTupleExpression( + new DBSPStringLiteral("b", false), + new DBSPI32Literal(15, true)), + new DBSPTupleExpression( + new DBSPStringLiteral("c", false), + new DBSPI32Literal())); + this.testQuery("", sql, + new InputOutputChangeStream().addPair(new Change(), new Change(result))); + } + + @Test + public void testUnnestMapFields() { + String sql = + "WITH T(i, m) as (VALUES(1, MAP[1, 2, 3, 4]), (2, MAP[5, NULL])) " + + "SELECT T.i, k, v FROM T CROSS JOIN UNNEST(T.m) AS pair(k, v)"; + DBSPZSetExpression result = new DBSPZSetExpression( + new DBSPTupleExpression( + new DBSPI32Literal(1, false), + new DBSPI32Literal(1, false), + new DBSPI32Literal(2, true)), + new DBSPTupleExpression( + new DBSPI32Literal(1, false), + new DBSPI32Literal(3, false), + new DBSPI32Literal(4, true)), + new DBSPTupleExpression( + new DBSPI32Literal(2, false), + new DBSPI32Literal(5, false), + new DBSPI32Literal())); + this.testQuery("", sql, new InputOutputChangeStream() + .addPair(new Change(), new Change(result))); + } } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/suites/TpcDsTest.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/suites/TpcDsTest.java index 28ff10a0d3c..6d996ea513e 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/suites/TpcDsTest.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/suites/TpcDsTest.java @@ -29,7 +29,5 @@ public void compileTpcds() throws IOException { compiler.submitStatementsForCompilation(tpcds); CompilerCircuit ccs = new CompilerCircuit(compiler); ccs.showErrors(); - // This crashes the Rust compiler! - // this.addRustTestCase("tpcds", ccs); } } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/BaseSQLTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/BaseSQLTests.java index 9b8fad4eec0..b7f99c13956 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/BaseSQLTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/BaseSQLTests.java @@ -54,11 +54,17 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; /** Base class for SQL-based tests. */ public class BaseSQLTests { + // Debugging: set to true to only compile SQL + public static final boolean skipRust = false; + // Debugging: set to only accept some tests + public static Predicate acceptTest = x -> true; + /** Override this method to prepare the tables on * which the tests are built. */ public void prepareInputs(DBSPCompiler compiler) {} @@ -258,14 +264,16 @@ public static void runAllTests() throws IOException, InterruptedException { List testers = test.createTesterCode(testNumber, RUST_DIRECTORY, inputFunctions); writer.add(test.ccs.circuit); for (var tester: testers) - writer.add(tester); + if (acceptTest.test(test.ccs)) + writer.add(tester); BaseSQLTests.testsExecuted++; testNumber++; } Utilities.enforce(firstCompiler != null); writer.write(firstCompiler); outputStream.close(); - Utilities.compileAndTestRust(RUST_DIRECTORY, true); + if (!skipRust) + Utilities.compileAndTestRust(RUST_DIRECTORY, true); } if (!toCheck.isEmpty()) { @@ -286,14 +294,16 @@ public static void runAllTests() throws IOException, InterruptedException { List testers = test.createTesterCode(testNumber, RUST_DIRECTORY, inputFunctions); writer.add(test.ccs.circuit); for (var tester: testers) - writer.add(tester); + if (acceptTest.test(test.ccs)) + writer.add(tester); BaseSQLTests.testsChecked++; testNumber++; } Utilities.enforce(firstCompiler != null); writer.write(firstCompiler); outputStream.close(); - Utilities.compileAndCheckRust(RUST_DIRECTORY, true); + if (!skipRust) + Utilities.compileAndCheckRust(RUST_DIRECTORY, true); } testsToRun.clear(); }