diff --git a/crates/sqllib/src/map.rs b/crates/sqllib/src/map.rs index bae0e80ebda..fc4b07e2506 100644 --- a/crates/sqllib/src/map.rs +++ b/crates/sqllib/src/map.rs @@ -313,3 +313,23 @@ where } some_function1!(map_values [I: Ord + Clone, T: Clone], Map, Array); + +#[doc(hidden)] +pub fn map_concat__(left: Map, right: Map) -> Map +where + I: Ord + Clone, + T: Clone, +{ + if right.is_empty() { + return left; + } + if left.is_empty() { + return right; + } + let mut out = BTreeMap::new(); + out.extend(left.iter().map(|(k, v)| (k.clone(), v.clone()))); + out.extend(right.iter().map(|(k, v)| (k.clone(), v.clone()))); + out.into() +} + +some_function2!(map_concat [I: Ord + Clone, T: Clone], Map, Map, Map); diff --git a/docs.feldera.com/docs/sql/function-index.md b/docs.feldera.com/docs/sql/function-index.md index 00b1ca87410..43d673ab4f0 100644 --- a/docs.feldera.com/docs/sql/function-index.md +++ b/docs.feldera.com/docs/sql/function-index.md @@ -163,6 +163,7 @@ * `MAKE_TIMESTAMP`: [datetime](datetime.md#make_timestamp) * `MAP` (aggregate): [aggregates](aggregates.md#map) * `MAP`: [map](map.md#map-literals) +* `MAP_CONCAT`: [map](map.md#map_concat) * `MAP_CONTAINS_KEY`: [map](map.md#map_contains_key) * `MAP_KEYS`: [map](map.md#map_keys) * `MAP_VALUES`: [map](map.md#map_values) diff --git a/docs.feldera.com/docs/sql/map.md b/docs.feldera.com/docs/sql/map.md index 8240a485c9e..ac4deeea377 100644 --- a/docs.feldera.com/docs/sql/map.md +++ b/docs.feldera.com/docs/sql/map.md @@ -49,9 +49,10 @@ Comparison operations (`=`, `<>`, `!=`, `>`, `<`, `>=`, `<=`) can be applied to |------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------| | _map_`[`_key_`]` | Returns the element in the map with the specified key. If there is no such key, the result is `NULL`. | `MAP['x', 4, 'y', 3]['x']` => 4 | | `CARDINALITY`(map) | Returns the number of key-value pairs in the map. | `CARDINALITY(MAP['x', 4])` => 1 | +| `MAP_CONCAT`(map1, map2) | Returns a map with the elements in both maps; keys in the second map win. Returns `NULL` if any argument is `NULL` | `MAP_CONCAT(MAP['x', 4], MAP['y', 2])` => `MAP['x', 4, 'y', 2]` | | `MAP_CONTAINS_KEY`(map, key) | Returns true when the map has an item with the specified key; `NULL` if any argument is `NULL`. | `MAP_CONTAINS_KEY(MAP['x', 4], 'x')` => `true` | | `MAP_KEYS`(map) | Returns a sorted ARRAY of the appropriate type with all the keys of the MAP. | `MAP_KEYS(MAP['x', 4, 'y', 5])` => `['x', 'y']` | -| `MAP_VALUES`(map) | Returns an ARRAY of the appropriate type with all the values of the MAP. | `MAP_VALUES(MAP['x', 4, 'y', 5])` => `[4, 5]` | +| `MAP_VALUES`(map) | Returns an ARRAY of the appropriate type with all the values of the MAP sorted by the order of the keys. | `MAP_VALUES(MAP['x', 4, 'y', 5])` => `[4, 5]` | ## The `UNNEST` Operator diff --git a/docs.feldera.com/docs/sql/unsupported-operations.md b/docs.feldera.com/docs/sql/unsupported-operations.md index a4cfe21bff8..8d4018309e6 100644 --- a/docs.feldera.com/docs/sql/unsupported-operations.md +++ b/docs.feldera.com/docs/sql/unsupported-operations.md @@ -79,12 +79,10 @@ Several `MAP` functions are not yet implemented: | Function | Status | |----------|--------| -| `MAP_CONCAT` | Not supported | | `MAP_ENTRIES` | Not supported | | `MAP_FROM_ARRAYS` | Not supported | | `MAP_FROM_ENTRIES` | Not supported | | `STR_TO_MAP` | Not supported | -| Building a `MAP` from a subquery returning a pair of columns | Not supported | A list of supported MAP operations is available [here](./map.md). See [#1907](https://github.com/feldera/feldera/issues/1907). diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java index 811435a31af..ef59edbf0bc 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java @@ -2233,11 +2233,34 @@ else if (arg0Type.is(DBSPTypeMap.class)) return next; } case MAP_KEYS: - case MAP_VALUES: + case MAP_VALUES: { validateArgCount(node, operationName, ops.size(), 1); DBSPExpression arg0 = ops.get(0); String method = getArrayOrMapCallName(call, arg0); return new DBSPApplyExpression(node, method, type, arg0); + } + case MAP_CONCAT: { + if (ops.isEmpty()) { + throw new CompilationError("Function " + Utilities.singleQuote(operationName) + + " with 0 arguments is not supported", node); + } + if (ops.size() == 1) { + return ops.get(0); + } + for (int i = 0; i < ops.size(); i++) { + DBSPExpression argi = ops.get(i); + // Types may not match exactly + argi = argi.cast(argi.getNode(), type, DBSPCastExpression.CastType.SqlUnsafe); + ops.set(i, argi); + } + DBSPExpression current = ops.get(0); + for (int i = 1; i < ops.size(); i++) { + DBSPExpression argi = ops.get(i); + String method = getArrayOrMapCallName(call, current, argi); + current = new DBSPApplyExpression(node, method, type, current, argi); + } + return current; + } case DOT: default: throw new UnimplementedException("Function " + Utilities.singleQuote(call.getOperator().toString()) diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteFunctions.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteFunctions.java index 7777bbf4603..31c8c86f038 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteFunctions.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteFunctions.java @@ -692,11 +692,8 @@ record Func(SqlOperator function, String functionName, SqlLibrary library, "runtime_aggtest/illarg_tests/test_{str_bin_type_fn,str_unicode_fn}.py", false), new Func(SqlLibraryOperators.IFNULL, "IFNULL", SqlLibrary.BIG_QUERY, "comparisons#ifnull", FunctionDocumentation.NO_FILE, false), - new Func(SqlLibraryOperators.MAP_KEYS, "MAP_KEYS", SqlLibrary.SPARK, - "map#map_keys", - "runtime_aggtest/illarg_tests/test_arr_map_type_fn.py", false), - new Func(SqlLibraryOperators.MAP_VALUES, "MAP_VALUES", SqlLibrary.SPARK, - "map#map_values", FunctionDocumentation.NO_FILE, false) + new Func(SqlLibraryOperators.MAP_CONCAT, "MAP_CONCAT", SqlLibrary.SPARK, + "map#map_concat", FunctionDocumentation.NO_FILE, false), // new Func(SqlLibraryOperators.SAFE_ORDINAL, "SAFE_ORDINAL", SqlLibrary.BIG_QUERY, "array", FunctionDocumentation.NO_FILE, false), }; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CustomFunctions.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CustomFunctions.java index b68b3cf3dde..343b80e6894 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CustomFunctions.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CustomFunctions.java @@ -83,6 +83,8 @@ public CustomFunctions() { this.functions.add(new MakeDateFunction()); this.functions.add(new MakeTimeFunction()); this.functions.add(new MakeTimestampFunction()); + this.functions.add(new MapKeysFunction()); + this.functions.add(new MapValuesFunction()); this.functions.add(new NowFunction()); this.functions.add(new ParseDateFunction()); this.functions.add(new ParseJsonFunction()); @@ -185,6 +187,20 @@ public CalciteFunctionClone(SqlFunction calciteFunction, String documentationFil } } + static class MapKeysFunction extends CalciteFunctionClone { + // Cannot be folded by Calcite since the result order is not deterministic in Calcite + private MapKeysFunction() { + super(SqlLibraryOperators.MAP_KEYS, "map#map_keys", "runtime_aggtest/illarg_tests/test_arr_map_type_fn.py"); + } + } + + static class MapValuesFunction extends CalciteFunctionClone { + // Cannot be folded by Calcite since the result order is not deterministic in Calcite + private MapValuesFunction() { + super(SqlLibraryOperators.MAP_VALUES, "map#map_values", FunctionDocumentation.NO_FILE); + } + } + static class FormatDateFunction extends CalciteFunctionClone { private FormatDateFunction() { super(SqlLibraryOperators.FORMAT_DATE, "datetime#format_date", FunctionDocumentation.NO_FILE); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/MapTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/MapTests.java index 829c300854c..40b9c61b596 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/MapTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/MapTests.java @@ -3,11 +3,10 @@ import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.sqlCompiler.compiler.TestUtil; import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteObject; -import org.dbsp.sqlCompiler.compiler.sql.tools.BaseSQLTests; import org.dbsp.sqlCompiler.compiler.sql.tools.Change; import org.dbsp.sqlCompiler.compiler.sql.tools.InputOutputChange; import org.dbsp.sqlCompiler.compiler.sql.tools.InputOutputChangeStream; -import org.dbsp.sqlCompiler.ir.expression.DBSPArrayExpression; +import org.dbsp.sqlCompiler.compiler.sql.tools.SqlIoTest; import org.dbsp.sqlCompiler.ir.expression.DBSPTupleExpression; import org.dbsp.sqlCompiler.ir.expression.literal.DBSPI32Literal; import org.dbsp.sqlCompiler.ir.expression.DBSPMapExpression; @@ -22,7 +21,7 @@ import java.nio.charset.Charset; -public class MapTests extends BaseSQLTests { +public class MapTests extends SqlIoTest { public DBSPCompiler compileQuery(String statements, String query) { DBSPCompiler compiler = this.testCompiler(); compiler.options.languageOptions.optimizationLevel = 0; @@ -32,14 +31,14 @@ public DBSPCompiler compileQuery(String statements, String query) { return compiler; } - void testQuery(String statements, String query, InputOutputChangeStream streams) { + void testQuery(String query, InputOutputChangeStream streams) { query = "CREATE VIEW V AS " + query; - DBSPCompiler compiler = this.compileQuery(statements, query); + DBSPCompiler compiler = this.compileQuery("", query); this.getCCS(compiler, streams); } private void testQuery(String query, DBSPZSetExpression expression) { - this.testQuery("", query, + this.testQuery(query, new InputOutputChangeStream().addChange( new InputOutputChange(new Change(), new Change("V", expression)))); } @@ -47,30 +46,27 @@ private void testQuery(String query, DBSPZSetExpression expression) { @Test public void testDuplicateKeys() { var compiler = this.compileQuery("", "CREATE VIEW V AS SELECT MAP['hi', 1, 'hi', 2]"); - TestUtil.assertMessagesContain(compiler, "Duplicate MAP key"); + TestUtil.assertMessagesContain(compiler, "warning: Duplicate MAP key"); } @Test public void mapLiteralTest() { - String query = "SELECT MAP['hi',2]"; - DBSPType str = DBSPTypeString.varchar(false); - this.testQuery(query, new DBSPZSetExpression( - new DBSPTupleExpression(new DBSPMapExpression( - new DBSPTypeMap( - str, - new DBSPTypeInteger(CalciteObject.EMPTY, 32, true, false), - false), - Linq.list(new DBSPStringLiteral(CalciteObject.EMPTY, str, "hi", Charset.defaultCharset()), - new DBSPI32Literal(2)))))); + this.qst(""" + SELECT MAP['hi',2]; + r + --- + { hi: 2 } + (1 row)"""); } @Test public void mapIndexTest() { - String query = "SELECT MAP['hi',2]['hi'], MAP['hi',2]['x']"; - this.testQuery(query, new DBSPZSetExpression( - new DBSPTupleExpression( - new DBSPI32Literal(2, true), - new DBSPI32Literal()))); + this.qst(""" + SELECT MAP['hi',2]['hi'], MAP['hi',2]['x']; + e0 | e1 + --------- + 2 | + (1 row)"""); } @Test @@ -89,88 +85,50 @@ public void mapBlackboxTest() { @Test public void mapCardinalityTest() { - String query = "SELECT CARDINALITY(MAP['hi',2])"; - this.testQuery(query, new DBSPZSetExpression( - new DBSPTupleExpression( - new DBSPI32Literal(1)))); + this.qst(""" + SELECT CARDINALITY(MAP['hi',2]); + r + --- + 1 + (1 row)"""); } @Test public void testMapSubquery() { - String ddl = "CREATE TABLE T(v varchar, x int)"; - String query = "SELECT MAP(SELECT * FROM T)"; - DBSPZSetExpression input = new DBSPZSetExpression( - new DBSPTupleExpression( - new DBSPStringLiteral("hello", true), - new DBSPI32Literal(10, true)), - new DBSPTupleExpression( - new DBSPStringLiteral("there", true), - new DBSPI32Literal(5, true))); - DBSPTypeMap mapType = new DBSPTypeMap( - DBSPTypeString.varchar(true), - new DBSPTypeInteger(CalciteObject.EMPTY, 32, true ,true), false); - DBSPZSetExpression result = new DBSPZSetExpression( - new DBSPTupleExpression( - new DBSPMapExpression( - mapType, - Linq.list( - new DBSPStringLiteral("there", true), - new DBSPI32Literal(5, true), - new DBSPStringLiteral("hello", true), - new DBSPI32Literal(10, true)) - ))); - this.testQuery(ddl, query, - new InputOutputChangeStream().addPair(new Change("T", input), new Change("V", result))); + var ccs = this.getCCS(""" + CREATE TABLE T(v varchar, x int); + CREATE VIEW V AS SELECT MAP(SELECT * FROM T);"""); + ccs.stepWeightOne("INSERT INTO T VALUES('hello', 10), ('there', 5)", """ + map + ------- + { hello: 10, there: 5 }"""); } @Test public void testUnnestMap() { - String sql = "select * from UNNEST(map['a', 12])"; - DBSPZSetExpression result = new DBSPZSetExpression( - new DBSPTupleExpression( - new DBSPStringLiteral("a", false), - new DBSPI32Literal(12, false))); - this.testQuery("", sql, - new InputOutputChangeStream().addPair(new Change(), new Change("V", result))); - } - - @Test - public void testUnnestMap2() { - String sql = "select * from UNNEST(map['a', 12, 'b', 15, 'c', NULL])"; - DBSPZSetExpression result = new DBSPZSetExpression( - new DBSPTupleExpression( - new DBSPStringLiteral("a", false), - new DBSPI32Literal(12, true)), - new DBSPTupleExpression( - new DBSPStringLiteral("b", false), - new DBSPI32Literal(15, true)), - new DBSPTupleExpression( - new DBSPStringLiteral("c", false), - new DBSPI32Literal())); - this.testQuery("", sql, - new InputOutputChangeStream().addPair(new Change(), new Change("V", result))); - } - - @Test - public void testUnnestMapFields() { - String sql = - "WITH T(i, m) as (VALUES(1, MAP[1, 2, 3, 4]), (2, MAP[5, NULL])) " + - "SELECT T.i, k, v FROM T CROSS JOIN UNNEST(T.m) AS pair(k, v)"; - DBSPZSetExpression result = new DBSPZSetExpression( - new DBSPTupleExpression( - new DBSPI32Literal(1, false), - new DBSPI32Literal(1, false), - new DBSPI32Literal(2, true)), - new DBSPTupleExpression( - new DBSPI32Literal(1, false), - new DBSPI32Literal(3, false), - new DBSPI32Literal(4, true)), - new DBSPTupleExpression( - new DBSPI32Literal(2, false), - new DBSPI32Literal(5, false), - new DBSPI32Literal())); - this.testQuery("", sql, new InputOutputChangeStream() - .addPair(new Change(), new Change("V", result))); + this.qst(""" + select * from UNNEST(map['a', 12]); + k | v + ------- + a | 12 + (1 row) + + select * from UNNEST(map['a', 12, 'b', 15, 'c', NULL]); + k | v + ------- + a | 12 + b | 15 + c | + (3 rows) + + WITH T(i, m) as (VALUES(1, MAP[1, 2, 3, 4]), (2, MAP[5, NULL])) + SELECT T.i, k, v FROM T CROSS JOIN UNNEST(T.m) AS pair(k, v); + i | k | v + ----------- + 1 | 1 | 2 + 1 | 3 | 4 + 2 | 5 | + (3 rows)"""); } @Test @@ -181,14 +139,12 @@ public void nullMapKey() { @Test public void testMapKeys() { - String sql = "SELECT map_keys(map['foo', 1, 'bar', 2])"; - DBSPZSetExpression result = new DBSPZSetExpression( - new DBSPTupleExpression(new DBSPArrayExpression( - false, - new DBSPStringLiteral("bar"), - new DBSPStringLiteral("foo")))); - this.testQuery("", sql, new InputOutputChangeStream() - .addPair(new Change(), new Change("V", result))); + this.qst(""" + SELECT map_keys(map['foo', 1, 'bar', 2]); + keys + -------------- + { bar, foo } + (1 row)"""); } @Test @@ -218,14 +174,12 @@ SELECT cast(contacts as MAP) contacts @Test public void testMapValues() { - String sql = "SELECT map_values(map['foo', 1, 'bar', 2])"; - DBSPZSetExpression result = new DBSPZSetExpression( - new DBSPTupleExpression(new DBSPArrayExpression( - false, - new DBSPI32Literal(2), - new DBSPI32Literal(1)))); - this.testQuery("", sql, new InputOutputChangeStream() - .addPair(new Change(), new Change("V", result))); + this.qst(""" + SELECT map_values(map['foo', 1, 'bar', 2]); + r + -------- + { 2, 1 } + (1 row)"""); } @Test @@ -252,4 +206,49 @@ SELECT cast(contacts as MAP) contacts null {"f":1}"""); } + + @Test + public void testMapConcat() { + this.qst(""" + SELECT MAP_CONCAT( + MAP[ 1, 'a' ], + MAP[ 2, 'b' ] + ); + +--------------+ + | EXPR$0 | + +--------------+ + | {1: a, 2: b} | + +--------------+ + (1 row) + + SELECT MAP_CONCAT(CAST(NULL AS MAP), MAP[1, 2]); + r + ---- + NULL + (1 row) + + SELECT MAP_CONCAT(MAP[1, 'a'], MAP[1, 'b']); + r + ---- + {1: b} + (1 row) + + SELECT MAP_CONCAT(MAP[1, 2], MAP[1.0, NULL]); + r + --- + {1.0: NULL} + (1 row) + + SELECT MAP_CONCAT(MAP[1, 2]); + r + --- + {1: 2} + (1 row) + + SELECT MAP_CONCAT(MAP[1, 2], MAP[1, 3], MAP[1, 4]); + r + --- + {1: 4} + (1 row)"""); + } } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/TableParser.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/TableParser.java index 99f09e4f85e..6ce62f798f5 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/TableParser.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/TableParser.java @@ -7,6 +7,7 @@ import org.dbsp.sqlCompiler.compiler.frontend.TableData; import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteObject; import org.dbsp.sqlCompiler.ir.expression.DBSPExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPMapExpression; import org.dbsp.sqlCompiler.ir.expression.DBSPTupleExpression; import org.dbsp.sqlCompiler.ir.expression.literal.DBSPBinaryLiteral; import org.dbsp.sqlCompiler.ir.expression.literal.DBSPBoolLiteral; @@ -37,6 +38,7 @@ import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTuple; import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTupleBase; import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeArray; +import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeMap; import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeZSet; import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeInteger; import org.dbsp.util.Linq; @@ -409,6 +411,32 @@ static DBSPExpression parseValue(DBSPType fieldType, String data, boolean trimTr yield new DBSPTupleExpression(CalciteObject.EMPTY, tuple, fields); } } + case MAP -> { + // This is rather fragile, but there is not fool-proof solution for parsing this + DBSPTypeMap map = fieldType.to(DBSPTypeMap.class); + if (trimmed.equals("NULL")) { + yield map.none(); + } else { + if (!trimmed.startsWith("{") || !trimmed.endsWith("}")) + throw new UnimplementedException("Expected map constant to be bracketed: " + trimmed); + trimmed = trimmed.substring(1, trimmed.length() - 1); + Utilities.enforce(!trimmed.isEmpty()); + + String[] parts = trimmed.split(","); + List keys = new ArrayList<>(); + List values = new ArrayList<>(); + for (String part: parts) { + String[] kv = part.split(":"); + Utilities.enforce(kv.length == 2, () -> "Map entry is not in 'key: value' form"); + DBSPExpression key = parseValue(map.getKeyType(), kv[0], trimTrailingSpaces); + keys.add(key); + DBSPExpression value = parseValue(map.getValueType(), kv[1], trimTrailingSpaces); + values.add(value); + } + + yield new DBSPMapExpression(map, keys, values); + } + } default -> throw new UnimplementedException( "Support for parsing fields of type " + fieldType + " not yet implemented", fieldType); };