Skip to content

[SQL] Support for the MAP_CONCAT SQL function #6515

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mihaibudiu merged 1 commit into from
Jun 24, 2026
Merged

[SQL] Support for the MAP_CONCAT SQL function #6515

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions crates/sqllib/src/map.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,3 +313,23 @@ where
}

some_function1!(map_values [I: Ord + Clone, T: Clone], Map<I, T>, Array<T>);

#[doc(hidden)]
pub fn map_concat__<I, T>(left: Map<I, T>, right: Map<I, T>) -> Map<I, T>
where
I: Ord + Clone,
T: Clone,
{
if right.is_empty() {
return left;
}
if left.is_empty() {
return right;
}
let mut out = BTreeMap::new();
out.extend(left.iter().map(|(k, v)| (k.clone(), v.clone())));
out.extend(right.iter().map(|(k, v)| (k.clone(), v.clone())));
out.into()
}

some_function2!(map_concat [I: Ord + Clone, T: Clone], Map<I, T>, Map<I, T>, Map<I, T>);
1 change: 1 addition & 0 deletions docs.feldera.com/docs/sql/function-index.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@
* `MAKE_TIMESTAMP`: [datetime](datetime.md#make_timestamp)
* `MAP` (aggregate): [aggregates](aggregates.md#map)
* `MAP`: [map](map.md#map-literals)
* `MAP_CONCAT`: [map](map.md#map_concat)
* `MAP_CONTAINS_KEY`: [map](map.md#map_contains_key)
* `MAP_KEYS`: [map](map.md#map_keys)
* `MAP_VALUES`: [map](map.md#map_values)
Expand Down
3 changes: 2 additions & 1 deletion docs.feldera.com/docs/sql/map.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,10 @@ Comparison operations (`=`, `<>`, `!=`, `>`, `<`, `>=`, `<=`) can be applied to
|------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------|
| _map_`[`_key_`]` | Returns the element in the map with the specified key. If there is no such key, the result is `NULL`. | `MAP['x', 4, 'y', 3]['x']` => 4 |
| <a id="cardinality"></a>`CARDINALITY`(map) | Returns the number of key-value pairs in the map. | `CARDINALITY(MAP['x', 4])` => 1 |
| <a id="map_concat"></a>`MAP_CONCAT`(map1, map2) | Returns a map with the elements in both maps; keys in the second map win. Returns `NULL` if any argument is `NULL` | `MAP_CONCAT(MAP['x', 4], MAP['y', 2])` => `MAP['x', 4, 'y', 2]` |
| <a id="map_contains_key"></a>`MAP_CONTAINS_KEY`(map, key) | Returns true when the map has an item with the specified key; `NULL` if any argument is `NULL`. | `MAP_CONTAINS_KEY(MAP['x', 4], 'x')` => `true` |
| <a id="map_keys"></a>`MAP_KEYS`(map) | Returns a sorted ARRAY of the appropriate type with all the keys of the MAP. | `MAP_KEYS(MAP['x', 4, 'y', 5])` => `['x', 'y']` |
| <a id="map_values"></a>`MAP_VALUES`(map) | Returns an ARRAY of the appropriate type with all the values of the MAP. | `MAP_VALUES(MAP['x', 4, 'y', 5])` => `[4, 5]` |
| <a id="map_values"></a>`MAP_VALUES`(map) | Returns an ARRAY of the appropriate type with all the values of the MAP sorted by the order of the keys. | `MAP_VALUES(MAP['x', 4, 'y', 5])` => `[4, 5]` |

## The `UNNEST` Operator

Expand Down
2 changes: 0 additions & 2 deletions docs.feldera.com/docs/sql/unsupported-operations.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,12 +79,10 @@ Several `MAP` functions are not yet implemented:

| Function | Status |
|----------|--------|
| `MAP_CONCAT` | Not supported |
| `MAP_ENTRIES` | Not supported |
| `MAP_FROM_ARRAYS` | Not supported |
| `MAP_FROM_ENTRIES` | Not supported |
| `STR_TO_MAP` | Not supported |
| Building a `MAP` from a subquery returning a pair of columns | Not supported |

A list of supported MAP operations is available [here](./map.md).
See [#1907](https://github.com/feldera/feldera/issues/1907).
Expand Down
25 changes: 24 additions & 1 deletion ...SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2233,11 +2233,34 @@ else if (arg0Type.is(DBSPTypeMap.class))
return next;
}
case MAP_KEYS:
case MAP_VALUES:
case MAP_VALUES: {
validateArgCount(node, operationName, ops.size(), 1);
DBSPExpression arg0 = ops.get(0);
String method = getArrayOrMapCallName(call, arg0);
return new DBSPApplyExpression(node, method, type, arg0);
}
case MAP_CONCAT: {
if (ops.isEmpty()) {
throw new CompilationError("Function " + Utilities.singleQuote(operationName) +
" with 0 arguments is not supported", node);
}
if (ops.size() == 1) {
return ops.get(0);
}
for (int i = 0; i < ops.size(); i++) {
DBSPExpression argi = ops.get(i);
// Types may not match exactly
argi = argi.cast(argi.getNode(), type, DBSPCastExpression.CastType.SqlUnsafe);
ops.set(i, argi);
}
DBSPExpression current = ops.get(0);
for (int i = 1; i < ops.size(); i++) {
DBSPExpression argi = ops.get(i);
String method = getArrayOrMapCallName(call, current, argi);
current = new DBSPApplyExpression(node, method, type, current, argi);
}
return current;
}
case DOT:
default:
throw new UnimplementedException("Function " + Utilities.singleQuote(call.getOperator().toString())
Expand Down
7 changes: 2 additions & 5 deletions ...rc/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteFunctions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -692,11 +692,8 @@ record Func(SqlOperator function, String functionName, SqlLibrary library,
"runtime_aggtest/illarg_tests/test_{str_bin_type_fn,str_unicode_fn}.py", false),
new Func(SqlLibraryOperators.IFNULL, "IFNULL", SqlLibrary.BIG_QUERY,
"comparisons#ifnull", FunctionDocumentation.NO_FILE, false),
new Func(SqlLibraryOperators.MAP_KEYS, "MAP_KEYS", SqlLibrary.SPARK,
"map#map_keys",
"runtime_aggtest/illarg_tests/test_arr_map_type_fn.py", false),
new Func(SqlLibraryOperators.MAP_VALUES, "MAP_VALUES", SqlLibrary.SPARK,
"map#map_values", FunctionDocumentation.NO_FILE, false)
new Func(SqlLibraryOperators.MAP_CONCAT, "MAP_CONCAT", SqlLibrary.SPARK,
"map#map_concat", FunctionDocumentation.NO_FILE, false),
// new Func(SqlLibraryOperators.SAFE_ORDINAL, "SAFE_ORDINAL", SqlLibrary.BIG_QUERY, "array", FunctionDocumentation.NO_FILE, false),
};

Expand Down
16 changes: 16 additions & 0 deletions ...src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CustomFunctions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ public CustomFunctions() {
this.functions.add(new MakeDateFunction());
this.functions.add(new MakeTimeFunction());
this.functions.add(new MakeTimestampFunction());
this.functions.add(new MapKeysFunction());
this.functions.add(new MapValuesFunction());
this.functions.add(new NowFunction());
this.functions.add(new ParseDateFunction());
this.functions.add(new ParseJsonFunction());
Expand Down Expand Up @@ -185,6 +187,20 @@ public CalciteFunctionClone(SqlFunction calciteFunction, String documentationFil
}
}

static class MapKeysFunction extends CalciteFunctionClone {
// Cannot be folded by Calcite since the result order is not deterministic in Calcite
private MapKeysFunction() {
super(SqlLibraryOperators.MAP_KEYS, "map#map_keys", "runtime_aggtest/illarg_tests/test_arr_map_type_fn.py");
}
}

static class MapValuesFunction extends CalciteFunctionClone {
// Cannot be folded by Calcite since the result order is not deterministic in Calcite
private MapValuesFunction() {
super(SqlLibraryOperators.MAP_VALUES, "map#map_values", FunctionDocumentation.NO_FILE);
}
}

static class FormatDateFunction extends CalciteFunctionClone {
private FormatDateFunction() {
super(SqlLibraryOperators.FORMAT_DATE, "datetime#format_date", FunctionDocumentation.NO_FILE);
Expand Down
Loading