diff --git a/crates/sqllib/src/variant.rs b/crates/sqllib/src/variant.rs index baf265d16f3..6472e27bab4 100644 --- a/crates/sqllib/src/variant.rs +++ b/crates/sqllib/src/variant.rs @@ -19,7 +19,7 @@ use serde::{Deserialize, Serialize}; use size_of::SizeOf; use std::borrow::Cow; use std::cmp::Ord; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::error::Error; use std::fmt; use std::fmt::Display; @@ -1026,6 +1026,449 @@ pub fn variantnull() -> Variant { Variant::VariantNull } +/////////////// JSON_EACH_* functions + +/// True if the variant holds a numeric value with no fractional part. +/// The integer extraction functions use this filter so that a field holding +/// e.g. 2.5 is not silently truncated to 2; such fields can still be +/// selected with `variant_filter_`. +fn is_integral_numeric_variant(value: &Variant) -> bool { + match value { + Variant::TinyInt(_) + | Variant::SmallInt(_) + | Variant::Int(_) + | Variant::BigInt(_) + | Variant::UTinyInt(_) + | Variant::USmallInt(_) + | Variant::UInt(_) + | Variant::UBigInt(_) => true, + Variant::Real(x) => x.into_inner().fract() == 0.0, + Variant::Double(x) => x.into_inner().fract() == 0.0, + Variant::SqlDecimal((sig, exp)) => match 10i128.checked_pow(*exp as u32) { + Some(divisor) => sig % divisor == 0, + None => *sig == 0, + }, + _ => false, + } +} + +/// Extract from a variant holding a map all fields whose values have a runtime +/// type accepted by `keep` and convert to `T`. The `keep` filter selects by +/// runtime type before conversion, so e.g. a string field never converts to a +/// number. Fields that do not qualify or do not convert are omitted; so are +/// fields with non-string keys. A variant that does not hold a map produces +/// an empty result. +fn json_each_typed(value: Variant, keep: fn(&Variant) -> bool) -> Map> +where + T: TryFrom, +{ + let mut result = BTreeMap::>::new(); + if let Variant::Map(map) = value { + for (key, val) in map.iter() { + let Variant::String(key) = key else { continue }; + if !keep(val) { + continue; + } + if let Ok(converted) = T::try_from(val.clone()) { + result.insert(key.clone(), Some(converted)); + } + } + } + result.into() +} + +macro_rules! json_each { + ($type_name:ident, $type:ty, $keep:expr) => { + ::paste::paste! { + #[doc(hidden)] + pub fn [](value: Variant) -> Map> { + json_each_typed(value, $keep) + } + + crate::some_polymorphic_function1!([], V, Variant, Map>); + } + }; +} + +json_each!(bigint, i64, is_integral_numeric_variant); +json_each!(string, SqlString, |v| matches!(v, Variant::String(_))); +json_each!(boolean, bool, |v| matches!(v, Variant::Boolean(_))); +json_each!(date, Date, |v| matches!( + v, + Variant::Date(_) | Variant::String(_) +)); +json_each!(time, Time, |v| matches!( + v, + Variant::Time(_) | Variant::String(_) +)); +json_each!(timestamp, Timestamp, |v| matches!( + v, + Variant::Timestamp(_) | Variant::String(_) +)); + +/////////////// JSON_OBJECT_KEYS and JSON_KEYS + +/// The top-level keys of a variant holding a map, sorted, following the +/// Postgres `json_object_keys` function. Non-string keys are skipped. +/// A variant that does not hold a map produces an empty result. +#[doc(hidden)] +pub fn json_object_keys_V(value: Variant) -> Array { + let mut result = Vec::new(); + if let Variant::Map(map) = value { + for key in map.keys() { + if let Variant::String(key) = key { + result.push(key.clone()); + } + } + } + result.into() +} + +crate::some_polymorphic_function1!(json_object_keys, V, Variant, Array); + +/// The keys of all nested objects in a variant, as dot-joined paths, +/// deduplicated and sorted, following the BigQuery `JSON_KEYS` function in +/// 'strict' mode: arrays are not traversed, and keys that are not +/// identifiers are double-quoted. Non-string keys are skipped. +/// A variant that does not hold a map produces an empty result. +#[doc(hidden)] +pub fn json_keys_V(value: Variant) -> Array { + fn collect(prefix: &str, map: &Map, result: &mut BTreeSet) { + for (key, val) in map.iter() { + let Variant::String(key) = key else { continue }; + let path = append_path_component(prefix, key.str()); + if let Variant::Map(inner) = val { + collect(&path, inner, result); + } + result.insert(SqlString::from(path)); + } + } + let mut result = BTreeSet::new(); + if let Variant::Map(map) = value { + collect("", &map, &mut result); + } + result.into_iter().collect::>().into() +} + +crate::some_polymorphic_function1!(json_keys, V, Variant, Array); + +/////////////// VARIANT_FILTER and VARIANT_DEEP_FILTER + +/// An item is kept only when the predicate is 'true', like a SQL WHERE clause. +fn predicate_keeps>>(result: B) -> bool { + result.into() == Some(true) +} + +/// True if the key can appear in a path without quotes. +fn is_identifier_key(key: &str) -> bool { + let mut chars = key.chars(); + match chars.next() { + Some(c) if c.is_ascii_alphabetic() || c == '_' => {} + _ => return false, + } + chars.all(|c| c.is_ascii_alphanumeric() || c == '_') +} + +/// Append a member-access component to a path. A key that is not an +/// identifier is double-quoted, with backslashes escaping embedded quotes +/// and backslashes, so that paths are unambiguous: the key `a.b` produces +/// the path component `"a.b"`, distinct from the nested path `a.b`. +fn append_path_component(prefix: &str, key: &str) -> String { + let mut result = String::with_capacity(prefix.len() + key.len() + 4); + result.push_str(prefix); + if !prefix.is_empty() { + result.push('.'); + } + if is_identifier_key(key) { + result.push_str(key); + } else { + result.push('"'); + for c in key.chars() { + if c == '"' || c == '\\' { + result.push('\\'); + } + result.push(c); + } + result.push('"'); + } + result +} + +/// Filter a variant with a predicate over (label, value) items. A variant +/// holding a map contributes one item per field, labeled by its key; any +/// other variant is a single item with a `None` label, kept whole or dropped +/// entirely. A dropped non-map variant produces `None` (SQL `NULL`). +#[doc(hidden)] +pub fn variant_filter_(value: Variant, predicate: F) -> Option +where + F: Fn(&Option, &Variant) -> B, + B: Into>, +{ + match value { + Variant::Map(map) => { + let mut result = BTreeMap::new(); + for (key, val) in map.iter() { + if predicate_keeps(predicate(&Some(key.clone()), val)) { + result.insert(key.clone(), val.clone()); + } + } + Some(Variant::Map(result.into())) + } + other => { + if predicate_keeps(predicate(&None, &other)) { + Some(other) + } else { + None + } + } + } +} + +#[doc(hidden)] +pub fn variant_filterN(value: Option, predicate: F) -> Option +where + F: Fn(&Option, &Variant) -> B, + B: Into>, +{ + variant_filter_(value?, predicate) +} + +/// Recurse into a kept container; leaves are kept as they are. +fn deep_filter_value(path: &str, value: &Variant, predicate: &F) -> Variant +where + F: Fn(&Option, &Variant) -> B, + B: Into>, +{ + match value { + Variant::Map(inner) => Variant::Map(deep_filter_map(path, inner, predicate)), + Variant::Array(inner) => Variant::Array(deep_filter_array(path, inner, predicate)), + other => other.clone(), + } +} + +fn deep_filter_map( + prefix: &str, + map: &Map, + predicate: &F, +) -> Map +where + F: Fn(&Option, &Variant) -> B, + B: Into>, +{ + let mut result = BTreeMap::new(); + for (key, val) in map.iter() { + let Variant::String(k) = key else { + // A key that is not a string cannot appear in a path; + // keep the field untouched rather than deleting data silently + result.insert(key.clone(), val.clone()); + continue; + }; + let path = append_path_component(prefix, k.str()); + let label = Some(SqlString::from_ref(&path)); + if predicate_keeps(predicate(&label, val)) { + result.insert(key.clone(), deep_filter_value(&path, val, predicate)); + } + } + result.into() +} + +fn deep_filter_array(prefix: &str, array: &Array, predicate: &F) -> Array +where + F: Fn(&Option, &Variant) -> B, + B: Into>, +{ + let mut result = Vec::new(); + for (index, val) in array.iter().enumerate() { + // SQL array indexes start from 1 + let path = format!("{prefix}[{}]", index + 1); + let label = Some(SqlString::from_ref(&path)); + if predicate_keeps(predicate(&label, val)) { + result.push(deep_filter_value(&path, val, predicate)); + } + } + result.into() +} + +/// Like `variant_filter_`, but recursive: the label is the dot-joined path of +/// the item, a string (array elements use 1-based bracket components, e.g. +/// "a[1].b"; keys that are not identifiers are double-quoted), and containers +/// kept by the predicate have their contents filtered recursively. The +/// predicate receives the original, unfiltered value of each item. +#[doc(hidden)] +pub fn variant_deep_filter_(value: Variant, predicate: F) -> Option +where + F: Fn(&Option, &Variant) -> B, + B: Into>, +{ + match value { + Variant::Map(map) => Some(Variant::Map(deep_filter_map("", &map, &predicate))), + Variant::Array(array) => Some(Variant::Array(deep_filter_array("", &array, &predicate))), + other => { + if predicate_keeps(predicate(&None, &other)) { + Some(other) + } else { + None + } + } + } +} + +#[doc(hidden)] +pub fn variant_deep_filterN(value: Option, predicate: F) -> Option +where + F: Fn(&Option, &Variant) -> B, + B: Into>, +{ + variant_deep_filter_(value?, predicate) +} + +/////////////// VARIANT_MAP + +/// Map a variant with a function over (label, value) items, building an +/// isomorphic result: a variant holding a map produces a map with the same +/// keys and the mapped values; any other variant is a single item with a +/// `None` label whose mapped value is the result. A SQL `NULL` produced by +/// the mapper becomes a variant `SqlNull` inside a map. Top-level only. +#[doc(hidden)] +pub fn variant_map_(value: Variant, mapper: F) -> Option +where + F: Fn(&Option, &Variant) -> R, + R: Into>, +{ + match value { + Variant::Map(map) => { + let mut result = BTreeMap::new(); + for (key, val) in map.iter() { + let mapped = mapper(&Some(key.clone()), val) + .into() + .unwrap_or(Variant::SqlNull); + result.insert(key.clone(), mapped); + } + Some(Variant::Map(result.into())) + } + other => mapper(&None, &other).into(), + } +} + +#[doc(hidden)] +pub fn variant_mapN(value: Option, mapper: F) -> Option +where + F: Fn(&Option, &Variant) -> R, + R: Into>, +{ + variant_map_(value?, mapper) +} + +/////////////// VARIANT_DEEP_MAP + +/// Map one node; containers recurse, leaves are transformed. +fn deep_map_value(path: &str, value: &Variant, mapper: &F) -> Variant +where + F: Fn(&Option, &Variant) -> R, + R: Into>, +{ + match value { + Variant::Map(inner) => Variant::Map(deep_map_map(path, inner, mapper)), + Variant::Array(inner) => Variant::Array(deep_map_array(path, inner, mapper)), + leaf => mapper(&Some(SqlString::from_ref(path)), leaf) + .into() + .unwrap_or(Variant::SqlNull), + } +} + +fn deep_map_map( + prefix: &str, + map: &Map, + mapper: &F, +) -> Map +where + F: Fn(&Option, &Variant) -> R, + R: Into>, +{ + let mut result = BTreeMap::new(); + for (key, val) in map.iter() { + let Variant::String(k) = key else { + // A key that is not a string cannot appear in a path; + // keep the field untouched + result.insert(key.clone(), val.clone()); + continue; + }; + let path = append_path_component(prefix, k.str()); + result.insert(key.clone(), deep_map_value(&path, val, mapper)); + } + result.into() +} + +fn deep_map_array(prefix: &str, array: &Array, mapper: &F) -> Array +where + F: Fn(&Option, &Variant) -> R, + R: Into>, +{ + let mut result = Vec::with_capacity(array.len()); + for (index, val) in array.iter().enumerate() { + // SQL array indexes start from 1 + let path = format!("{prefix}[{}]", index + 1); + result.push(deep_map_value(&path, val, mapper)); + } + result.into() +} + +/// Like `variant_map_`, but recursive, and applied only to leaves: the +/// structure of nested objects and arrays is preserved exactly, and the +/// mapper transforms every non-container value, labeled by its dot-joined +/// path (array elements use 1-based bracket components, e.g. "a[1].b"; +/// keys that are not identifiers are double-quoted). JSON nulls are +/// leaves too. +#[doc(hidden)] +pub fn variant_deep_map_(value: Variant, mapper: F) -> Option +where + F: Fn(&Option, &Variant) -> R, + R: Into>, +{ + match value { + Variant::Map(map) => Some(Variant::Map(deep_map_map("", &map, &mapper))), + Variant::Array(array) => Some(Variant::Array(deep_map_array("", &array, &mapper))), + leaf => mapper(&None, &leaf).into(), + } +} + +#[doc(hidden)] +pub fn variant_deep_mapN(value: Option, mapper: F) -> Option +where + F: Fn(&Option, &Variant) -> R, + R: Into>, +{ + variant_deep_map_(value?, mapper) +} + +/////////////// VARIANT_MERGE + +/// Merge two variants recursively, following the JSON Merge Patch algorithm +/// (RFC 7386) with one difference: JSON null values are ordinary values and +/// never delete fields. When both arguments hold maps, their fields are +/// merged, recursing on common keys; in every other case, including two +/// arrays, the second argument wins. +#[doc(hidden)] +pub fn variant_merge_V_V(left: Variant, right: Variant) -> Variant { + match (left, right) { + (left @ Variant::Map(_), Variant::Map(right)) if right.is_empty() => left, + (Variant::Map(left), Variant::Map(right)) => { + let mut result = (*left).clone(); + for (key, value) in right.iter() { + let merged = match result.remove(key) { + Some(existing) => variant_merge_V_V(existing, value.clone()), + None => value.clone(), + }; + result.insert(key.clone(), merged); + } + Variant::Map(result.into()) + } + (_, right) => right, + } +} + +crate::some_polymorphic_function2!(variant_merge, V, Variant, V, Variant, Variant); + pub fn from_json_string(json: &str) -> Option where T: for<'de> DeserializeWithContext<'de, SqlSerdeConfig, Variant>, @@ -1326,4 +1769,431 @@ mod test { serde_json::from_str::(&s).unwrap() ); } + + /// A heterogeneous JSON object exercising every extraction function. + fn each_test_object() -> Variant { + serde_json::from_str::( + r#"{ + "i": 1, + "neg": -5, + "big": 5000000000, + "huge": 18446744073709551615, + "dec": 2.5, + "s": "text", + "snum": "7", + "b": true, + "n": null, + "arr": [1, 2], + "obj": {"x": 1}, + "date": "2024-01-01", + "time": "17:30:40", + "ts": "2024-12-19 16:39:57" + }"#, + ) + .unwrap() + } + + fn keys(map: &crate::Map>) -> Vec<&str> { + map.keys().map(|k| k.str()).collect() + } + + #[test] + fn json_each_bigint_extracts_only_numeric_fields_in_range() { + use super::json_each_bigint_V; + + let bigints = json_each_bigint_V(each_test_object()); + // "huge" exceeds the i64 range; "dec" is fractional; + // "snum" is a string, never parsed; "n" is a null + assert_eq!(keys(&bigints), vec!["big", "i", "neg"]); + assert_eq!( + bigints.get(&SqlString::from_ref("big")), + Some(&Some(5000000000i64)) + ); + assert_eq!(bigints.get(&SqlString::from_ref("i")), Some(&Some(1))); + assert_eq!(bigints.get(&SqlString::from_ref("neg")), Some(&Some(-5))); + } + + #[test] + fn json_each_string_keeps_only_string_fields() { + use super::json_each_string_V; + + let strings = json_each_string_V(each_test_object()); + // Only fields holding strings; numbers and booleans are not stringified + assert_eq!(keys(&strings), vec!["date", "s", "snum", "time", "ts"]); + assert_eq!( + strings.get(&SqlString::from_ref("s")), + Some(&Some(SqlString::from_ref("text"))) + ); + } + + #[test] + fn json_each_datetime_parses_strings() { + use super::{ + json_each_boolean_V, json_each_date_V, json_each_time_V, json_each_timestamp_V, + }; + use chrono::{NaiveDate, NaiveTime}; + + let bools = json_each_boolean_V(each_test_object()); + assert_eq!(keys(&bools), vec!["b"]); + + // JSON has no date or time types, so strings that parse using the + // grammar of the corresponding SQL literal qualify; each grammar + // accepts only its own fields of the test object, and strings such + // as "text" or "7" parse as none of them + let dates = json_each_date_V(each_test_object()); + assert_eq!(keys(&dates), vec!["date"]); + assert_eq!( + dates.get(&SqlString::from_ref("date")), + Some(&Some(Date::from_date( + NaiveDate::from_ymd_opt(2024, 1, 1).unwrap() + ))) + ); + assert_eq!(keys(&json_each_time_V(each_test_object())), vec!["time"]); + // a date-only string is also a valid midnight timestamp + assert_eq!( + keys(&json_each_timestamp_V(each_test_object())), + vec!["date", "ts"] + ); + + // Genuinely typed values, as produced by CAST(x AS VARIANT), also + // qualify + let date = Date::from_date(NaiveDate::from_ymd_opt(2024, 1, 1).unwrap()); + let time = Time::from_time(NaiveTime::from_hms_opt(17, 30, 40).unwrap()); + let typed = Variant::Map( + [ + ( + Variant::String(SqlString::from_ref("d")), + Variant::Date(date), + ), + ( + Variant::String(SqlString::from_ref("t")), + Variant::Time(time), + ), + ] + .into_iter() + .collect::>() + .into(), + ); + let dates = json_each_date_V(typed.clone()); + assert_eq!(keys(&dates), vec!["d"]); + assert_eq!(dates.get(&SqlString::from_ref("d")), Some(&Some(date))); + let times = json_each_time_V(typed); + assert_eq!(keys(×), vec!["t"]); + } + + #[test] + fn json_each_non_map_returns_empty() { + use super::{json_each_bigint_V, json_each_bigint_VN, json_each_string_V}; + + assert!(json_each_bigint_V(Variant::BigInt(5)).is_empty()); + assert!(json_each_string_V(serde_json::from_str::("[1, 2]").unwrap()).is_empty()); + assert!(json_each_bigint_V(Variant::VariantNull).is_empty()); + assert!(json_each_bigint_V(Variant::SqlNull).is_empty()); + // SQL NULL argument propagates to a NULL result + assert_eq!(json_each_bigint_VN(None), None); + assert!( + json_each_bigint_VN(Some(Variant::Boolean(true))) + .unwrap() + .is_empty() + ); + + // Non-string keys are skipped + let int_keys = Variant::Map( + [(Variant::BigInt(1), Variant::BigInt(2))] + .into_iter() + .collect::>() + .into(), + ); + assert!(json_each_bigint_V(int_keys).is_empty()); + } + + fn strs(array: &crate::Array) -> Vec<&str> { + array.iter().map(|s| s.str()).collect() + } + + #[test] + fn json_object_keys_returns_top_level_keys() { + use super::{json_object_keys_V, json_object_keys_VN}; + + // All top-level keys sorted, including those holding nulls, + // arrays and nested objects + let keys = json_object_keys_V(each_test_object()); + assert_eq!( + strs(&keys), + vec![ + "arr", "b", "big", "date", "dec", "huge", "i", "n", "neg", "obj", "s", "snum", + "time", "ts" + ] + ); + + assert!(json_object_keys_V(Variant::BigInt(5)).is_empty()); + assert!(json_object_keys_V(serde_json::from_str::("[1, 2]").unwrap()).is_empty()); + assert_eq!(json_object_keys_VN(None), None); + } + + #[test] + fn variant_filter_by_predicate() { + use super::variant_filter_; + + // Keep only string-valued fields + let filtered = variant_filter_(each_test_object(), |_k, v| matches!(v, Variant::String(_))); + let Some(Variant::Map(map)) = &filtered else { + panic!("expected a map") + }; + let keys: Vec<&str> = map + .keys() + .map(|k| match k { + Variant::String(s) => s.str(), + _ => panic!("expected string keys"), + }) + .collect(); + assert_eq!(keys, vec!["date", "s", "snum", "time", "ts"]); + + // Strip fields holding JSON nulls; "n" disappears + let stripped = variant_filter_(each_test_object(), |_k, v| { + !matches!(v, Variant::VariantNull) + }); + let Some(Variant::Map(map)) = &stripped else { + panic!("expected a map") + }; + assert!(!map.contains_key(&Variant::String(SqlString::from_ref("n"))) && map.len() == 13); + + // A non-map variant is a single item with a None label + let kept = variant_filter_(Variant::BigInt(5), |k, _v| k.is_none()); + assert_eq!(kept, Some(Variant::BigInt(5))); + let dropped = variant_filter_(Variant::BigInt(5), |_k, v| matches!(v, Variant::String(_))); + assert_eq!(dropped, None); + + // Arrays are kept whole or dropped, not filtered element-wise + let arr = serde_json::from_str::("[1, 2]").unwrap(); + assert_eq!( + variant_filter_(arr.clone(), |k, _v| k.is_none()), + Some(arr.clone()) + ); + assert_eq!(variant_filter_(arr, |k, _v| k.is_some()), None); + + // A NULL predicate result drops, like a SQL WHERE clause + let dropped = variant_filter_(Variant::BigInt(5), |_k, _v| None::); + assert_eq!(dropped, None); + } + + #[test] + fn variant_deep_filter_recurses_with_paths() { + use super::variant_deep_filter_; + + fn nested() -> Variant { + serde_json::from_str::( + r#"{"a": {"b": 1, "c": {"d": 2}}, "e": [{"f": 3}, 4], "g": 5}"#, + ) + .unwrap() + } + fn path_str(k: &Option) -> String { + match k { + Some(s) => s.str().to_string(), + None => String::new(), + } + } + fn to_json(v: &Option) -> String { + v.as_ref().unwrap().to_json_string().unwrap() + } + + // Dropping an inner path removes only that subtree + let result = variant_deep_filter_(nested(), |k, _v| path_str(k) != "a.c"); + assert_eq!(to_json(&result), r#"{"a":{"b":1},"e":[{"f":3},4],"g":5}"#); + + // Array elements have 1-based bracket path components and recurse + let result = variant_deep_filter_(nested(), |k, _v| path_str(k) != "e[1].f"); + assert_eq!( + to_json(&result), + r#"{"a":{"b":1,"c":{"d":2}},"e":[{},4],"g":5}"# + ); + + // Dropping an element shrinks the array + let result = variant_deep_filter_(nested(), |k, _v| path_str(k) != "e[1]"); + assert_eq!( + to_json(&result), + r#"{"a":{"b":1,"c":{"d":2}},"e":[4],"g":5}"# + ); + + // A top-level array is filtered element-wise + let array = serde_json::from_str::(r#"[1, {"x": 2}, 3]"#).unwrap(); + let result = variant_deep_filter_(array, |k, _v| path_str(k) != "[2].x"); + assert_eq!(to_json(&result), r#"[1,{},3]"#); + + // A scalar is a single item with no label + assert_eq!( + variant_deep_filter_(Variant::BigInt(5), |k, _v| k.is_none()), + Some(Variant::BigInt(5)) + ); + assert_eq!( + variant_deep_filter_(Variant::BigInt(5), |k, _v| k.is_some()), + None + ); + } + + #[test] + fn deep_path_components_are_quoted() { + use super::{variant_deep_filter_, variant_deep_map_}; + + // A key that is not an identifier is double-quoted in paths, so a + // key containing a dot cannot be confused with nesting + let v = + serde_json::from_str::(r#"{"example.com": {"a": 1}, "example": {"b": 2}}"#) + .unwrap(); + let kept = variant_deep_filter_(v, |p, _v| { + let path = p.as_ref().map(|s| s.str()).unwrap_or(""); + path == "example" || !path.starts_with("example.") + }); + assert_eq!( + kept.unwrap().to_json_string().unwrap(), + r#"{"example":{},"example.com":{"a":1}}"# + ); + + // The mapper sees the quoted path, with embedded quotes escaped + let v = serde_json::from_str::(r#"{"a\"b": 1, "_ok9": 2}"#).unwrap(); + let mapped = variant_deep_map_(v, |p, _v| p.as_ref().map(|s| Variant::String(s.clone()))); + assert_eq!( + mapped.unwrap().to_json_string().unwrap(), + r#"{"_ok9":"_ok9","a\"b":"\"a\\\"b\""}"# + ); + } + + #[test] + fn variant_merge_merges_recursively() { + use super::{variant_merge_V_V, variant_merge_VN_V}; + + fn parse(s: &str) -> Variant { + serde_json::from_str::(s).unwrap() + } + fn merged_json(left: &str, right: &str) -> String { + variant_merge_V_V(parse(left), parse(right)) + .to_json_string() + .unwrap() + } + + // Objects merge recursively; fields of the second win on common keys + assert_eq!( + merged_json( + r#"{"a": {"x": 1, "y": 2}, "b": 1}"#, + r#"{"a": {"x": 9, "z": 3}, "c": 4}"# + ), + r#"{"a":{"x":9,"y":2,"z":3},"b":1,"c":4}"# + ); + + // Arrays are replaced, not concatenated + assert_eq!( + merged_json(r#"{"a": [1, 2]}"#, r#"{"a": [3]}"#), + r#"{"a":[3]}"# + ); + + // A JSON null is an ordinary value; it does not delete the field + assert_eq!( + merged_json(r#"{"a": 1, "b": 2}"#, r#"{"a": null}"#), + r#"{"a":null,"b":2}"# + ); + + // When either argument is not an object, the second wins + assert_eq!(merged_json("5", "6"), "6"); + assert_eq!(merged_json(r#"{"a": 1}"#, "[1]"), "[1]"); + assert_eq!(merged_json("[1]", r#"{"a": 1}"#), r#"{"a":1}"#); + + // A SQL NULL argument propagates + assert_eq!(variant_merge_VN_V(None, parse(r#"{"a": 1}"#)), None); + } + + #[test] + fn variant_map_builds_isomorphic_object() { + use super::variant_map_; + + // Replace every value by its key + let mapped = variant_map_(each_test_object(), |k, _v| k.clone()); + let Some(Variant::Map(map)) = &mapped else { + panic!("expected a map") + }; + assert_eq!(map.len(), 14); + assert_eq!( + map.get(&Variant::String(SqlString::from_ref("i"))), + Some(&Variant::String(SqlString::from_ref("i"))) + ); + + // A SQL NULL mapper result becomes a SqlNull variant inside the map + let nulled = variant_map_(each_test_object(), |_k, _v| None::); + let Some(Variant::Map(map)) = &nulled else { + panic!("expected a map") + }; + assert!(map.values().all(|v| matches!(v, Variant::SqlNull))); + + // A non-map variant is a single item with a None label + let mapped = variant_map_(Variant::BigInt(5), |k, v| { + assert!(k.is_none()); + match v { + Variant::BigInt(x) => Some(Variant::BigInt(x + 1)), + _ => None, + } + }); + assert_eq!(mapped, Some(Variant::BigInt(6))); + // A NULL mapper result for a non-map variant is a SQL NULL + assert_eq!( + variant_map_(Variant::BigInt(5), |_k, _v| None::), + None + ); + } + + #[test] + fn variant_deep_map_transforms_leaves() { + use super::variant_deep_map_; + + let nested = + serde_json::from_str::(r#"{"a": {"b": 1}, "e": [1, "x"], "n": null}"#) + .unwrap(); + + // Replace every leaf by its path: structure is preserved exactly, + // and JSON nulls are leaves too + let mapped = variant_deep_map_(nested.clone(), |p, _v| { + p.as_ref().map(|s| Variant::String(s.clone())) + }); + assert_eq!( + mapped.unwrap().to_json_string().unwrap(), + r#"{"a":{"b":"a.b"},"e":["e[1]","e[2]"],"n":"n"}"# + ); + + // A SQL NULL mapper result becomes a JSON null in place + let nulled = variant_deep_map_(nested, |_p, _v| None::); + assert_eq!( + nulled.unwrap().to_json_string().unwrap(), + r#"{"a":{"b":null},"e":[null,null],"n":null}"# + ); + + // A top-level scalar is a single leaf with no label + let mapped = variant_deep_map_(Variant::BigInt(5), |p, v| { + assert!(p.is_none()); + match v { + Variant::BigInt(x) => Some(Variant::BigInt(x + 1)), + _ => None, + } + }); + assert_eq!(mapped, Some(Variant::BigInt(6))); + } + + #[test] + fn json_keys_returns_nested_paths() { + use super::json_keys_V; + + // Every key at every level; arrays are not traversed ("e.f" is absent) + let v = serde_json::from_str::( + r#"{"a": {"b": 1, "c": {"d": 2}}, "e": [{"f": 3}], "g": 4}"#, + ) + .unwrap(); + assert_eq!( + strs(&json_keys_V(v)), + vec!["a", "a.b", "a.c", "a.c.d", "e", "g"] + ); + + // A key containing a dot is escaped using double quotes, like in + // BigQuery, so it cannot collide with a nested path + let v = serde_json::from_str::(r#"{"a.b": 1, "a": {"b": 2}}"#).unwrap(); + assert_eq!(strs(&json_keys_V(v)), vec!["\"a.b\"", "a", "a.b"]); + + assert!(json_keys_V(Variant::BigInt(5)).is_empty()); + } } diff --git a/docs.feldera.com/docs/sql/casts.md b/docs.feldera.com/docs/sql/casts.md index af3927f722c..034f24900ef 100644 --- a/docs.feldera.com/docs/sql/casts.md +++ b/docs.feldera.com/docs/sql/casts.md @@ -70,7 +70,7 @@ SELECT cast(row(1, 2) as row(a integer, b tinyint)) as r; ## Safe casts -The `SAFE_CAST` function has the same syntax as `CAST`. `SAFE_CAST` +The `SAFE_CAST` function has the same syntax as `CAST`. `SAFE_CAST` produces the same result as `CAST` for all legal inputs. The main difference is that `SAFE_CAST` never produces a runtime error, producing a `NULL` value when a conversion is illegal. diff --git a/docs.feldera.com/docs/sql/function-index.md b/docs.feldera.com/docs/sql/function-index.md index f5c295c9124..6235e075ef0 100644 --- a/docs.feldera.com/docs/sql/function-index.md +++ b/docs.feldera.com/docs/sql/function-index.md @@ -143,6 +143,14 @@ * `IS UNKNOWN`: [operators](operators.md#isnull) * `IS_INF`: [float](float.md#is_inf) * `IS_NAN`: [float](float.md#is_nan) +* `JSON_EACH_BIGINT`: [json](json.md#json_each) +* `JSON_EACH_BOOLEAN`: [json](json.md#json_each) +* `JSON_EACH_DATE`: [json](json.md#json_each) +* `JSON_EACH_STRING`: [json](json.md#json_each) +* `JSON_EACH_TIME`: [json](json.md#json_each) +* `JSON_EACH_TIMESTAMP`: [json](json.md#json_each) +* `JSON_KEYS`: [json](json.md#json_keys) +* `JSON_OBJECT_KEYS`: [json](json.md#json_object_keys) * `LAG` (aggregate): [aggregates](aggregates.md#lag) * `LATERAL`: [grammar](grammar.md#lateral) * `LEAD` (aggregate): [aggregates](aggregates.md#lead) @@ -212,7 +220,7 @@ * `ROUND`: [float](float.md#round), [float](float.md#round2), [decimal](decimal.md#round), [decimal](decimal.md#round2) * `ROW`: [types](types.md#row_constructor) * `ROW_NUMBER` (aggregate): [aggregates](aggregates.md#row_number) -* `SAFE_CAST`: [casts](casts.md#safe-casts) +* `SAFE_CAST`: [casts](casts.md#safe_cast) * `SAFE_OFFSET`: [array](array.md#safe_offset) * `SEC`: [float](float.md#sec) * `SECH`: [float](float.md#sech) @@ -258,6 +266,11 @@ * `VAR_POP` (aggregate): [aggregates](aggregates.md#var_pop) * `VAR_SAMP` (aggregate): [aggregates](aggregates.md#var_samp) * `VARIANCE` (aggregate): [aggregates](aggregates.md#variance) +* `VARIANT_DEEP_FILTER`: [json](json.md#variant_deep_filter) +* `VARIANT_DEEP_MAP`: [json](json.md#variant_deep_map) +* `VARIANT_FILTER`: [json](json.md#variant_filter) +* `VARIANT_MAP`: [json](json.md#variant_map) +* `VARIANT_MERGE`: [json](json.md#variant_merge) * `WEEK`: [datetime](datetime.md#week) * `XXHASH`: [string](string.md#xxhash), [binary](binary.md#xxhash) * `YEAR`: [datetime](datetime.md#year) diff --git a/docs.feldera.com/docs/sql/json.md b/docs.feldera.com/docs/sql/json.md index 4ef075b33ab..b5f3afcebe9 100644 --- a/docs.feldera.com/docs/sql/json.md +++ b/docs.feldera.com/docs/sql/json.md @@ -142,6 +142,14 @@ the map corresponds to a field of the user-defined structure. | `TYPEOF(variant)` | Argument must be a `VARIANT` value. Returns a string describing the runtime type of the value | | `PARSE_JSON(string)` | Parses a string that represents a JSON value, returns a `VARIANT` object, or `NULL` if parsing fails (more details [below](#parse_json)) | | `TO_JSON(variant)` | Argument must be a `VARIANT` value. Returns a string that represents the serialization of a `VARIANT` value. If the value cannot be represented as JSON, the result is `NULL` (more details [below](#to_json)) | +| `JSON_EACH_(variant)` | A family of functions; each extracts from a `VARIANT` holding a JSON object the fields whose values have a specified runtime type, as a `MAP` (more details [below](#json_each)) | +| `JSON_OBJECT_KEYS(variant)` | Returns the top-level keys of a `VARIANT` holding a JSON object, as a sorted `ARRAY` of strings (more details [below](#json_object_keys)) | +| `JSON_KEYS(variant)` | Returns the keys of all nested objects in a `VARIANT`, as a sorted `ARRAY` of dot-joined paths (more details [below](#json_keys)) | +| `VARIANT_FILTER(variant, lambda)` | Returns a `VARIANT` with the items of the input for which a predicate lambda is true (more details [below](#variant_filter)) | +| `VARIANT_DEEP_FILTER(variant, lambda)` | Like `VARIANT_FILTER`, but recursive: the predicate receives the dot-joined path of each nested item (more details [below](#variant_deep_filter)) | +| `VARIANT_MAP(variant, lambda)` | Builds a `VARIANT` isomorphic to the input, with each value replaced by the lambda's result (more details [below](#variant_map)) | +| `VARIANT_DEEP_MAP(variant, lambda)` | Like `VARIANT_MAP`, but recursive: transforms only the leaves, labeled by their dot-joined path (more details [below](#variant_deep_map)) | +| `VARIANT_MERGE(variant, variant)` | Merges two `VARIANT` values recursively; the second wins on conflicts (more details [below](#variant_merge)) | ### `PARSE_JSON` @@ -182,6 +190,322 @@ SELECT CAST( - a `VARIANT` wrapping a `MAP` whose keys have any SQL `CHAR` type, or `VARIANT` values wrapping `CHAR` values will generate a JSON object, by recursively converting each key-value pair. - a `VARIANT` wrapping a `DATE`, `TIME`, or `DATETIME` value will be serialized as a JSON string +### `JSON_EACH` + +The `JSON_EACH_` functions generalize the Postgres `json_each_text` +function. Each function in the family extracts from a `VARIANT` value +holding a JSON object the fields whose values have a specified runtime +type. The result is a `MAP` from field name to field value; the `MAP` +value type is nullable, but the extracted values are never `NULL`. + +| Function | Result type | Fields extracted | +|---------------------------------|--------------------------------|------------------| +| `JSON_EACH_BIGINT(variant)` | `MAP` | Numeric values with no fractional part that fit in `BIGINT` | +| `JSON_EACH_STRING(variant)` | `MAP` | String values | +| `JSON_EACH_BOOLEAN(variant)` | `MAP` | Boolean values | +| `JSON_EACH_DATE(variant)` | `MAP` | `DATE` values, and strings that parse as dates | +| `JSON_EACH_TIME(variant)` | `MAP` | `TIME` values, and strings that parse as times | +| `JSON_EACH_TIMESTAMP(variant)` | `MAP` | `TIMESTAMP` values, and strings that parse as timestamps | + +These functions obey the following rules: + +- A field is selected based on the runtime type of its value. With + the exception of the date and time functions described below, values + are never parsed from strings, and never converted to strings: a + field holding the string `"7"` is not returned by `JSON_EACH_BIGINT`, + and a field holding the number `7` is not returned by + `JSON_EACH_STRING`. +- `JSON_EACH_BIGINT` does not truncate: a field holding `2.5` is not + returned. Such fields can be selected with + [`VARIANT_FILTER`](#variant_filter) using a `TYPEOF` predicate, + which keeps the values as `VARIANT`, avoiding a commitment to a + fixed `DECIMAL` precision and scale. +- Fields holding JSON `null` values are never returned. +- Fields whose keys are not strings are never returned. +- A `VARIANT` that does not hold an object (e.g., a scalar, an array, + or a `null`) produces an empty map. +- A SQL `NULL` argument produces a SQL `NULL` result. +- JSON has no date or time types, so `JSON_EACH_DATE`, + `JSON_EACH_TIME`, and `JSON_EACH_TIMESTAMP` also accept string + values, parsing them with the grammar of the corresponding SQL + literal (e.g., `'2024-01-01'`, `'17:30:40'`, `'2024-12-19 + 16:39:57'`); strings that do not parse are omitted. Like in a + `CAST`, a date-only string is also a valid midnight timestamp. + Values typed as `DATE`, `TIME`, or `TIMESTAMP`, which arise from + expressions such as `CAST(MAP['d', DATE '2024-01-01'] AS VARIANT)`, + qualify as well. + +These functions are commonly combined with [`UNNEST`](map.md#the-unnest-operator) +to produce a table with a row for each extracted field: + +```sql +CREATE TABLE data(json VARIANT); + +CREATE VIEW ints AS +SELECT t.k, t.v +FROM data, UNNEST(JSON_EACH_BIGINT(data.json)) AS t(k, v); +``` + +### `JSON_OBJECT_KEYS` + +`JSON_OBJECT_KEYS(variant)` returns the top-level keys of a `VARIANT` +holding a JSON object, as a sorted `ARRAY` of strings, following the +Postgres function with the same name. All keys are returned, +including keys whose values are JSON `null` values, nested objects, or +arrays; keys that are not strings are skipped. A `VARIANT` that does +not hold an object produces an empty array; a SQL `NULL` argument +produces a SQL `NULL` result. + +```sql +SELECT JSON_OBJECT_KEYS(PARSE_JSON('{"a": 1, "b": {"c": 2}, "d": null}')); +-- [a, b, d] +``` + +### `JSON_KEYS` + +`JSON_KEYS(variant)` returns the keys of all nested objects in a +`VARIANT`, as dot-joined paths, deduplicated and sorted, following the +BigQuery function with the same name in its default `'strict'` mode. +Objects nested inside arrays are not traversed. Keys that are not +strings are skipped. A `VARIANT` that does not hold an object +produces an empty array; a SQL `NULL` argument produces a SQL `NULL` +result. (The BigQuery `max_depth` and `mode` arguments are not +supported.) + +Like the BigQuery function, `JSON_KEYS` escapes keys containing +special characters using double quotes, so paths are unambiguous: the +object `{"a.b": 1}` produces the path `"a.b"`, including the quotes, +distinct from the path `a.b` produced by `{"a": {"b": 1}}`. The +[`VARIANT_DEEP_FILTER`](#variant_deep_filter) and +[`VARIANT_DEEP_MAP`](#variant_deep_map) functions use the same +quoting in their paths. + +```sql +SELECT JSON_KEYS(PARSE_JSON('{"a": {"b": 1, "c": {"d": 2}}, "e": [{"f": 3}], "g": 4}')); +-- [a, a.b, a.c, a.c.d, e, g] +``` + +### `VARIANT_FILTER` + +`VARIANT_FILTER(variant, (key, value) -> predicate)` keeps the parts +of a `VARIANT` for which a predicate is true. Consider this call: + +```sql +SELECT VARIANT_FILTER( + PARSE_JSON('{"name": "Ada", + "age": 36, + "address": {"city": "Boston", "zip": "02115"}, + "tags": [1, 2], + "note": null}'), + (k, x) -> TYPEOF(x) = 'VARCHAR'); +``` + +The predicate is called once per top-level field of the object; both +arguments are `VARIANT` values, and a field holding a nested object or +array is passed whole, as a single value: + +| `k` | `x` | `TYPEOF(x) = 'VARCHAR'` | +|-------------|------------------------------------|-------------------------| +| `'address'` | `{"city": "Boston", "zip": "02115"}` | `FALSE` | +| `'age'` | `36` | `FALSE` | +| `'name'` | `'Ada'` | `TRUE` | +| `'note'` | JSON `null` | `FALSE` | +| `'tags'` | `[1, 2]` | `FALSE` | + +The result is the object `{"name": "Ada"}`. A field is kept only when +the predicate evaluates to `TRUE`. Note that the predicate is never +called on the nested fields `city` and `zip`: the whole `address` +object is one item, kept or dropped as a unit. Use `VARIANT_DEEP_FILTER` +if deep inspection is required. + +When the variant does not hold an object, the predicate is called once, +with a `NULL` key and the value; the result is the value unchanged when +the predicate is true, and SQL `NULL` otherwise: + +```sql +SELECT VARIANT_FILTER(PARSE_JSON('5'), (k, x) -> k IS NULL); +-- 5 +SELECT VARIANT_FILTER(PARSE_JSON('5'), (k, x) -> k IS NOT NULL); +-- NULL + +-- remove fields holding JSON nulls +SELECT VARIANT_FILTER(v, (k, x) -> x <> VARIANTNULL()); + +-- keep fields whose key starts with 'user' +SELECT VARIANT_FILTER(v, (k, x) -> CAST(k AS VARCHAR) LIKE 'user%'); + +-- keep strings that parse as dates +SELECT t.k, t.d FROM data, UNNEST( + CAST(VARIANT_FILTER(data.json, + (k, v) -> TYPEOF(v) = 'VARCHAR' + AND CAST(v AS DATE) IS NOT NULL) + AS MAP)) AS t(k, d); +``` + +### `VARIANT_DEEP_FILTER` + +`VARIANT_DEEP_FILTER(variant, (path, value) -> predicate)` is the +recursive version of `VARIANT_FILTER`. The predicate receives the +dot-joined path of each item instead of its key; the path is a +nullable `VARCHAR`. + +- fields of objects are labeled by their path, e.g. `a.b.c`; +- array elements are items too, labeled with 1-based bracket + components, e.g. `e[1].f`; a dropped element shrinks the array; +- the predicate receives the original, unfiltered value of each item; + dropping an item removes its whole subtree, and the predicate is not + evaluated on the contents of a dropped container; +- a `VARIANT` holding a top-level array is filtered element-wise, with + paths `[1]`, `[2]`, etc.; +- a scalar or JSON `null` is a single item with a `NULL` path, kept + whole or dropped to SQL `NULL`, as in `VARIANT_FILTER`; +- fields with non-string keys are kept untouched, without invoking the + predicate + +Field names with special characters are +double-quoted in the path, and backslashes escape embedded quotes and +backslashes. E.g.: + +```json +{ "example.com": { "a": 1 }, "example": { "b": 2 } } +``` + +The field `a` has the path `"example.com".a`, including the quotes, so +the predicate `p LIKE 'example.%'` selects only the subtree of the key +`example`. + +```sql +-- remove one subtree +SELECT VARIANT_DEEP_FILTER(PARSE_JSON('{"a": {"b": 1, "c": {"d": 2}}}'), + (p, x) -> p <> 'a.c'); +-- {"a":{"b":1}} + +-- keep only paths under 'a', at any depth +SELECT VARIANT_DEEP_FILTER(v, (p, x) -> p = 'a' OR p LIKE 'a.%'); +``` + +### `VARIANT_MAP` + +`VARIANT_MAP(variant, (key, value) -> expression)` shallowly transforms the +values of a `VARIANT` with a lambda, building a result isomorphic to +the input: + +- a `VARIANT` holding an object produces an object with the same keys, + where each value is replaced by the lambda's result; +- any other `VARIANT` is a single item with a `NULL` key; the lambda's + result is the result of the function; +- the lambda may produce a value of any type; it is converted to a + `VARIANT` automatically; +- a SQL `NULL` produced by the lambda becomes a JSON `null` inside an + object; +- the function does not recurse into nested objects; +- a SQL `NULL` argument produces a SQL `NULL` result. + +```sql +-- double every numeric value; non-numbers become JSON nulls +SELECT TO_JSON(VARIANT_MAP(PARSE_JSON('{"a": 1, "b": "x"}'), + (k, x) -> CAST(x AS BIGINT) * 2)); +-- {"a":2,"b":null} + +-- replace each value by its runtime type +SELECT TO_JSON(VARIANT_MAP(PARSE_JSON('{"a": 1, "b": "x"}'), + (k, x) -> TYPEOF(x))); +-- {"a":"BIGINT UNSIGNED","b":"VARCHAR"} +``` + +### `VARIANT_DEEP_MAP` + +`VARIANT_DEEP_MAP(variant, (path, value) -> expression)` is the +recursive version of `VARIANT_MAP`. The structure of nested objects +and arrays is preserved exactly; the lambda transforms only the +leaves, i.e. the values that are not objects or arrays: + +- the first lambda argument is the leaf's dot-joined path, a nullable + `VARCHAR`, with the same syntax as in `VARIANT_DEEP_FILTER`: array + elements use 1-based bracket components, e.g. `e[1].f`; +- JSON `null` values are leaves too, and are passed to the lambda; +- the lambda may produce a value of any type; it is converted to a + `VARIANT` automatically, and a SQL `NULL` result becomes a JSON + `null` in place; +- a top-level scalar is a single leaf with a `NULL` path; the lambda's + result is the result of the function; +- fields with non-string keys are kept untouched, without + transformation or recursion; +- a SQL `NULL` argument produces a SQL `NULL` result. + +:::note + +The two recursive functions treat containers differently, by design: +`VARIANT_DEEP_FILTER` applies its predicate to every nested item, +including objects and arrays, since filtering decides which subtrees +survive; `VARIANT_DEEP_MAP` applies its lambda only to leaves, since +mapping transforms values while preserving the structure. + +::: + +```sql +-- double every number, at any depth; other leaves become JSON nulls +SELECT TO_JSON(VARIANT_DEEP_MAP(PARSE_JSON('{"a": {"b": 1}, "e": [1, 2], "s": "x"}'), + (p, x) -> CAST(x AS BIGINT) * 2)); +-- {"a":{"b":2},"e":[2,4],"s":null} + +-- redact all values under the 'user' subtree +SELECT TO_JSON(VARIANT_DEEP_MAP(PARSE_JSON('{"user": {"name": "Ada", "ssn": "123"}, "id": 7}'), + (p, x) -> CASE WHEN p LIKE 'user.%' + THEN CAST('***' AS VARIANT) + ELSE x END)); +-- {"id":7,"user":{"name":"***","ssn":"***"}} +``` + +### `VARIANT_MERGE` + +`VARIANT_MERGE(v1, v2)` merges two `VARIANT` values recursively, +following the JSON Merge Patch algorithm (RFC 7386) with one +difference: JSON `null` values are ordinary values, and never delete +fields. + +- When both arguments hold objects, their fields are merged: fields + present on only one side are kept, and fields present on both sides + are merged recursively. +- In every other case the second argument wins: scalars, arrays, and + mixed combinations are replaced, not combined. In particular, two + arrays are not concatenated. +- A JSON `null` on the right replaces the value; it does not remove + the field. Removing fields is + [`VARIANT_FILTER`](#variant_filter)'s job. +- A SQL `NULL` argument produces a SQL `NULL` result. + +```sql +-- fields of the second argument win on common keys, recursively: +-- "x" is overridden, "y" is kept, "z" and "c" are added +SELECT TO_JSON(VARIANT_MERGE( + PARSE_JSON('{"a": {"x": 1, "y": 2}, "b": 1}'), + PARSE_JSON('{"a": {"x": 9, "z": 3}, "c": 4}'))); +-- {"a":{"x":9,"y":2,"z":3},"b":1,"c":4} + +-- arrays are replaced, not concatenated +SELECT TO_JSON(VARIANT_MERGE( + PARSE_JSON('{"tags": [1, 2]}'), + PARSE_JSON('{"tags": [3]}'))); +-- {"tags":[3]} + +-- a JSON null is an ordinary value; it does not remove the field +SELECT TO_JSON(VARIANT_MERGE( + PARSE_JSON('{"a": 1, "b": 2}'), + PARSE_JSON('{"a": null}'))); +-- {"a":null,"b":2} + +-- inserting a field is a merge with a one-field object +SELECT TO_JSON(VARIANT_MERGE( + PARSE_JSON('{"a": 1}'), + PARSE_JSON('{"new": 5}'))); +-- {"a":1,"new":5} +``` + +When the inserted value is computed rather than constant, build the +one-field object with the `MAP` constructor instead of `PARSE_JSON`: +`VARIANT_MERGE(v, CAST(MAP['new', CAST(x AS VARIANT)] AS VARIANT))`. + ## Processing JSON data using `VARIANT` The `VARIANT` type enables efficient JSON processing in SQL. In this sense it is similar to diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java index 39b08a50299..d96d3e099bc 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/ExpressionCompiler.java @@ -1596,12 +1596,26 @@ else if (arg0Type.is(DBSPTypeMap.class)) case "typeof": return compileFunction(call, node, type, ops, 1); case "parse_json": - case "to_json": { + case "to_json": + case "json_each_bigint": + case "json_each_string": + case "json_each_boolean": + case "json_each_date": + case "json_each_time": + case "json_each_timestamp": + case "json_keys": + case "json_object_keys": { DBSPExpression expr = this.strictnessCheck(ops, type); if (expr != null) return expr; return compilePolymorphicFunction(false, call, node, type, ops, 1); } + case "variant_merge": { + DBSPExpression expr = this.strictnessCheck(ops, type); + if (expr != null) + return expr; + return compilePolymorphicFunction(false, call, node, type, ops, 2); + } case "sequence": { for (int i = 0; i < ops.size(); i++) this.ensureInteger(node, ops, i); @@ -1721,6 +1735,31 @@ else if (arg0Type.is(DBSPTypeMap.class)) node, method, type.withMayBeNull(nullable), ops.get(0), ops.get(1)) .cast(node, type, DBSPCastExpression.CastType.SqlUnsafe); } + case "variant_filter": + case "variant_deep_filter": { + validateArgCount(node, operationName, ops.size(), 2); + // The result is nullable regardless of the argument: + // a dropped non-map variant produces NULL + String method = opName + + ops.get(0).getType().nullableUnderlineSuffix(); + return new DBSPApplyExpression(node, method, type, ops.get(0), ops.get(1)); + } + case "variant_map": + case "variant_deep_map": { + validateArgCount(node, operationName, ops.size(), 2); + // The lambda may produce a value of any type; + // convert its result to a VARIANT + DBSPClosureExpression mapper = ops.get(1).to(DBSPClosureExpression.class); + DBSPType variant = DBSPTypeVariant.INSTANCE_NULLABLE; + DBSPExpression converted = mapper; + if (!mapper.getResultType().sameType(variant)) + converted = mapper.body + .cast(node, variant, DBSPCastExpression.CastType.SqlUnsafe) + .closure(mapper.parameters); + String method = opName + + ops.get(0).getType().nullableUnderlineSuffix(); + return new DBSPApplyExpression(node, method, type, ops.get(0), converted); + } case "convert_timezone": { validateArgCount(node, operationName, ops.size(), 3); ops.set(0, this.makeTimezone(ops.get(0))); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteFunctions.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteFunctions.java index eac9496f62a..55bb692642d 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteFunctions.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CalciteFunctions.java @@ -711,8 +711,15 @@ public SqlOperatorTable getFunctions() { if (func.library != SqlLibrary.STANDARD) operators.add(func.function); } + // Remove some standard functions that we don't want. + List standard = new ArrayList<>(); + for (SqlOperator operator : SqlLibraryOperatorTableFactory.INSTANCE + .getOperatorTable(SqlLibrary.STANDARD).getOperatorList()) { + if (!operator.getName().startsWith("JSON_")) + standard.add(operator); + } return SqlOperatorTables.chain( - SqlLibraryOperatorTableFactory.INSTANCE.getOperatorTable(SqlLibrary.STANDARD), + new SqlToRelCompiler.CaseInsensitiveOperatorTable(standard), new SqlToRelCompiler.CaseInsensitiveOperatorTable( SqlOperatorTables.spatialInstance().getOperatorList()), new SqlToRelCompiler.CaseInsensitiveOperatorTable(operators)); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CustomFunctions.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CustomFunctions.java index ad47a2c1f39..017200ca320 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CustomFunctions.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CustomFunctions.java @@ -1,6 +1,7 @@ package org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlCallBinding; @@ -77,6 +78,14 @@ public CustomFunctions() { this.functions.add(GreatestNonNullsFunction.INSTANCE); this.functions.add(GunzipFunction.INSTANCE); this.functions.add(InitcapSpacesFunction.INSTANCE); + this.functions.add(JsonEachFunction.BIGINT); + this.functions.add(JsonEachFunction.BOOLEAN); + this.functions.add(JsonEachFunction.DATE); + this.functions.add(JsonEachFunction.STRING); + this.functions.add(JsonEachFunction.TIME); + this.functions.add(JsonEachFunction.TIMESTAMP); + this.functions.add(JsonKeysFunction.KEYS); + this.functions.add(JsonKeysFunction.OBJECT_KEYS); this.functions.add(LeastNonNullsFunction.INSTANCE); this.functions.add(MakeDateFunction.INSTANCE); this.functions.add(MakeTimeFunction.INSTANCE); @@ -94,6 +103,11 @@ public CustomFunctions() { this.functions.add(SplitPartFunction.INSTANCE); this.functions.add(ToIntFunction.INSTANCE); this.functions.add(ToJsonFunction.INSTANCE); + this.functions.add(VariantFilterFunction.DEEP); + this.functions.add(VariantFilterFunction.INSTANCE); + this.functions.add(VariantMapFunction.DEEP); + this.functions.add(VariantMapFunction.INSTANCE); + this.functions.add(VariantMergeFunction.INSTANCE); this.functions.add(XxHashFunction.INSTANCE); this.functions.add(WriteLogFunction.INSTANCE); this.udf = new HashMap<>(); @@ -602,6 +616,81 @@ private ParseTimestampFunction() { } } + /** JSON_EACH_<T>(variant) returns a MAP<VARCHAR, T> holding all fields of + * a variant object whose values have the runtime type T. Fields with values + * of other types are not present in the result. Since JSON has no date or + * time types, the DATE, TIME, and TIMESTAMP functions also accept strings, + * parsed using the grammar of the corresponding SQL literal. A variant + * that does not hold an object produces an empty map. */ + static class JsonEachFunction extends NonOptimizedFunction { + static final JsonEachFunction BIGINT = new JsonEachFunction("JSON_EACH_BIGINT", SqlTypeName.BIGINT); + static final JsonEachFunction STRING = new JsonEachFunction("JSON_EACH_STRING", SqlTypeName.VARCHAR); + static final JsonEachFunction BOOLEAN = new JsonEachFunction("JSON_EACH_BOOLEAN", SqlTypeName.BOOLEAN); + static final JsonEachFunction DATE = new JsonEachFunction("JSON_EACH_DATE", SqlTypeName.DATE); + static final JsonEachFunction TIME = new JsonEachFunction("JSON_EACH_TIME", SqlTypeName.TIME); + static final JsonEachFunction TIMESTAMP = new JsonEachFunction("JSON_EACH_TIMESTAMP", SqlTypeName.TIMESTAMP); + + private JsonEachFunction(String name, SqlTypeName valueTypeName) { + super(name, + opBinding -> mapReturnType(opBinding, valueTypeName), + OperandTypes.VARIANT, + SqlFunctionCategory.USER_DEFINED_FUNCTION, + "json#json_each", FunctionDocumentation.NO_FILE); + } + + /** MAP<VARCHAR, valueTypeName> with nullable values; + * the map itself is nullable iff the argument is. */ + private static RelDataType mapReturnType(SqlOperatorBinding opBinding, SqlTypeName valueTypeName) { + RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); + RelDataType keyType = typeFactory.createSqlType(SqlTypeName.VARCHAR); + RelDataType valueType = typeFactory.createTypeWithNullability( + typeFactory.createSqlType(valueTypeName), true); + RelDataType mapType = typeFactory.createMapType(keyType, valueType); + return typeFactory.createTypeWithNullability( + mapType, opBinding.getOperandType(0).isNullable()); + } + } + + /** Functions returning the keys of a variant object as ARRAY<VARCHAR>. + * A variant that does not hold an object produces an empty array. */ + static class JsonKeysFunction extends NonOptimizedFunction { + static final JsonKeysFunction OBJECT_KEYS = + new JsonKeysFunction("JSON_OBJECT_KEYS", "json#json_object_keys"); + static final JsonKeysFunction KEYS = + new JsonKeysFunction("JSON_KEYS", "json#json_keys"); + + private JsonKeysFunction(String name, String documentation) { + super(name, + JsonKeysFunction::arrayReturnType, + OperandTypes.VARIANT, + SqlFunctionCategory.USER_DEFINED_FUNCTION, + documentation, FunctionDocumentation.NO_FILE); + } + + /** ARRAY<VARCHAR> with non-null elements; + * the array itself is nullable iff the argument is. */ + private static RelDataType arrayReturnType(SqlOperatorBinding opBinding) { + RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); + RelDataType elementType = typeFactory.createSqlType(SqlTypeName.VARCHAR); + RelDataType arrayType = typeFactory.createArrayType(elementType, -1); + return typeFactory.createTypeWithNullability( + arrayType, opBinding.getOperandType(0).isNullable()); + } + } + + static class VariantMergeFunction extends NonOptimizedFunction { + static final VariantMergeFunction INSTANCE = new VariantMergeFunction(); + + private VariantMergeFunction() { + super("VARIANT_MERGE", + ReturnTypes.VARIANT.andThen(SqlTypeTransforms.TO_NULLABLE), + OperandTypes.sequence("VARIANT_MERGE(, )", + OperandTypes.VARIANT, OperandTypes.VARIANT), + SqlFunctionCategory.USER_DEFINED_FUNCTION, + "json#variant_merge", FunctionDocumentation.NO_FILE); + } + } + /** RLIKE used as a function. RLIKE in SQL uses infix notation */ static class RlikeFunction extends NonOptimizedFunction { static final RlikeFunction INSTANCE = new RlikeFunction(); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/VariantFilterFunction.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/VariantFilterFunction.java new file mode 100644 index 00000000000..a50ebf2cf94 --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/VariantFilterFunction.java @@ -0,0 +1,84 @@ +package org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlOperandCountRange; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.type.FunctionSqlType; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.SqlOperandCountRanges; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; + +/** Calcite-level implementation of the VARIANT_FILTER and VARIANT_DEEP_FILTER functions. + * Both take (variant, (label, value) -> predicate) and return a VARIANT with + * the items of the input variant for which the predicate is true. */ +class VariantFilterFunction extends CustomFunctions.NonOptimizedFunction { + private VariantFilterFunction(String name, SqlTypeName labelTypeName, String documentation) { + super(name, + FILTER_INFERENCE, + makeChecker(name, labelTypeName), + SqlFunctionCategory.USER_DEFINED_FUNCTION, + documentation, FunctionDocumentation.NO_FILE); + } + + /** Always nullable: a dropped non-map variant produces NULL */ + static final SqlReturnTypeInference FILTER_INFERENCE = opBinding -> { + RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); + return typeFactory.createTypeWithNullability( + typeFactory.createSqlType(SqlTypeName.VARIANT), true); + }; + + /** Checks (VARIANT, (labelTypeName, VARIANT) -> BOOLEAN) operands */ + static SqlOperandTypeChecker makeChecker(String name, SqlTypeName labelTypeName) { + String signature = name + "(, >)"; + return new SqlOperandTypeChecker() { + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + if (!OperandTypes.VARIANT.checkSingleOperandType( + callBinding, callBinding.operand(0), 0, throwOnFailure)) + return false; + + RelDataTypeFactory typeFactory = callBinding.getTypeFactory(); + // The label is NULL when the variant does not hold a map + RelDataType labelType = typeFactory.createTypeWithNullability( + typeFactory.createSqlType(labelTypeName), true); + RelDataType valueType = typeFactory.createSqlType(SqlTypeName.VARIANT); + GenericLambdaTypeChecker lambdaChecker = + new GenericLambdaTypeChecker(signature, labelType, valueType); + if (!lambdaChecker.checkSingleOperandType( + callBinding, callBinding.operand(1), 1, throwOnFailure)) + return false; + + RelDataType functionType = SqlTypeUtil.deriveType(callBinding, callBinding.operand(1)); + if (!(functionType instanceof FunctionSqlType fType) + || fType.getReturnType().getSqlTypeName() != SqlTypeName.BOOLEAN) { + if (throwOnFailure) + throw callBinding.newValidationSignatureError(); + return false; + } + return true; + } + + @Override + public SqlOperandCountRange getOperandCountRange() { + return SqlOperandCountRanges.of(2); + } + + @Override + public String getAllowedSignatures(SqlOperator op, String opName) { + return signature; + } + }; + } + + static final VariantFilterFunction INSTANCE = new VariantFilterFunction( + "VARIANT_FILTER", SqlTypeName.VARIANT, "json#variant_filter"); + static final VariantFilterFunction DEEP = new VariantFilterFunction( + "VARIANT_DEEP_FILTER", SqlTypeName.VARCHAR, "json#variant_deep_filter"); +} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/VariantMapFunction.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/VariantMapFunction.java new file mode 100644 index 00000000000..3c585b2744a --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/VariantMapFunction.java @@ -0,0 +1,73 @@ +package org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlOperandCountRange; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.SqlOperandCountRanges; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; + +/** Calcite-level implementation of the VARIANT_MAP and VARIANT_DEEP_MAP functions. + * Both take (variant, (label, value) -> expression) and build a result + * isomorphic to the input, with values replaced by the lambda's result. */ +class VariantMapFunction extends CustomFunctions.NonOptimizedFunction { + private VariantMapFunction(String name, SqlTypeName labelTypeName, String documentation) { + super(name, + MAP_INFERENCE, + makeChecker(name, labelTypeName), + SqlFunctionCategory.USER_DEFINED_FUNCTION, + documentation, FunctionDocumentation.NO_FILE); + } + + /** Always nullable: mapping a non-map variant can produce NULL */ + static final SqlReturnTypeInference MAP_INFERENCE = opBinding -> { + RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); + return typeFactory.createTypeWithNullability( + typeFactory.createSqlType(SqlTypeName.VARIANT), true); + }; + + /** Checks (VARIANT, (labelTypeName, VARIANT) -> any type) operands */ + static SqlOperandTypeChecker makeChecker(String name, SqlTypeName labelTypeName) { + String signature = name + "(, >)"; + return new SqlOperandTypeChecker() { + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + if (!OperandTypes.VARIANT.checkSingleOperandType( + callBinding, callBinding.operand(0), 0, throwOnFailure)) + return false; + + RelDataTypeFactory typeFactory = callBinding.getTypeFactory(); + // The label is NULL when the variant does not hold a map + RelDataType labelType = typeFactory.createTypeWithNullability( + typeFactory.createSqlType(labelTypeName), true); + RelDataType valueType = typeFactory.createSqlType(SqlTypeName.VARIANT); + GenericLambdaTypeChecker lambdaChecker = + new GenericLambdaTypeChecker(signature, labelType, valueType); + // The lambda may return any type; the result is converted to VARIANT + return lambdaChecker.checkSingleOperandType( + callBinding, callBinding.operand(1), 1, throwOnFailure); + } + + @Override + public SqlOperandCountRange getOperandCountRange() { + return SqlOperandCountRanges.of(2); + } + + @Override + public String getAllowedSignatures(SqlOperator op, String opName) { + return signature; + } + }; + } + + static final VariantMapFunction INSTANCE = new VariantMapFunction( + "VARIANT_MAP", SqlTypeName.VARIANT, "json#variant_map"); + static final VariantMapFunction DEEP = new VariantMapFunction( + "VARIANT_DEEP_MAP", SqlTypeName.VARCHAR, "json#variant_deep_map"); +} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/VariantTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/VariantTests.java index de27aa1d0f2..41950046bc6 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/VariantTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/VariantTests.java @@ -82,6 +82,461 @@ public void testQuery(String query, DBSPExpression... fields) { ccs.addChange(change); } + /** A heterogeneous JSON object exercising every JSON_EACH_* function. */ + static final String JSON_EACH_OBJECT = "PARSE_JSON('{\"b\": true, \"big\": 5000000000, \"dec\": 2.5, " + + "\"i\": 1, \"n\": null, \"neg\": -5, \"s\": \"text\", \"snum\": \"7\", \"arr\": [1, 2]}')"; + + @Test + public void testJsonEachBigint() { + // "dec" is fractional and "snum" is a string: neither is a BIGINT field + this.qst(""" + SELECT * FROM UNNEST(JSON_EACH_BIGINT(%s)) AS kv(k, v); + k | v + ---------- + big | 5000000000 + i | 1 + neg | -5 + (3 rows)""".formatted(JSON_EACH_OBJECT)); + } + + @Test + public void testJsonEachStringBoolean() { + // Non-string scalars are not stringified + this.qst(""" + SELECT * FROM UNNEST(JSON_EACH_STRING(%s)) AS kv(k, v); + k | v + ----------- + s | text + snum | 7 + (2 rows) + + SELECT * FROM UNNEST(JSON_EACH_BOOLEAN(%s)) AS kv(k, v); + k | v + -------- + b | true + (1 row)""".formatted(JSON_EACH_OBJECT, JSON_EACH_OBJECT)); + } + + @Test + public void testJsonEachDateTime() { + // JSON has no date/time types, so strings are parsed using the + // grammar of the corresponding SQL literal; strings that do not + // parse are omitted. + this.qst(""" + SELECT * FROM UNNEST(JSON_EACH_DATE(PARSE_JSON('{"d": "2024-01-01", "s": "text", "t": "17:30:40"}'))) AS kv(k, v); + k | v + -------- + d | 2024-01-01 + (1 row) + + SELECT * FROM UNNEST(JSON_EACH_TIME(PARSE_JSON('{"d": "2024-01-01", "s": "text", "t": "17:30:40"}'))) AS kv(k, v); + k | v + -------- + t | 17:30:40 + (1 row) + + SELECT * FROM UNNEST(JSON_EACH_TIMESTAMP(PARSE_JSON('{"d": "2024-01-01", "ts": "2024-12-19 16:39:57"}'))) AS kv(k, v); + k | v + --------- + d | 2024-01-01 00:00:00 + ts | 2024-12-19 16:39:57 + (2 rows) + + SELECT * FROM UNNEST(JSON_EACH_DATE(CAST(MAP['d', CAST(DATE '2024-01-01' AS VARIANT), 's', CAST('x' AS VARIANT)] AS VARIANT))) AS kv(k, v); + k | v + -------- + d | 2024-01-01 + (1 row) + + SELECT * FROM UNNEST(JSON_EACH_TIME(CAST(MAP['d', CAST(DATE '2024-01-01' AS VARIANT), 's', CAST('x' AS VARIANT)] AS VARIANT))) AS kv(k, v); + k | v + -------- + (0 rows)"""); + } + + @Test + public void testJsonEachNonObject() { + // Negative tests for JSON_EACH_BIGINT + this.qst(""" + SELECT * FROM UNNEST(JSON_EACH_BIGINT(PARSE_JSON('[1, 2]'))) AS kv(k, v); + k | v + -------- + (0 rows) + + SELECT * FROM UNNEST(JSON_EACH_BIGINT(PARSE_JSON('5'))) AS kv(k, v); + k | v + -------- + (0 rows) + + SELECT * FROM UNNEST(JSON_EACH_BIGINT(PARSE_JSON('null'))) AS kv(k, v); + k | v + -------- + (0 rows) + + SELECT * FROM UNNEST(JSON_EACH_BIGINT(CAST(NULL AS VARIANT))) AS kv(k, v); + k | v + -------- + (0 rows)"""); + } + + @Test + public void testJsonFilter() { + // Filter by runtime type, by value, and by key; TO_JSON serializes + // the VARIANT results, which the test harness cannot compare directly + this.qst(""" + SELECT TO_JSON(VARIANT_FILTER(%s, (k, v) -> TYPEOF(v) = 'VARCHAR')); + r + --- + {"s":"text","snum":"7"} + (1 row) + + SELECT TO_JSON(VARIANT_FILTER(PARSE_JSON('{"name": "Ada", "age": 36, "address": {"city": "Boston", "zip": "02115"}, "tags": [1, 2], "note": null}'), (k, x) -> TYPEOF(x) = 'VARCHAR')); + r + --- + {"name":"Ada"} + (1 row) + + SELECT TO_JSON(VARIANT_FILTER(%s, (k, v) -> v <> VARIANTNULL())); + r + --- + {"arr":[1,2],"b":true,"big":5000000000,"dec":2.5,"i":1,"neg":-5,"s":"text","snum":"7"} + (1 row) + + SELECT TO_JSON(VARIANT_FILTER(%s, (k, v) -> CAST(k AS VARCHAR) LIKE 's%%')); + r + --- + {"s":"text","snum":"7"} + (1 row)""".formatted(JSON_EACH_OBJECT, JSON_EACH_OBJECT, JSON_EACH_OBJECT)); + } + + @Test + public void testJsonFilterNonObject() { + // A non-map variant is a single item with a NULL label: + // kept whole or dropped to SQL NULL + this.qst(""" + SELECT TO_JSON(VARIANT_FILTER(PARSE_JSON('5'), (k, v) -> k IS NULL)); + r + --- + 5 + (1 row) + + SELECT TO_JSON(VARIANT_FILTER(PARSE_JSON('5'), (k, v) -> k IS NOT NULL)); + r + --- + NULL + (1 row) + + SELECT TO_JSON(VARIANT_FILTER(PARSE_JSON('[1, 2]'), (k, v) -> k IS NULL)); + r + --- + [1,2] + (1 row) + + SELECT TO_JSON(VARIANT_FILTER(CAST(NULL AS VARIANT), (k, v) -> TRUE)); + r + --- + NULL + (1 row)"""); + } + + /** A nested JSON object exercising VARIANT_DEEP_FILTER and JSON_KEYS. */ + static final String NESTED_OBJECT = + "PARSE_JSON('{\"a\": {\"b\": 1, \"c\": {\"d\": 2}}, \"e\": [{\"f\": 3}, 4], \"g\": 5}')"; + + @Test + public void testDeepFilterQuotedPaths() { + this.qst(""" + SELECT TO_JSON(VARIANT_DEEP_FILTER(PARSE_JSON('{"example.com": {"a": 1}, "example": {"b": 2}}'), (p, v) -> p = 'example' OR p NOT LIKE 'example.%')); + r + --- + {"example":{},"example.com":{"a":1}} + (1 row)"""); + } + + @Test + public void testJsonDeepFilter() { + // Paths are dot-joined; array elements use 1-based bracket components; + // dropping an inner path removes only that subtree + this.qst(""" + SELECT TO_JSON(VARIANT_DEEP_FILTER(PARSE_JSON('{"a": {"b": 1, "c": {"d": 2}}}'), (p, x) -> p <> 'a.c')); + r + --- + {"a":{"b":1}} + (1 row) + + SELECT TO_JSON(VARIANT_DEEP_FILTER(%s, (p, v) -> p <> 'a.c')); + r + --- + {"a":{"b":1},"e":[{"f":3},4],"g":5} + (1 row) + + SELECT TO_JSON(VARIANT_DEEP_FILTER(%s, (p, v) -> p <> 'e[1].f')); + r + --- + {"a":{"b":1,"c":{"d":2}},"e":[{},4],"g":5} + (1 row) + + SELECT TO_JSON(VARIANT_DEEP_FILTER(%s, (p, v) -> p <> 'e[1]')); + r + --- + {"a":{"b":1,"c":{"d":2}},"e":[4],"g":5} + (1 row) + + SELECT TO_JSON(VARIANT_DEEP_FILTER(PARSE_JSON('[1, {"x": 2}, 3]'), (p, v) -> p <> '[2].x')); + r + --- + [1,{},3] + (1 row) + + SELECT TO_JSON(VARIANT_DEEP_FILTER(PARSE_JSON('5'), (p, v) -> p IS NULL)); + r + --- + 5 + (1 row) + + SELECT TO_JSON(VARIANT_DEEP_FILTER(CAST(NULL AS VARIANT), (p, v) -> TRUE)); + r + --- + NULL + (1 row)""".formatted(NESTED_OBJECT, NESTED_OBJECT, NESTED_OBJECT)); + } + + @Test + public void testVariantMap() { + this.qst(""" + SELECT TO_JSON(VARIANT_MAP(PARSE_JSON('{"a": 1, "b": 2}'), (k, v) -> CAST(v AS BIGINT) * 2)); + r + --- + {"a":2,"b":4} + (1 row) + + SELECT TO_JSON(VARIANT_MAP(PARSE_JSON('{"a": 1, "b": 2}'), (k, v) -> k)); + r + --- + {"a":"a","b":"b"} + (1 row) + + SELECT TO_JSON(VARIANT_MAP(PARSE_JSON('{"a": 1, "b": "x"}'), (k, v) -> CAST(v AS BIGINT) * 2)); + r + --- + {"a":2,"b":null} + (1 row) + + SELECT TO_JSON(VARIANT_MAP(PARSE_JSON('{"a": 1, "b": "x"}'), (k, v) -> TYPEOF(v))); + r + --- + {"a":"BIGINT UNSIGNED","b":"VARCHAR"} + (1 row) + + SELECT TO_JSON(VARIANT_MAP(PARSE_JSON('{"a": 1, "b": "x"}'), (k, v) -> CAST(v AS BIGINT))); + r + --- + {"a":1,"b":null} + (1 row) + + SELECT TO_JSON(VARIANT_MAP(PARSE_JSON('5'), (k, v) -> CAST(v AS BIGINT) + 1)); + r + --- + 6 + (1 row) + + SELECT TO_JSON(VARIANT_MAP(CAST(NULL AS VARIANT), (k, v) -> v)); + r + --- + NULL + (1 row)"""); + } + + @Test + public void testVariantDeepMap() { + // Structure is preserved exactly; the lambda transforms only leaves, + // labeled by their dot-joined path; JSON nulls are leaves too + this.qst(""" + SELECT TO_JSON(VARIANT_DEEP_MAP(PARSE_JSON('{"a": {"b": 1}, "e": [1, 2]}'), (p, v) -> p)); + r + --- + {"a":{"b":"a.b"},"e":["e[1]","e[2]"]} + (1 row) + + SELECT TO_JSON(VARIANT_DEEP_MAP(PARSE_JSON('{"a": {"b": 1}, "e": [1, 2], "s": "x"}'), (p, v) -> CAST(v AS BIGINT) * 2)); + r + --- + {"a":{"b":2},"e":[2,4],"s":null} + (1 row) + + SELECT TO_JSON(VARIANT_DEEP_MAP(PARSE_JSON('{"user": {"name": "Ada", "ssn": "123"}, "id": 7}'), (p, x) -> CASE WHEN p LIKE 'user.%' THEN CAST('***' AS VARIANT) ELSE x END)); + r + --- + {"id":7,"user":{"name":"***","ssn":"***"}} + (1 row) + + SELECT TO_JSON(VARIANT_DEEP_MAP(PARSE_JSON('5'), (p, v) -> CAST(v AS BIGINT) + 1)); + r + --- + 6 + (1 row) + + SELECT TO_JSON(VARIANT_DEEP_MAP(CAST(NULL AS VARIANT), (p, v) -> v)); + r + --- + NULL + (1 row)"""); + } + + @Test + public void testVariantMerge() { + this.qst(""" + SELECT TO_JSON(VARIANT_MERGE(PARSE_JSON('{"a": {"x": 1, "y": 2}, "b": 1}'), PARSE_JSON('{"a": {"x": 9, "z": 3}, "c": 4}'))); + r + --- + {"a":{"x":9,"y":2,"z":3},"b":1,"c":4} + (1 row) + + SELECT TO_JSON(VARIANT_MERGE(PARSE_JSON('{"a": 1}'), CAST(MAP['new', CAST(5 AS VARIANT)] AS VARIANT))); + r + --- + {"a":1,"new":5} + (1 row) + + SELECT TO_JSON(VARIANT_MERGE(PARSE_JSON('{"a": [1, 2]}'), PARSE_JSON('{"a": [3]}'))); + r + --- + {"a":[3]} + (1 row) + + + SELECT TO_JSON(VARIANT_MERGE(PARSE_JSON('{"a": 1}'), PARSE_JSON('{"new": 5}'))); + r + --- + {"a":1,"new":5} + (1 row) + + SELECT TO_JSON(VARIANT_MERGE(PARSE_JSON('{"a": 1, "b": 2}'), PARSE_JSON('{"a": null}'))); + r + --- + {"a":null,"b":2} + (1 row) + + SELECT TO_JSON(VARIANT_MERGE(PARSE_JSON('5'), PARSE_JSON('6'))); + r + --- + 6 + (1 row) + + SELECT TO_JSON(VARIANT_MERGE(CAST(NULL AS VARIANT), PARSE_JSON('{"a": 1}'))); + r + --- + NULL + (1 row)"""); + } + + @Test + public void testLambdaTypeChecking() { + // A predicate with the wrong number of parameters is rejected + this.queryFailingInCompilation( + "SELECT VARIANT_FILTER(PARSE_JSON('{}'), (k) -> TRUE)", + "VARIANT_FILTER(, >)"); + this.queryFailingInCompilation( + "SELECT VARIANT_FILTER(PARSE_JSON('{}'), (k, v, x) -> TRUE)", + "VARIANT_FILTER(, >)"); + // A predicate that does not return BOOLEAN is rejected + this.queryFailingInCompilation( + "SELECT VARIANT_FILTER(PARSE_JSON('{}'), (k, v) -> 5)", + "VARIANT_FILTER(, >)"); + // The first argument must be a VARIANT + this.queryFailingInCompilation( + "SELECT VARIANT_FILTER(5, (k, v) -> TRUE)", + "VARIANT_FILTER(, >)"); + // A non-lambda second argument is rejected + this.queryFailingInCompilation( + "SELECT VARIANT_FILTER(PARSE_JSON('{}'), 5)", + "VARIANT_FILTER(, >)"); + // In VARIANT_DEEP_FILTER the path is a VARCHAR, not a VARIANT: + // TYPEOF requires a VARIANT argument + this.queryFailingInCompilation( + "SELECT VARIANT_DEEP_FILTER(PARSE_JSON('{}'), (p, v) -> TYPEOF(p) = 'VARCHAR')", + "TYPEOF"); + // The same rules apply to the map functions; arity checked here + this.queryFailingInCompilation( + "SELECT VARIANT_MAP(PARSE_JSON('{}'), (k) -> k)", + "VARIANT_MAP(, >)"); + } + + @Test + public void testJsonObjectKeys() { + // Top-level keys, sorted, including those holding nulls, arrays + // and nested objects; non-objects and NULL produce no rows + this.qst(""" + SELECT * FROM UNNEST(JSON_OBJECT_KEYS(%s)) AS t(k); + k + ------ + arr + b + big + dec + i + n + neg + s + snum + (9 rows) + + SELECT * FROM UNNEST(JSON_OBJECT_KEYS(PARSE_JSON('{"a": 1, "b": {"c": 2}, "d": null}'))) AS t(k); + k + ------ + a + b + d + (3 rows) + + SELECT * FROM UNNEST(JSON_OBJECT_KEYS(PARSE_JSON('[1, 2]'))) AS t(k); + k + ------ + (0 rows) + + SELECT * FROM UNNEST(JSON_OBJECT_KEYS(CAST(NULL AS VARIANT))) AS t(k); + k + ------ + (0 rows)""".formatted(JSON_EACH_OBJECT)); + } + + @Test + public void testJsonKeys() { + // Every key at every level as a dot-joined path; arrays are not + // traversed, so "e.f" is absent. Keys containing special characters + // are escaped using double quotes, like in BigQuery, so a key with a + // dot cannot collide with a nested path. + this.qst(""" + SELECT * FROM UNNEST(JSON_KEYS(PARSE_JSON('{"a.b": 1, "a": {"b": 2}}'))) AS t(k); + k + ------ + "a.b" + a + a.b + (3 rows) + + SELECT * FROM UNNEST(JSON_KEYS(PARSE_JSON('{"a": {"b": 1, "c": {"d": 2}}, "e": [{"f": 3}], "g": 4}'))) AS t(k); + k + ------ + a + a.b + a.c + a.c.d + e + g + (6 rows) + + SELECT * FROM UNNEST(JSON_KEYS(PARSE_JSON('5'))) AS t(k); + k + ------ + (0 rows) + + SELECT * FROM UNNEST(JSON_KEYS(CAST(NULL AS VARIANT))) AS t(k); + k + ------ + (0 rows)"""); + } + @Test public void testUDT() { this.compileRustTestCase("""