From 60f5342fb62fc58cdb427d4c29037f0bc904fff1 Mon Sep 17 00:00:00 2001 From: Leonid Ryzhyk Date: Fri, 10 Jul 2026 17:09:52 -0700 Subject: [PATCH 1/9] [adapters] Parse Debezium logical types in the Avro input connector Debezium encodes temporal and variable-scale-decimal columns in forms that differ from the standard Avro types, tagging them with a `connect.name` attribute rather than a native Avro logical type. This commit adds support for several such types: - Temporal: Time, MicroTime, NanoTime, ZonedTime, Timestamp, MicroTimestamp, NanoTimestamp, ZonedTimestamp, and the org.apache.kafka.connect.data.* aliases from time.precision.mode=connect, parsed into TIME/TIMESTAMP. Date already deserialized correctly as a plain day count and needs no conversion. - io.debezium.data.VariableScaleDecimal, a {scale, value} record used for NUMERIC/DECIMAL columns without a fixed scale, parsed into DECIMAL. The implementation could have been simpler if it didn't have to fight with the `apache-avro` crate, which drops attributes on primitive schemas, so these annotations were lost. Signed-off-by: Leonid Ryzhyk --- crates/adapters/src/format/avro.rs | 2 + crates/adapters/src/format/avro/coercion.rs | 388 ++++++++++++ crates/adapters/src/format/avro/debezium.rs | 402 ++++++++++++ .../adapters/src/format/avro/deserializer.rs | 71 ++- crates/adapters/src/format/avro/input.rs | 14 +- crates/adapters/src/format/avro/schema.rs | 19 + crates/adapters/src/format/avro/test.rs | 593 +++++++++++++++++- crates/feldera-types/src/format/avro.rs | 30 + docs.feldera.com/docs/formats/avro.md | 34 + 9 files changed, 1542 insertions(+), 11 deletions(-) create mode 100644 crates/adapters/src/format/avro/coercion.rs create mode 100644 crates/adapters/src/format/avro/debezium.rs diff --git a/crates/adapters/src/format/avro.rs b/crates/adapters/src/format/avro.rs index 178bac15250..1e7cfac0840 100644 --- a/crates/adapters/src/format/avro.rs +++ b/crates/adapters/src/format/avro.rs @@ -5,6 +5,8 @@ use feldera_adapterlib::catalog::AvroSchemaRefs; use feldera_types::format::avro::AvroSchemaRegistryConfig; use schema_registry_converter::blocking::schema_registry::SrSettings; +mod coercion; +mod debezium; pub mod deserializer; pub mod input; pub mod output; diff --git a/crates/adapters/src/format/avro/coercion.rs b/crates/adapters/src/format/avro/coercion.rs new file mode 100644 index 00000000000..38b2117171e --- /dev/null +++ b/crates/adapters/src/format/avro/coercion.rs @@ -0,0 +1,388 @@ +//! Value coercions for the Avro input connector. +//! +//! Some source systems encode a column in a form that differs from the Avro +//! type Feldera expects for the target column. Debezium, for example, encodes a +//! microsecond timestamp as a plain `long` tagged with a `connect.name` +//! attribute, and a variable-scale decimal as a `{scale, value}` record. A +//! [`Coercion`] recognizes such a field and converts its decoded value into a +//! form the target column's deserializer accepts. +//! +//! This module is the generic layer: it identifies coercions from schema +//! metadata, converts values, and validates schemas. The per-type mechanics +//! live in source-specific modules (currently [`super::debezium`]). +//! +//! # Adding a coercion +//! +//! 1. Add a variant to [`Coercion`]. +//! 2. Recognize it in [`Coercion::from_connect_name`] (for Kafka Connect +//! `connect.name` annotations) or extend [`field_coercion`] with a new +//! detector (for other sources). +//! 3. Implement its conversion in [`Coercion::coerce`] and its schema check in +//! [`Coercion::validate`]. +//! 4. If the annotation sits on a primitive Avro type, whose attributes +//! `apache-avro` drops during parsing, [`hoist_coercible_types`] recovers it +//! automatically once step 2 recognizes it. + +use std::collections::BTreeMap; + +use apache_avro::{Schema as AvroSchema, types::Value}; +use feldera_adapterlib::catalog::AvroSchemaRefs; +use feldera_types::program_schema::{ColumnType, SqlType}; +use serde_json::{Map, Value as JsonValue}; + +use super::debezium::{self, DebeziumTimeType}; +use super::schema::{schema_json, schema_unwrap_optional}; +use crate::format::avro::resolve_ref; + +/// `connect.name` of the Debezium variable-scale decimal type. +const DEBEZIUM_VARIABLE_SCALE_DECIMAL: &str = "io.debezium.data.VariableScaleDecimal"; + +/// A value coercion applied to a single Avro field. +/// +/// Identified from schema metadata and applied by the deserializer, a coercion +/// bridges the gap between a field's on-the-wire Avro type and the Feldera +/// column type it feeds. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Coercion { + /// A Debezium temporal type, converted to microseconds. + DebeziumTime(DebeziumTimeType), + /// The `io.debezium.data.VariableScaleDecimal` record, converted to a + /// decimal string. + VariableScaleDecimal, +} + +/// The Serde value a coercion produces. The deserializer dispatches it onto the +/// target column's visitor. +pub enum Coerced { + /// A 64-bit integer, visited as `i64`. + I64(i64), + /// A string, visited as `str`. + Str(String), +} + +impl Coercion { + /// Recognize a coercion from a Kafka Connect `connect.name` annotation. + pub fn from_connect_name(connect_name: &str) -> Option { + if let Some(time_type) = DebeziumTimeType::from_connect_name(connect_name) { + Some(Coercion::DebeziumTime(time_type)) + } else if connect_name == DEBEZIUM_VARIABLE_SCALE_DECIMAL { + Some(Coercion::VariableScaleDecimal) + } else { + None + } + } + + /// A representative name for this coercion, used in error messages. + fn name(self) -> &'static str { + match self { + Coercion::DebeziumTime(time_type) => time_type.canonical_connect_name(), + Coercion::VariableScaleDecimal => DEBEZIUM_VARIABLE_SCALE_DECIMAL, + } + } + + /// Convert a decoded Avro value into the form the target column expects. + pub fn coerce(self, value: &Value) -> Result { + Ok(match self { + Coercion::DebeziumTime(time_type) => Coerced::I64(time_type.to_micros(value)?), + Coercion::VariableScaleDecimal => { + Coerced::Str(debezium::variable_scale_decimal_to_string(value)?) + } + }) + } + + /// Check that this coercion can populate `column_type` from `avro_schema`. + pub fn validate( + self, + avro_schema: &AvroSchema, + refs: &AvroSchemaRefs, + column_type: &ColumnType, + ) -> Result<(), String> { + let (avro_schema, _) = schema_unwrap_optional(avro_schema); + + match self { + Coercion::DebeziumTime(time_type) => { + let (expected_sql, matches_sql) = if time_type.is_timestamp() { + ( + "TIMESTAMP", + matches!(column_type.typ, SqlType::Timestamp | SqlType::TimestampTz), + ) + } else { + ("TIME", matches!(column_type.typ, SqlType::Time)) + }; + if !matches_sql { + return Err(self.wrong_column_error(expected_sql, column_type)); + } + + let matches_primitive = match time_type.avro_primitive() { + "int" => avro_schema == &AvroSchema::Int, + "long" => avro_schema == &AvroSchema::Long, + "string" => avro_schema == &AvroSchema::String, + _ => false, + }; + if !matches_primitive { + return Err(format!( + "invalid Avro schema for Debezium type '{}': expected '{}', but found {}", + self.name(), + time_type.avro_primitive(), + schema_json(avro_schema) + )); + } + } + Coercion::VariableScaleDecimal => { + if column_type.typ != SqlType::Decimal { + return Err(self.wrong_column_error("DECIMAL", column_type)); + } + + let resolved = resolve_ref(avro_schema, refs) + .map_err(|name| format!("error resolving Avro schema reference: {name}"))?; + let valid = matches!( + resolved, + AvroSchema::Record(record) + if record.lookup.contains_key("scale") && record.lookup.contains_key("value") + ); + if !valid { + return Err(format!( + "invalid Avro schema for Debezium type '{}': expected a record with 'scale' and 'value' fields, but found {}", + self.name(), + schema_json(resolved) + )); + } + } + } + + Ok(()) + } + + fn wrong_column_error(self, expected_sql: &str, column_type: &ColumnType) -> String { + format!( + "Debezium type '{}' can only be deserialized into a SQL {expected_sql} column, but the column has type '{}'", + self.name(), + column_type.typ + ) + } +} + +/// Detect the coercion, if any, implied by a record field. +/// +/// Annotations on primitive types are recovered from the field's custom +/// attributes, where [`hoist_coercible_types`] places them. Annotations on +/// named types (records, enums) are read from the (ref-resolved) schema, which +/// preserves them natively. +pub fn field_coercion( + custom_attributes: &BTreeMap, + schema: &AvroSchema, + refs: &AvroSchemaRefs, +) -> Option { + if let Some(coercion) = custom_attributes + .get("connect.name") + .and_then(JsonValue::as_str) + .and_then(Coercion::from_connect_name) + { + return Some(coercion); + } + + let (schema, _) = schema_unwrap_optional(schema); + let schema = resolve_ref(schema, refs).ok()?; + let AvroSchema::Record(record) = schema else { + return None; + }; + record + .attributes + .get("connect.name") + .and_then(JsonValue::as_str) + .and_then(Coercion::from_connect_name) +} + +/// Rewrite an Avro schema so that coercion annotations on primitive types +/// survive parsing by `apache-avro`. +/// +/// `apache-avro` discards attributes on primitive schemas, so a `connect.name` +/// nested inside a field's `type` object is lost. This copies each recognized +/// annotation up to its enclosing record field, where the crate preserves it as +/// a field-level custom attribute. Named types (records, enums) keep their +/// attributes natively and are left untouched. +/// +/// Returns `None` if the input is not valid JSON, letting the caller fall back +/// to the original string so the normal schema-parsing error surfaces. +pub fn hoist_coercible_types(schema_json: &str) -> Option { + let mut value: JsonValue = serde_json::from_str(schema_json).ok()?; + hoist_walk(&mut value); + serde_json::to_string(&value).ok() +} + +/// Recursively visit every record schema, hoisting primitive annotations onto +/// its fields. +fn hoist_walk(value: &mut JsonValue) { + match value { + JsonValue::Object(map) => { + if let Some(JsonValue::Array(fields)) = map.get_mut("fields") { + for field in fields.iter_mut() { + if let JsonValue::Object(field_map) = field { + hoist_field(field_map); + } + } + } + for child in map.values_mut() { + hoist_walk(child); + } + } + JsonValue::Array(items) => { + for item in items.iter_mut() { + hoist_walk(item); + } + } + _ => {} + } +} + +/// Copy a coercion annotation from a field's primitive `type` up to the field. +fn hoist_field(field: &mut Map) { + if field.contains_key("connect.name") { + return; + } + if let Some(connect_name) = field.get("type").and_then(type_hoistable_connect_name) { + field.insert("connect.name".to_string(), JsonValue::String(connect_name)); + } +} + +/// Find a hoistable annotation inside a field's `type`, whether the type is a +/// bare object or a nullable union. +fn type_hoistable_connect_name(type_value: &JsonValue) -> Option { + match type_value { + JsonValue::Object(obj) => object_hoistable_connect_name(obj), + JsonValue::Array(variants) => variants.iter().find_map(|variant| match variant { + JsonValue::Object(obj) => object_hoistable_connect_name(obj), + _ => None, + }), + _ => None, + } +} + +/// Return an object schema's `connect.name` if it is a recognized coercion on a +/// primitive type. Named types keep their attributes natively, so they are not +/// hoisted. +fn object_hoistable_connect_name(obj: &Map) -> Option { + let type_name = obj.get("type").and_then(JsonValue::as_str)?; + if !is_primitive_type(type_name) { + return None; + } + let connect_name = obj.get("connect.name")?.as_str()?; + Coercion::from_connect_name(connect_name).map(|_| connect_name.to_string()) +} + +fn is_primitive_type(type_name: &str) -> bool { + matches!( + type_name, + "null" | "boolean" | "int" | "long" | "float" | "double" | "bytes" | "string" + ) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn detects_and_hoists_primitive_annotation() { + // A temporal annotation on a primitive type is hoisted onto the field + // and then recognized. + let schema = r#"{ + "type": "record", + "name": "Row", + "fields": [ + { "name": "ts", "type": { "type": "long", "connect.name": "io.debezium.time.MicroTimestamp" } }, + { "name": "t", "type": ["null", { "type": "long", "connect.name": "io.debezium.time.NanoTime" }] }, + { "name": "e", "type": { "type": "string", "connect.name": "io.debezium.data.Enum" } }, + { "name": "plain", "type": "long" } + ] + }"#; + + let hoisted = hoist_coercible_types(schema).unwrap(); + let avro = AvroSchema::parse_str(&hoisted).unwrap(); + let AvroSchema::Record(record) = avro else { + panic!("expected record schema"); + }; + let refs = AvroSchemaRefs::new(); + + assert_eq!( + field_coercion( + &record.fields[0].custom_attributes, + &record.fields[0].schema, + &refs + ), + Some(Coercion::DebeziumTime(DebeziumTimeType::TimestampMicros)) + ); + assert_eq!( + field_coercion( + &record.fields[1].custom_attributes, + &record.fields[1].schema, + &refs + ), + Some(Coercion::DebeziumTime(DebeziumTimeType::TimeNanos)) + ); + // Non-temporal and plain fields are not coerced. + assert_eq!( + field_coercion( + &record.fields[2].custom_attributes, + &record.fields[2].schema, + &refs + ), + None + ); + assert_eq!( + field_coercion( + &record.fields[3].custom_attributes, + &record.fields[3].schema, + &refs + ), + None + ); + } + + #[test] + fn detects_named_type_annotation_without_hoisting() { + // A record annotation is recognized natively, and the same annotation is + // recognized through a by-name reference (the second field). + let schema = r#"{ + "type": "record", + "name": "Row", + "fields": [ + { "name": "amount", "type": { + "type": "record", + "name": "VariableScaleDecimal", + "namespace": "io.debezium.data", + "fields": [ + { "name": "scale", "type": "int" }, + { "name": "value", "type": "bytes" } + ], + "connect.name": "io.debezium.data.VariableScaleDecimal" + }}, + { "name": "amount_ref", "type": ["null", "io.debezium.data.VariableScaleDecimal"] } + ] + }"#; + + // The record annotation is not hoisted onto the field. + let hoisted = hoist_coercible_types(schema).unwrap(); + let parsed: JsonValue = serde_json::from_str(&hoisted).unwrap(); + assert!(parsed["fields"][0].get("connect.name").is_none()); + + let avro = AvroSchema::parse_str(&hoisted).unwrap(); + let resolved = apache_avro::schema::ResolvedSchema::try_from(&avro).unwrap(); + let refs: AvroSchemaRefs = resolved + .get_names() + .iter() + .map(|(name, schema)| (name.clone(), (*schema).clone())) + .collect(); + let AvroSchema::Record(record) = &avro else { + panic!("expected record schema"); + }; + + for field in &record.fields { + assert_eq!( + field_coercion(&field.custom_attributes, &field.schema, &refs), + Some(Coercion::VariableScaleDecimal), + "field {} not recognized", + field.name + ); + } + } +} diff --git a/crates/adapters/src/format/avro/debezium.rs b/crates/adapters/src/format/avro/debezium.rs new file mode 100644 index 00000000000..3704842bd8c --- /dev/null +++ b/crates/adapters/src/format/avro/debezium.rs @@ -0,0 +1,402 @@ +//! Debezium-specific value conversions used by the Avro input connector. +//! +//! Debezium does not use Avro's native `timestamp-*`/`time-*`/`decimal` logical +//! types. Instead it tags plain Avro types with a `connect.name` attribute that +//! names a Debezium semantic type, e.g.: +//! +//! ```json +//! { "type": "long", "connect.name": "io.debezium.time.MicroTimestamp" } +//! ``` +//! +//! This module maps those semantic types to the representation Feldera's column +//! types expect. The generic machinery that recognizes annotations, drives the +//! deserializer, and validates schemas lives in [`super::coercion`]; this module +//! provides only the Debezium-specific pieces it delegates to. +//! +//! See the [Debezium temporal types documentation][docs] for the wire format of +//! each type. +//! +//! [docs]: https://debezium.io/documentation/reference/stable/connectors/postgresql.html#postgresql-temporal-types + +use apache_avro::{BigDecimal, types::Value}; +use chrono::{DateTime, Timelike, Utc}; +use num_bigint::BigInt; + +/// Number of microseconds in a full day, used to normalize `ZonedTime`. +const MICROS_PER_DAY: i64 = 86_400 * 1_000_000; + +/// A Debezium temporal semantic type carried by a `connect.name` attribute. +/// +/// Covers the `TIME` and `TIMESTAMP` families. `io.debezium.time.Date` and +/// `org.apache.kafka.connect.data.Date` are intentionally omitted: they are +/// plain day counts that already deserialize correctly into a Feldera `DATE` +/// column without any conversion. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DebeziumTimeType { + /// Milliseconds since midnight (`int`). + /// + /// `io.debezium.time.Time`, `org.apache.kafka.connect.data.Time`. + TimeMillis, + /// Microseconds since midnight (`long`). `io.debezium.time.MicroTime`. + TimeMicros, + /// Nanoseconds since midnight (`long`). `io.debezium.time.NanoTime`. + TimeNanos, + /// ISO-8601 time-with-offset string, e.g. `10:15:30+01:00`. + /// `io.debezium.time.ZonedTime`. + ZonedTime, + /// Milliseconds since the Unix epoch (`long`). + /// + /// `io.debezium.time.Timestamp`, `org.apache.kafka.connect.data.Timestamp`. + TimestampMillis, + /// Microseconds since the Unix epoch (`long`). + /// `io.debezium.time.MicroTimestamp`. + TimestampMicros, + /// Nanoseconds since the Unix epoch (`long`). + /// `io.debezium.time.NanoTimestamp`. + TimestampNanos, + /// ISO-8601 timestamp-with-offset string, + /// e.g. `2011-12-03T10:15:30.030431+01:00`. `io.debezium.time.ZonedTimestamp`. + ZonedTimestamp, +} + +impl DebeziumTimeType { + /// Map a `connect.name` attribute value to a temporal type, or `None` if it + /// is not a recognized temporal type. + pub fn from_connect_name(connect_name: &str) -> Option { + Some(match connect_name { + "io.debezium.time.Time" | "org.apache.kafka.connect.data.Time" => Self::TimeMillis, + "io.debezium.time.MicroTime" => Self::TimeMicros, + "io.debezium.time.NanoTime" => Self::TimeNanos, + "io.debezium.time.ZonedTime" => Self::ZonedTime, + "io.debezium.time.Timestamp" | "org.apache.kafka.connect.data.Timestamp" => { + Self::TimestampMillis + } + "io.debezium.time.MicroTimestamp" => Self::TimestampMicros, + "io.debezium.time.NanoTimestamp" => Self::TimestampNanos, + "io.debezium.time.ZonedTimestamp" => Self::ZonedTimestamp, + _ => return None, + }) + } + + /// A representative `connect.name` for this type, used in error messages. + pub fn canonical_connect_name(self) -> &'static str { + match self { + Self::TimeMillis => "io.debezium.time.Time", + Self::TimeMicros => "io.debezium.time.MicroTime", + Self::TimeNanos => "io.debezium.time.NanoTime", + Self::ZonedTime => "io.debezium.time.ZonedTime", + Self::TimestampMillis => "io.debezium.time.Timestamp", + Self::TimestampMicros => "io.debezium.time.MicroTimestamp", + Self::TimestampNanos => "io.debezium.time.NanoTimestamp", + Self::ZonedTimestamp => "io.debezium.time.ZonedTimestamp", + } + } + + /// `true` if this type populates a `TIMESTAMP` column, `false` for a `TIME` + /// column. + pub fn is_timestamp(self) -> bool { + matches!( + self, + Self::TimestampMillis + | Self::TimestampMicros + | Self::TimestampNanos + | Self::ZonedTimestamp + ) + } + + /// The Avro primitive type name that carries this Debezium type on the wire. + pub fn avro_primitive(self) -> &'static str { + match self { + Self::TimeMillis => "int", + Self::TimeMicros | Self::TimeNanos => "long", + Self::ZonedTime => "string", + Self::TimestampMillis | Self::TimestampMicros | Self::TimestampNanos => "long", + Self::ZonedTimestamp => "string", + } + } + + /// Convert a decoded Avro value into microseconds: since the Unix epoch for + /// timestamp types, since midnight for time types. + pub fn to_micros(self, value: &Value) -> Result { + match self { + Self::TimeMillis | Self::TimestampMillis => millis_to_micros(as_i64(value)?), + Self::TimeMicros | Self::TimestampMicros => as_i64(value), + Self::TimeNanos | Self::TimestampNanos => Ok(as_i64(value)? / 1_000), + Self::ZonedTimestamp => parse_zoned_timestamp_micros(as_str(value)?), + Self::ZonedTime => parse_zoned_time_micros(as_str(value)?), + } + } +} + +/// Convert an `io.debezium.data.VariableScaleDecimal` record into a decimal +/// string. +/// +/// The type is used for `NUMERIC`/`DECIMAL` columns without a fixed scale. On +/// the wire it is a record with a `scale` (`int`) and a `value` (`bytes`, the +/// big-endian two's-complement unscaled integer). The decimal string it yields +/// is parsed by the target `DECIMAL` column deserializer. +pub fn variable_scale_decimal_to_string(value: &Value) -> Result { + let Value::Record(fields) = value else { + return Err(format!( + "expected a VariableScaleDecimal record, but found {value:?}" + )); + }; + + let mut scale = None; + let mut unscaled = None; + for (name, field) in fields { + match name.as_str() { + "scale" => scale = Some(record_int(field)?), + "value" => unscaled = Some(record_bytes(field)?), + _ => {} + } + } + + let scale = scale + .ok_or_else(|| "VariableScaleDecimal record is missing the 'scale' field".to_string())?; + let unscaled = unscaled + .ok_or_else(|| "VariableScaleDecimal record is missing the 'value' field".to_string())?; + + let unscaled = BigInt::from_signed_bytes_be(unscaled); + Ok(BigDecimal::new(unscaled, scale as i64).to_string()) +} + +/// Read an integer out of an Avro `int` or `long` value. +fn as_i64(value: &Value) -> Result { + match value { + Value::Int(i) => Ok(*i as i64), + Value::Long(i) => Ok(*i), + other => Err(format!( + "expected an integer value for a Debezium temporal type, but found {other:?}" + )), + } +} + +/// Read a string out of an Avro `string` value. +fn as_str(value: &Value) -> Result<&str, String> { + match value { + Value::String(s) => Ok(s.as_str()), + other => Err(format!( + "expected a string value for a Debezium zoned temporal type, but found {other:?}" + )), + } +} + +/// Read the `scale` field of a VariableScaleDecimal record. +fn record_int(value: &Value) -> Result { + match value { + Value::Int(i) => Ok(*i), + Value::Union(_, inner) => record_int(inner), + other => Err(format!( + "expected an int for the VariableScaleDecimal 'scale' field, but found {other:?}" + )), + } +} + +/// Read the `value` field of a VariableScaleDecimal record. +fn record_bytes(value: &Value) -> Result<&[u8], String> { + match value { + Value::Bytes(bytes) | Value::Fixed(_, bytes) => Ok(bytes), + Value::Union(_, inner) => record_bytes(inner), + other => Err(format!( + "expected bytes for the VariableScaleDecimal 'value' field, but found {other:?}" + )), + } +} + +fn millis_to_micros(millis: i64) -> Result { + millis.checked_mul(1_000).ok_or_else(|| { + format!("millisecond value {millis} overflows when converted to microseconds") + }) +} + +/// Parse an ISO-8601 `ZonedTimestamp` string into microseconds since the epoch. +fn parse_zoned_timestamp_micros(s: &str) -> Result { + let dt = DateTime::parse_from_rfc3339(s.trim()) + .map_err(|e| format!("invalid Debezium ZonedTimestamp '{s}': {e}"))?; + Ok(dt.timestamp_micros()) +} + +/// Parse an ISO-8601 `ZonedTime` string (e.g. `10:15:30+01:00`) into +/// microseconds since midnight, normalized to UTC. +/// +/// A Feldera `TIME` has no timezone, so the offset is folded into the +/// wall-clock time modulo 24 hours. +fn parse_zoned_time_micros(s: &str) -> Result { + let s = s.trim(); + // Reuse the RFC3339 parser by pinning the time to an arbitrary date. The + // date is discarded after normalizing to UTC; only the time of day matters. + let dt = DateTime::parse_from_rfc3339(&format!("1970-01-01T{s}")) + .map_err(|e| format!("invalid Debezium ZonedTime '{s}': {e}"))?; + let time = dt.with_timezone(&Utc).time(); + let micros = + time.num_seconds_from_midnight() as i64 * 1_000_000 + time.nanosecond() as i64 / 1_000; + Ok(micros.rem_euclid(MICROS_PER_DAY)) +} + +#[cfg(test)] +mod test { + use super::*; + use apache_avro::types::Value; + + #[test] + fn connect_name_mapping() { + assert_eq!( + DebeziumTimeType::from_connect_name("io.debezium.time.MicroTimestamp"), + Some(DebeziumTimeType::TimestampMicros) + ); + assert_eq!( + DebeziumTimeType::from_connect_name("org.apache.kafka.connect.data.Timestamp"), + Some(DebeziumTimeType::TimestampMillis) + ); + // Date is handled by the plain-int path, not as a coercion. + assert_eq!( + DebeziumTimeType::from_connect_name("io.debezium.time.Date"), + None + ); + assert_eq!( + DebeziumTimeType::from_connect_name("io.debezium.data.Json"), + None + ); + } + + #[test] + fn numeric_timestamps_to_micros() { + // 2021-01-01T00:00:00Z = 1_609_459_200 s. + let secs = 1_609_459_200i64; + assert_eq!( + DebeziumTimeType::TimestampMillis + .to_micros(&Value::Long(secs * 1_000)) + .unwrap(), + secs * 1_000_000 + ); + assert_eq!( + DebeziumTimeType::TimestampMicros + .to_micros(&Value::Long(secs * 1_000_000)) + .unwrap(), + secs * 1_000_000 + ); + assert_eq!( + DebeziumTimeType::TimestampNanos + .to_micros(&Value::Long(secs * 1_000_000_000)) + .unwrap(), + secs * 1_000_000 + ); + } + + #[test] + fn numeric_times_to_micros() { + // 01:02:03.004 = 3_723_004 ms since midnight. + let millis = 3_723_004i64; + let micros = millis * 1_000; + assert_eq!( + DebeziumTimeType::TimeMillis + .to_micros(&Value::Int(millis as i32)) + .unwrap(), + micros + ); + assert_eq!( + DebeziumTimeType::TimeMicros + .to_micros(&Value::Long(micros)) + .unwrap(), + micros + ); + assert_eq!( + DebeziumTimeType::TimeNanos + .to_micros(&Value::Long(micros * 1_000)) + .unwrap(), + micros + ); + } + + #[test] + fn zoned_timestamp_to_micros() { + // Example from the Debezium docs, with a +01:00 offset. + let micros = DebeziumTimeType::ZonedTimestamp + .to_micros(&Value::String( + "2011-12-03T10:15:30.030431+01:00".to_string(), + )) + .unwrap(); + // Equivalent UTC instant: 2011-12-03T09:15:30.030431Z. + let expected = DateTime::parse_from_rfc3339("2011-12-03T09:15:30.030431Z") + .unwrap() + .timestamp_micros(); + assert_eq!(micros, expected); + } + + #[test] + fn zoned_time_to_micros() { + // 10:15:30+01:00 normalizes to 09:15:30 UTC. + let micros = DebeziumTimeType::ZonedTime + .to_micros(&Value::String("10:15:30+01:00".to_string())) + .unwrap(); + let expected = (9 * 3_600 + 15 * 60 + 30) * 1_000_000; + assert_eq!(micros, expected); + } + + #[test] + fn zoned_time_wraps_across_midnight() { + // 00:30:00+01:00 is 23:30:00 UTC on the previous day; only the time + // of day survives. + let micros = DebeziumTimeType::ZonedTime + .to_micros(&Value::String("00:30:00+01:00".to_string())) + .unwrap(); + let expected = (23i64 * 3_600 + 30 * 60) * 1_000_000; + assert_eq!(micros, expected); + } + + #[test] + fn zoned_time_utc_with_fraction() { + let micros = DebeziumTimeType::ZonedTime + .to_micros(&Value::String("13:37:03.123456Z".to_string())) + .unwrap(); + let expected = (13i64 * 3_600 + 37 * 60 + 3) * 1_000_000 + 123_456; + assert_eq!(micros, expected); + } + + #[test] + fn invalid_zoned_values_error() { + assert!( + DebeziumTimeType::ZonedTimestamp + .to_micros(&Value::String("not-a-timestamp".to_string())) + .is_err() + ); + assert!( + DebeziumTimeType::ZonedTime + .to_micros(&Value::String("99:99:99Z".to_string())) + .is_err() + ); + } + + #[test] + fn variable_scale_decimal() { + let record = |scale: i32, unscaled: i64| { + Value::Record(vec![ + ("scale".to_string(), Value::Int(scale)), + ( + "value".to_string(), + Value::Bytes(BigInt::from(unscaled).to_signed_bytes_be()), + ), + ]) + }; + + assert_eq!( + variable_scale_decimal_to_string(&record(2, 12345)).unwrap(), + "123.45" + ); + assert_eq!( + variable_scale_decimal_to_string(&record(3, -6789)).unwrap(), + "-6.789" + ); + assert_eq!( + variable_scale_decimal_to_string(&record(0, 42)).unwrap(), + "42" + ); + } + + #[test] + fn variable_scale_decimal_missing_field_errors() { + let record = Value::Record(vec![("scale".to_string(), Value::Int(2))]); + assert!(variable_scale_decimal_to_string(&record).is_err()); + } +} diff --git a/crates/adapters/src/format/avro/deserializer.rs b/crates/adapters/src/format/avro/deserializer.rs index 6fa7a5fa43a..df8dc257d4e 100644 --- a/crates/adapters/src/format/avro/deserializer.rs +++ b/crates/adapters/src/format/avro/deserializer.rs @@ -47,6 +47,7 @@ use std::{ use crate::format::avro::resolve_ref; +use super::coercion::{Coerced, Coercion, field_coercion}; use super::input::avro_de_config; fn deserialize_decimal(d: &Decimal, schema: &Schema) -> Result { @@ -70,6 +71,10 @@ pub struct Deserializer<'de> { schema: &'de Schema, refs: &'de AvroSchemaRefs, input: &'de Value, + /// Value coercion for the enclosing record field, if any. When set, the + /// decoded Avro value is converted (see [`Coercion`]) before being handed to + /// the target column's deserializer. + coercion: Option, } struct SeqDeserializer<'de> { @@ -99,6 +104,21 @@ impl<'de> Deserializer<'de> { input, schema, refs, + coercion: None, + } + } + + fn with_coercion( + input: &'de Value, + schema: &'de Schema, + refs: &'de AvroSchemaRefs, + coercion: Option, + ) -> Self { + Deserializer { + input, + schema, + refs, + coercion, } } } @@ -180,6 +200,26 @@ fn unwrap_union<'de>(v: &'de Value, schema: &'de Schema) -> (&'de Value, &'de Sc } } +/// Apply a [`Coercion`] to `value` and hand the result to `visitor`. +/// +/// This is the single point at which coercions produce Serde values, called +/// from every deserialize entry point a coerced field can reach. +// The error type is determined by the `apache_avro` crate. +#[allow(clippy::result_large_err)] +fn visit_coerced<'de, V: Visitor<'de>>( + coercion: Coercion, + value: &Value, + visitor: V, +) -> Result { + match coercion + .coerce(value) + .map_err(|e| de::Error::custom(format!("error applying Avro value coercion: {e}")))? + { + Coerced::I64(i) => visitor.visit_i64(i), + Coerced::Str(s) => visitor.visit_str(&s), + } +} + impl<'de> de::Deserializer<'de> for &'_ Deserializer<'de> { type Error = Error; @@ -188,6 +228,9 @@ impl<'de> de::Deserializer<'de> for &'_ Deserializer<'de> { V: Visitor<'de>, { let (val, schema) = unwrap_union(self.input, self.schema); + if let Some(coercion) = self.coercion { + return visit_coerced(coercion, val, visitor); + } match val { Value::Null => visitor.visit_unit(), &Value::Boolean(b) => visitor.visit_bool(b), @@ -233,7 +276,11 @@ impl<'de> de::Deserializer<'de> for &'_ Deserializer<'de> { where V: Visitor<'de>, { - match unwrap_union(self.input, self.schema).0 { + let (val, _) = unwrap_union(self.input, self.schema); + if let Some(coercion) = self.coercion { + return visit_coerced(coercion, val, visitor); + } + match val { Value::String(s) | Value::Enum(_, s) => visitor.visit_borrowed_str(s), Value::Bytes(bytes) | Value::Fixed(_, bytes) => { let s = @@ -251,7 +298,11 @@ impl<'de> de::Deserializer<'de> for &'_ Deserializer<'de> { where V: Visitor<'de>, { - match unwrap_union(self.input, self.schema).0 { + let (val, _) = unwrap_union(self.input, self.schema); + if let Some(coercion) = self.coercion { + return visit_coerced(coercion, val, visitor); + } + match val { Value::String(s) | Value::Enum(_, s) => visitor.visit_borrowed_str(s), Value::Bytes(bytes) | Value::Fixed(_, bytes) => { let s = String::from_utf8(bytes.to_owned()) @@ -310,13 +361,19 @@ impl<'de> de::Deserializer<'de> for &'_ Deserializer<'de> { self.schema ))); }; - visitor.visit_some(&Deserializer::new( + visitor.visit_some(&Deserializer::with_coercion( inner, &union_schema.variants()[*i as usize], self.refs, + self.coercion, )) } - v => visitor.visit_some(&Deserializer::new(v, self.schema, self.refs)), + v => visitor.visit_some(&Deserializer::with_coercion( + v, + self.schema, + self.refs, + self.coercion, + )), } } @@ -517,10 +574,12 @@ impl<'de> de::MapAccess<'de> for RecordDeserializer<'de> { { match self.value.take() { Some(value) => { - let result = seed.deserialize(&Deserializer::new( + let field = &self.record_schema.fields[self.next_field_index]; + let result = seed.deserialize(&Deserializer::with_coercion( value, - &self.record_schema.fields[self.next_field_index].schema, + &field.schema, self.refs, + field_coercion(&field.custom_attributes, &field.schema, self.refs), )); self.next_field_index += 1; result diff --git a/crates/adapters/src/format/avro/input.rs b/crates/adapters/src/format/avro/input.rs index 381d93fea1c..b320a28d64b 100644 --- a/crates/adapters/src/format/avro/input.rs +++ b/crates/adapters/src/format/avro/input.rs @@ -284,7 +284,19 @@ impl AvroParser { &self, schema_str: &str, ) -> Result<(AvroSchema, AvroSchemaRefs, AvroSchema), ControllerError> { - let schema = AvroSchema::parse_str(schema_str).map_err(|e| { + // Debezium encodes some columns with `connect.name` annotations that + // `apache-avro` drops for primitive types. Hoist them onto their record + // fields so they survive parsing (see `super::coercion`). + let schema_str: Cow = + if matches!(self.config.update_format, AvroUpdateFormat::Debezium) { + super::coercion::hoist_coercible_types(schema_str) + .map(Cow::Owned) + .unwrap_or(Cow::Borrowed(schema_str)) + } else { + Cow::Borrowed(schema_str) + }; + + let schema = AvroSchema::parse_str(&schema_str).map_err(|e| { ControllerError::invalid_parser_configuration( &self.endpoint_name, &format!("error parsing Avro schema: {e}"), diff --git a/crates/adapters/src/format/avro/schema.rs b/crates/adapters/src/format/avro/schema.rs index f4d5c44cc84..d8092395346 100644 --- a/crates/adapters/src/format/avro/schema.rs +++ b/crates/adapters/src/format/avro/schema.rs @@ -13,6 +13,7 @@ use feldera_adapterlib::catalog::AvroSchemaRefs; use feldera_types::program_schema::{ColumnType, Field, Relation, SqlIdentifier, SqlType}; use tracing::warn; +use crate::format::avro::coercion::field_coercion; use crate::format::avro::resolve_ref; /// Indicates whether the field has an optional type (`["null", T]`) and, @@ -82,6 +83,24 @@ pub fn validate_struct_schema( } }; + // A field carrying a coercion annotation (e.g. a Debezium temporal type + // or variable-scale decimal) is validated against the SQL column type by + // the coercion itself, since its Avro schema differs from the column's + // natural Avro type. + if let Some(coercion) = + field_coercion(&avro_field.custom_attributes, &avro_field.schema, refs) + { + coercion + .validate(&avro_field.schema, refs, &field.columntype) + .map_err(|e| { + format!( + "error validating schema for column '{}': {e}", + field.name.name() + ) + })?; + continue; + } + validate_field_schema( &avro_field.name, &avro_field.schema, diff --git a/crates/adapters/src/format/avro/test.rs b/crates/adapters/src/format/avro/test.rs index 98e945d8798..e3f511a3261 100644 --- a/crates/adapters/src/format/avro/test.rs +++ b/crates/adapters/src/format/avro/test.rs @@ -16,11 +16,12 @@ use crate::{catalog::SerBatchReader, util::run_in_posix_runtime}; use apache_avro::{ Schema as AvroSchema, from_avro_datum, schema::ResolvedSchema, to_avro_datum, types::Value, }; +use chrono::{DateTime, Utc}; use dbsp::trace::BatchReaderFactories; use dbsp::typed_batch::{DynSpineSnapshot, SpineSnapshot as TypedSpineSnapshot, TypedBatch}; use dbsp::{DBData, IndexedZSetReader, OrdIndexedZSet, OrdZSet, ZWeight, utils::Tup2}; use feldera_adapterlib::transport::OutputBatchType; -use feldera_sqllib::{ByteArray, Uuid, Variant}; +use feldera_sqllib::{ByteArray, Date, SqlDecimal, Time, Timestamp, TimestampTz, Uuid, Variant}; use feldera_types::{ deserialize_table_record, format::avro::{AvroEncoderConfig, AvroEncoderKeyMode}, @@ -33,6 +34,7 @@ use feldera_types::{ serialize_struct, }; use itertools::Itertools; +use num_bigint::BigInt; use proptest::prelude::*; use proptest::proptest; use rand::rngs::StdRng; @@ -122,6 +124,18 @@ impl DebeziumMessage { /// Debezium message Avro schema with the specified inner record schema. fn debezium_avro_schema(value_schema: &str, value_type_name: &str) -> AvroSchema { + let schema_str = debezium_avro_schema_str(value_schema, value_type_name); + + println!("Debezium Avro schema: {schema_str}"); + + AvroSchema::parse_str(&schema_str).unwrap() +} + +/// Raw JSON of the Debezium envelope schema embedding the given inner record +/// schema. Unlike [`debezium_avro_schema`], the string is returned verbatim so +/// that Debezium `connect.name` annotations survive (parsing them into an +/// `AvroSchema` drops attributes on primitive types). +fn debezium_avro_schema_str(value_schema: &str, value_type_name: &str) -> String { // Note: placing `after` before `before` to trigger schema reference resolution. let schema_str = r#"{ "type": "record", @@ -202,9 +216,7 @@ fn debezium_avro_schema(value_schema: &str, value_type_name: &str) -> AvroSchema "connect.name": "test_namespace.Envelope" }"#.replace("VALUE_SCHEMA", value_schema).replace("VALUE_TYPE", value_type_name); - println!("Debezium Avro schema: {schema_str}"); - - AvroSchema::parse_str(&schema_str).unwrap() + schema_str } fn serialize_record(x: &T, schema: &AvroSchema) -> Vec @@ -1063,6 +1075,579 @@ fn test_ms_time() { run_parser_test(vec![test]); } +/// Raw wire representation of a row carrying every Debezium temporal type. +/// +/// The fields hold the exact values Debezium puts on the wire (milliseconds, +/// microseconds, or nanoseconds since epoch/midnight, or ISO-8601 strings) so +/// that the test can encode a realistic Debezium message. The connector must +/// convert these into the [`TestTemporal`] representation on the way in. +#[derive(Debug, Clone, PartialEq, Eq)] +struct TestTemporalRaw { + id: i64, + ts_millis: i64, + ts_micros: i64, + ts_nanos: i64, + ts_zoned: String, + ts_connect: i64, + ts_opt: Option, + t_millis: i32, + t_micros: i64, + t_nanos: i64, + t_zoned: String, + t_connect: i32, + dt: i32, +} + +serialize_table_record!(TestTemporalRaw[13]{ + r#id["id"]: i64, + r#ts_millis["ts_millis"]: i64, + r#ts_micros["ts_micros"]: i64, + r#ts_nanos["ts_nanos"]: i64, + r#ts_zoned["ts_zoned"]: String, + r#ts_connect["ts_connect"]: i64, + r#ts_opt["ts_opt"]: Option, + r#t_millis["t_millis"]: i32, + r#t_micros["t_micros"]: i64, + r#t_nanos["t_nanos"]: i64, + r#t_zoned["t_zoned"]: String, + r#t_connect["t_connect"]: i32, + r#dt["dt"]: i32 +}); + +deserialize_table_record!(TestTemporalRaw["Temporal", Variant, 13] { + (r#id, "id", false, i64, |_| None), + (r#ts_millis, "ts_millis", false, i64, |_| None), + (r#ts_micros, "ts_micros", false, i64, |_| None), + (r#ts_nanos, "ts_nanos", false, i64, |_| None), + (r#ts_zoned, "ts_zoned", false, String, |_| None), + (r#ts_connect, "ts_connect", false, i64, |_| None), + (r#ts_opt, "ts_opt", true, Option, |_| Some(None)), + (r#t_millis, "t_millis", false, i32, |_| None), + (r#t_micros, "t_micros", false, i64, |_| None), + (r#t_nanos, "t_nanos", false, i64, |_| None), + (r#t_zoned, "t_zoned", false, String, |_| None), + (r#t_connect, "t_connect", false, i32, |_| None), + (r#dt, "dt", false, i32, |_| None) +}); + +/// Decoded representation of [`TestTemporalRaw`]: every temporal column parsed +/// into the matching Feldera type. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct TestTemporal { + id: i64, + ts_millis: Timestamp, + ts_micros: Timestamp, + ts_nanos: Timestamp, + ts_zoned: TimestampTz, + ts_connect: Timestamp, + ts_opt: Option, + t_millis: Time, + t_micros: Time, + t_nanos: Time, + t_zoned: Time, + t_connect: Time, + dt: Date, +} + +serialize_table_record!(TestTemporal[13]{ + r#id["id"]: i64, + r#ts_millis["ts_millis"]: Timestamp, + r#ts_micros["ts_micros"]: Timestamp, + r#ts_nanos["ts_nanos"]: Timestamp, + r#ts_zoned["ts_zoned"]: TimestampTz, + r#ts_connect["ts_connect"]: Timestamp, + r#ts_opt["ts_opt"]: Option, + r#t_millis["t_millis"]: Time, + r#t_micros["t_micros"]: Time, + r#t_nanos["t_nanos"]: Time, + r#t_zoned["t_zoned"]: Time, + r#t_connect["t_connect"]: Time, + r#dt["dt"]: Date +}); + +deserialize_table_record!(TestTemporal["Temporal", Variant, 13] { + (r#id, "id", false, i64, |_| None), + (r#ts_millis, "ts_millis", false, Timestamp, |_| None), + (r#ts_micros, "ts_micros", false, Timestamp, |_| None), + (r#ts_nanos, "ts_nanos", false, Timestamp, |_| None), + (r#ts_zoned, "ts_zoned", false, TimestampTz, |_| None), + (r#ts_connect, "ts_connect", false, Timestamp, |_| None), + (r#ts_opt, "ts_opt", true, Option, |_| Some(None)), + (r#t_millis, "t_millis", false, Time, |_| None), + (r#t_micros, "t_micros", false, Time, |_| None), + (r#t_nanos, "t_nanos", false, Time, |_| None), + (r#t_zoned, "t_zoned", false, Time, |_| None), + (r#t_connect, "t_connect", false, Time, |_| None), + (r#dt, "dt", false, Date, |_| None) +}); + +impl TestTemporal { + /// Avro value schema covering every supported Debezium temporal type. Each + /// column is a plain `int`/`long`/`string` tagged with a Debezium + /// `connect.name`, exactly as Debezium emits them. + fn value_avro_schema() -> &'static str { + r#"{ + "type": "record", + "name": "Temporal", + "connect.name": "test_namespace.Temporal", + "fields": [ + { "name": "id", "type": "long" }, + { "name": "ts_millis", "type": { "type": "long", "connect.name": "io.debezium.time.Timestamp" } }, + { "name": "ts_micros", "type": { "type": "long", "connect.name": "io.debezium.time.MicroTimestamp" } }, + { "name": "ts_nanos", "type": { "type": "long", "connect.name": "io.debezium.time.NanoTimestamp" } }, + { "name": "ts_zoned", "type": { "type": "string", "connect.name": "io.debezium.time.ZonedTimestamp" } }, + { "name": "ts_connect", "type": { "type": "long", "connect.name": "org.apache.kafka.connect.data.Timestamp" } }, + { "name": "ts_opt", "type": ["null", { "type": "long", "connect.name": "io.debezium.time.MicroTimestamp" }], "default": null }, + { "name": "t_millis", "type": { "type": "int", "connect.name": "io.debezium.time.Time" } }, + { "name": "t_micros", "type": { "type": "long", "connect.name": "io.debezium.time.MicroTime" } }, + { "name": "t_nanos", "type": { "type": "long", "connect.name": "io.debezium.time.NanoTime" } }, + { "name": "t_zoned", "type": { "type": "string", "connect.name": "io.debezium.time.ZonedTime" } }, + { "name": "t_connect", "type": { "type": "int", "connect.name": "org.apache.kafka.connect.data.Time" } }, + { "name": "dt", "type": { "type": "int", "connect.name": "io.debezium.time.Date" } } + ] + }"# + } + + fn schema() -> Vec { + vec![ + Field::new("id".into(), ColumnType::bigint(false)), + Field::new("ts_millis".into(), ColumnType::timestamp(false)), + Field::new("ts_micros".into(), ColumnType::timestamp(false)), + Field::new("ts_nanos".into(), ColumnType::timestamp(false)), + Field::new("ts_zoned".into(), ColumnType::timestamp_tz(false)), + Field::new("ts_connect".into(), ColumnType::timestamp(false)), + Field::new("ts_opt".into(), ColumnType::timestamp(true)), + Field::new("t_millis".into(), ColumnType::time(false)), + Field::new("t_micros".into(), ColumnType::time(false)), + Field::new("t_nanos".into(), ColumnType::time(false)), + Field::new("t_zoned".into(), ColumnType::time(false)), + Field::new("t_connect".into(), ColumnType::time(false)), + Field::new("dt".into(), ColumnType::date(false)), + ] + } + + fn relation_schema() -> Relation { + Relation { + name: SqlIdentifier::new("Temporal", false), + fields: Self::schema(), + materialized: false, + properties: BTreeMap::new(), + primary_key: None, + } + } +} + +/// Parse every Debezium temporal semantic type into the matching Feldera +/// `TIME`/`TIMESTAMP`/`DATE` column. +#[test] +fn test_debezium_temporal_types() { + // Base instant with millisecond precision so its millisecond, microsecond, + // and nanosecond encodings all denote the same point in time. + let instant = DateTime::parse_from_rfc3339("2021-06-15T12:30:45.123Z") + .unwrap() + .with_timezone(&Utc); + let ts_micros = instant.timestamp_micros(); + let ts_millis = ts_micros / 1_000; + let ts_nanos = ts_micros * 1_000; + let dt_days = (instant.timestamp() / 86_400) as i32; + + // Time of day 12:30:45.123, again at millisecond precision. + let time_micros = (12 * 3_600 + 30 * 60 + 45) * 1_000_000 + 123_000; + let time_millis = (time_micros / 1_000) as i32; + let time_nanos = time_micros * 1_000; + let time_as_time = Time::from_nanoseconds((time_micros * 1_000) as u64); + + let raw = TestTemporalRaw { + id: 1, + ts_millis, + ts_micros, + ts_nanos, + // +02:00 wall clock for the same instant, to exercise offset handling. + ts_zoned: "2021-06-15T14:30:45.123+02:00".to_string(), + ts_connect: ts_millis, + ts_opt: Some(ts_micros), + t_millis: time_millis, + t_micros: time_micros, + t_nanos: time_nanos, + t_zoned: "14:30:45.123+02:00".to_string(), + t_connect: time_millis, + dt: dt_days, + }; + + let expected = TestTemporal { + id: 1, + ts_millis: Timestamp::from_microseconds(ts_micros), + ts_micros: Timestamp::from_microseconds(ts_micros), + ts_nanos: Timestamp::from_microseconds(ts_micros), + ts_zoned: TimestampTz::from_microseconds(ts_micros), + ts_connect: Timestamp::from_microseconds(ts_micros), + ts_opt: Some(Timestamp::from_microseconds(ts_micros)), + t_millis: time_as_time, + t_micros: time_as_time, + t_nanos: time_as_time, + t_zoned: time_as_time, + t_connect: time_as_time, + dt: Date::from_days(dt_days), + }; + + // Second row exercises a NULL value in the nullable timestamp column. + let raw_null = TestTemporalRaw { + id: 2, + ts_opt: None, + ..raw.clone() + }; + let expected_null = TestTemporal { + id: 2, + ts_opt: None, + ..expected.clone() + }; + + // Encode a Debezium message from the raw values. The envelope schema is + // parsed here purely to drive Avro encoding; the connector receives the raw + // schema string (below) with the `connect.name` annotations intact. + let envelope_str = debezium_avro_schema_str(TestTemporal::value_avro_schema(), "Temporal"); + let envelope_schema = AvroSchema::parse_str(&envelope_str).unwrap(); + let resolved = ResolvedSchema::try_from(&envelope_schema).unwrap(); + + let encode = |row: &TestTemporalRaw| { + let mut buffer = vec![0; 5]; + let serializer = AvroSchemaSerializer::new(&envelope_schema, resolved.get_names(), true); + let dbz_message = DebeziumMessage::new("u", Some(row.clone()), Some(row.clone())); + let val = dbz_message + .serialize_with_context(serializer, &avro_ser_config()) + .unwrap(); + let mut avro_record = to_avro_datum(&envelope_schema, val).unwrap(); + buffer.append(&mut avro_record); + (buffer, vec![]) + }; + + let test = TestCase { + relation_schema: TestTemporal::relation_schema(), + config: AvroParserConfig { + update_format: AvroUpdateFormat::Debezium, + schema: Some(envelope_str), + skip_schema_id: false, + registry_config: Default::default(), + }, + input_batches: vec![encode(&raw), encode(&raw_null)], + expected_output: vec![ + MockUpdate::Delete(expected.clone()), + MockUpdate::Insert(expected.clone()), + MockUpdate::Delete(expected_null.clone()), + MockUpdate::Insert(expected_null.clone()), + ], + }; + + run_parser_test(vec![test]); +} + +/// A Debezium temporal type mapped to an incompatible SQL column must be +/// rejected during schema validation. +#[test] +fn test_debezium_temporal_type_mismatch() { + use super::schema::validate_struct_schema; + + let value_schema = r#"{ + "type": "record", + "name": "Temporal", + "connect.name": "test_namespace.Temporal", + "fields": [ + { "name": "id", "type": { "type": "long", "connect.name": "io.debezium.time.Timestamp" } } + ] + }"#; + + let hoisted = super::coercion::hoist_coercible_types(value_schema).unwrap(); + let schema = AvroSchema::parse_str(&hoisted).unwrap(); + let resolved = ResolvedSchema::try_from(&schema).unwrap(); + let refs = resolved + .get_names() + .iter() + .map(|(name, schema)| (name.clone(), (*schema).clone())) + .collect(); + + // The `id` column is BIGINT, but the Avro field is a Debezium timestamp. + let fields = vec![Field::new("id".into(), ColumnType::bigint(false))]; + let err = validate_struct_schema(&schema, &refs, &fields).unwrap_err(); + assert!( + err.contains("io.debezium.time.Timestamp") && err.contains("TIMESTAMP"), + "unexpected error message: {err}" + ); +} + +/// Raw wire representation of the same instant encoded four ways: as a +/// `ZonedTimestamp` (string) and a `MicroTimestamp` (long), each destined for +/// both a `TIMESTAMP` and a `TIMESTAMP WITH TIME ZONE` column. +#[derive(Debug, Clone)] +struct TsTargetsRaw { + z_ts: String, + z_tstz: String, + n_ts: i64, + n_tstz: i64, +} + +serialize_table_record!(TsTargetsRaw[4]{ + r#z_ts["z_ts"]: String, + r#z_tstz["z_tstz"]: String, + r#n_ts["n_ts"]: i64, + r#n_tstz["n_tstz"]: i64 +}); + +/// Decoded form of [`TsTargetsRaw`]. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct TsTargets { + z_ts: Timestamp, + z_tstz: TimestampTz, + n_ts: Timestamp, + n_tstz: TimestampTz, +} + +deserialize_table_record!(TsTargets["TsTargets", Variant, 4] { + (r#z_ts, "z_ts", false, Timestamp, |_| None), + (r#z_tstz, "z_tstz", false, TimestampTz, |_| None), + (r#n_ts, "n_ts", false, Timestamp, |_| None), + (r#n_tstz, "n_tstz", false, TimestampTz, |_| None) +}); + +impl TsTargets { + fn value_avro_schema() -> &'static str { + r#"{ + "type": "record", + "name": "TsTargets", + "connect.name": "test_namespace.TsTargets", + "fields": [ + { "name": "z_ts", "type": { "type": "string", "connect.name": "io.debezium.time.ZonedTimestamp" } }, + { "name": "z_tstz", "type": { "type": "string", "connect.name": "io.debezium.time.ZonedTimestamp" } }, + { "name": "n_ts", "type": { "type": "long", "connect.name": "io.debezium.time.MicroTimestamp" } }, + { "name": "n_tstz", "type": { "type": "long", "connect.name": "io.debezium.time.MicroTimestamp" } } + ] + }"# + } + + fn relation_schema() -> Relation { + Relation { + name: SqlIdentifier::new("TsTargets", false), + fields: vec![ + Field::new("z_ts".into(), ColumnType::timestamp(false)), + Field::new("z_tstz".into(), ColumnType::timestamp_tz(false)), + Field::new("n_ts".into(), ColumnType::timestamp(false)), + Field::new("n_tstz".into(), ColumnType::timestamp_tz(false)), + ], + materialized: false, + properties: BTreeMap::new(), + primary_key: None, + } + } +} + +/// A Debezium timestamp type deserializes into either a `TIMESTAMP` or a +/// `TIMESTAMP WITH TIME ZONE` column. This covers both the string-encoded +/// `ZonedTimestamp` and a numeric type against both column types. +#[test] +fn test_debezium_timestamp_column_targets() { + let instant = DateTime::parse_from_rfc3339("2021-06-15T12:30:45.123Z") + .unwrap() + .with_timezone(&Utc); + let micros = instant.timestamp_micros(); + // Same instant expressed in a +02:00 offset. + let zoned = "2021-06-15T14:30:45.123+02:00".to_string(); + + let raw = TsTargetsRaw { + z_ts: zoned.clone(), + z_tstz: zoned.clone(), + n_ts: micros, + n_tstz: micros, + }; + let expected = TsTargets { + z_ts: Timestamp::from_microseconds(micros), + z_tstz: TimestampTz::from_microseconds(micros), + n_ts: Timestamp::from_microseconds(micros), + n_tstz: TimestampTz::from_microseconds(micros), + }; + + let envelope_str = debezium_avro_schema_str(TsTargets::value_avro_schema(), "TsTargets"); + let envelope_schema = AvroSchema::parse_str(&envelope_str).unwrap(); + let resolved = ResolvedSchema::try_from(&envelope_schema).unwrap(); + + let mut buffer = vec![0; 5]; + let serializer = AvroSchemaSerializer::new(&envelope_schema, resolved.get_names(), true); + let dbz_message = DebeziumMessage::new("u", Some(raw.clone()), Some(raw.clone())); + let val = dbz_message + .serialize_with_context(serializer, &avro_ser_config()) + .unwrap(); + let mut avro_record = to_avro_datum(&envelope_schema, val).unwrap(); + buffer.append(&mut avro_record); + + let test = TestCase { + relation_schema: TsTargets::relation_schema(), + config: AvroParserConfig { + update_format: AvroUpdateFormat::Debezium, + schema: Some(envelope_str), + skip_schema_id: false, + registry_config: Default::default(), + }, + input_batches: vec![(buffer, vec![])], + expected_output: vec![ + MockUpdate::Delete(expected.clone()), + MockUpdate::Insert(expected.clone()), + ], + }; + + run_parser_test(vec![test]); +} + +/// Raw wire representation of a Debezium `VariableScaleDecimal`: a record with +/// the scale and the big-endian two's-complement unscaled value. +#[derive(Debug, Clone)] +struct VariableScaleDecimalRaw { + scale: i32, + value: ByteArray, +} + +serialize_table_record!(VariableScaleDecimalRaw[2]{ + r#scale["scale"]: i32, + r#value["value"]: ByteArray +}); + +/// Raw wire representation of a row with variable-scale decimal columns, one of +/// them nullable and encoded via a by-name reference to the same record type. +#[derive(Debug, Clone)] +struct TestDecimalRaw { + id: i64, + amount: VariableScaleDecimalRaw, + amount_opt: Option, +} + +serialize_table_record!(TestDecimalRaw[3]{ + r#id["id"]: i64, + r#amount["amount"]: VariableScaleDecimalRaw, + r#amount_opt["amount_opt"]: Option +}); + +/// Decoded representation of [`TestDecimalRaw`]: every variable-scale decimal +/// parsed into a Feldera `DECIMAL` column. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct TestDecimal { + id: i64, + amount: SqlDecimal<10, 3>, + amount_opt: Option>, +} + +deserialize_table_record!(TestDecimal["Dec", Variant, 3] { + (r#id, "id", false, i64, |_| None), + (r#amount, "amount", false, SqlDecimal<10, 3>, |_| None), + (r#amount_opt, "amount_opt", true, Option>, |_| Some(None)) +}); + +impl TestDecimal { + /// Avro value schema with two `VariableScaleDecimal` columns; `amount_opt` + /// references the record type by name, exercising reference resolution. + fn value_avro_schema() -> &'static str { + r#"{ + "type": "record", + "name": "Dec", + "connect.name": "test_namespace.Dec", + "fields": [ + { "name": "id", "type": "long" }, + { "name": "amount", "type": { + "type": "record", + "name": "VariableScaleDecimal", + "namespace": "io.debezium.data", + "fields": [ + { "name": "scale", "type": "int" }, + { "name": "value", "type": "bytes" } + ], + "connect.version": 1, + "connect.name": "io.debezium.data.VariableScaleDecimal" + }}, + { "name": "amount_opt", "type": ["null", "io.debezium.data.VariableScaleDecimal"], "default": null } + ] + }"# + } + + fn relation_schema() -> Relation { + Relation { + name: SqlIdentifier::new("Dec", false), + fields: vec![ + Field::new("id".into(), ColumnType::bigint(false)), + Field::new("amount".into(), ColumnType::decimal(10, 3, false)), + Field::new("amount_opt".into(), ColumnType::decimal(10, 3, true)), + ], + materialized: false, + properties: BTreeMap::new(), + primary_key: None, + } + } +} + +/// Parse Debezium `io.debezium.data.VariableScaleDecimal` records into Feldera +/// `DECIMAL` columns, including a nullable column and a by-name type reference. +#[test] +fn test_debezium_variable_scale_decimal() { + let raw_dec = |scale: i32, unscaled: i64| VariableScaleDecimalRaw { + scale, + value: ByteArray::new(&BigInt::from(unscaled).to_signed_bytes_be()), + }; + + // The wire scale differs from the column scale (3), which is the whole point + // of the variable-scale type. + let row1 = TestDecimalRaw { + id: 1, + amount: raw_dec(2, 12345), // 123.45 + amount_opt: Some(raw_dec(3, -6789)), // -6.789 + }; + let row2 = TestDecimalRaw { + id: 2, + amount: raw_dec(0, 42), // 42 + amount_opt: None, + }; + + let expected1 = TestDecimal { + id: 1, + amount: SqlDecimal::<10, 3>::new(12345, 2).unwrap(), + amount_opt: Some(SqlDecimal::<10, 3>::new(-6789, 3).unwrap()), + }; + let expected2 = TestDecimal { + id: 2, + amount: SqlDecimal::<10, 3>::new(42, 0).unwrap(), + amount_opt: None, + }; + + let envelope_str = debezium_avro_schema_str(TestDecimal::value_avro_schema(), "Dec"); + let envelope_schema = AvroSchema::parse_str(&envelope_str).unwrap(); + let resolved = ResolvedSchema::try_from(&envelope_schema).unwrap(); + + let encode = |row: &TestDecimalRaw| { + let mut buffer = vec![0; 5]; + let serializer = AvroSchemaSerializer::new(&envelope_schema, resolved.get_names(), true); + let dbz_message = DebeziumMessage::new("u", Some(row.clone()), Some(row.clone())); + let val = dbz_message + .serialize_with_context(serializer, &avro_ser_config()) + .unwrap(); + let mut avro_record = to_avro_datum(&envelope_schema, val).unwrap(); + buffer.append(&mut avro_record); + (buffer, vec![]) + }; + + let test = TestCase { + relation_schema: TestDecimal::relation_schema(), + config: AvroParserConfig { + update_format: AvroUpdateFormat::Debezium, + schema: Some(envelope_str), + skip_schema_id: false, + registry_config: Default::default(), + }, + input_batches: vec![encode(&row1), encode(&row2)], + expected_output: vec![ + MockUpdate::Delete(expected1.clone()), + MockUpdate::Insert(expected1.clone()), + MockUpdate::Delete(expected2.clone()), + MockUpdate::Insert(expected2.clone()), + ], + }; + + run_parser_test(vec![test]); +} + /// Type used to serialize different integer types as Avro `int`. #[derive( Debug, diff --git a/crates/feldera-types/src/format/avro.rs b/crates/feldera-types/src/format/avro.rs index 7f2b6b046f1..0ed15ea5767 100644 --- a/crates/feldera-types/src/format/avro.rs +++ b/crates/feldera-types/src/format/avro.rs @@ -25,6 +25,36 @@ pub enum AvroUpdateFormat { Raw, /// Debezium data change event format. + /// + /// ### Temporal types + /// + /// Debezium encodes temporal columns with a `connect.name` annotation + /// rather than a native Avro logical type. The input connector recognizes + /// these annotations and converts each value into the matching Feldera + /// column type: + /// + /// | `connect.name` | wire encoding | Feldera column | + /// |-----------------------------------------|-----------------------------------|----------------| + /// | `io.debezium.time.Date` | `int` days since epoch | `DATE` | + /// | `io.debezium.time.Time` | `int` milliseconds since midnight | `TIME` | + /// | `io.debezium.time.MicroTime` | `long` microseconds since midnight| `TIME` | + /// | `io.debezium.time.NanoTime` | `long` nanoseconds since midnight | `TIME` | + /// | `io.debezium.time.ZonedTime` | ISO-8601 `string` | `TIME` | + /// | `io.debezium.time.Timestamp` | `long` milliseconds since epoch | `TIMESTAMP` | + /// | `io.debezium.time.MicroTimestamp` | `long` microseconds since epoch | `TIMESTAMP` | + /// | `io.debezium.time.NanoTimestamp` | `long` nanoseconds since epoch | `TIMESTAMP` | + /// | `io.debezium.time.ZonedTimestamp` | ISO-8601 `string` | `TIMESTAMP` | + /// + /// Any timestamp type populates either a `TIMESTAMP` or a `TIMESTAMP WITH + /// TIME ZONE` column; the connector stores the same instant (microseconds + /// since the Unix epoch) in both cases. + /// + /// The `org.apache.kafka.connect.data.{Date,Time,Timestamp}` types emitted + /// with `time.precision.mode=connect` are also supported. + /// + /// `io.debezium.data.VariableScaleDecimal` (used for `NUMERIC`/`DECIMAL` + /// columns without a fixed scale, encoded as a `{scale, value}` record) is + /// parsed into a `DECIMAL` column. #[serde(rename = "debezium")] Debezium, diff --git a/docs.feldera.com/docs/formats/avro.md b/docs.feldera.com/docs/formats/avro.md index 18386d10f71..cc03c15e73a 100644 --- a/docs.feldera.com/docs/formats/avro.md +++ b/docs.feldera.com/docs/formats/avro.md @@ -96,6 +96,40 @@ A SQL column and a field in the Avro schema are compatible if the following cond | `VARIANT` | `string` | values of type `VARIANT` are deserialized from JSON-encoded strings (see [`VARIANT` documetation](/sql/json)) | | user-defined types | `record` | Avro record schema must match SQL user-defined type definition according to the same schema compatibility rules as for SQL tables| +### Debezium logical types + +When `update_format` is set to `debezium`, the connector additionally recognizes +Debezium's temporal and decimal +[semantic types](https://debezium.io/documentation/reference/stable/connectors/postgresql.html#postgresql-data-types). +Debezium encodes these as plain Avro types annotated with a `connect.name` +attribute rather than as native Avro logical types. The connector converts each +value into the matching SQL column type: + +| `connect.name` | Avro encoding | SQL column | +|-----------------------------------------|-------------------------------------------------------------|-------------| +| `io.debezium.time.Date` | `int` days since epoch | `DATE` | +| `io.debezium.time.Time` | `int` milliseconds since midnight | `TIME` | +| `io.debezium.time.MicroTime` | `long` microseconds since midnight | `TIME` | +| `io.debezium.time.NanoTime` | `long` nanoseconds since midnight | `TIME` | +| `io.debezium.time.ZonedTime` | ISO-8601 `string` (e.g. `10:15:30+01:00`) | `TIME` | +| `io.debezium.time.Timestamp` | `long` milliseconds since epoch | `TIMESTAMP` | +| `io.debezium.time.MicroTimestamp` | `long` microseconds since epoch | `TIMESTAMP` | +| `io.debezium.time.NanoTimestamp` | `long` nanoseconds since epoch | `TIMESTAMP` | +| `io.debezium.time.ZonedTimestamp` | ISO-8601 `string` (e.g. `2011-12-03T10:15:30.030431+01:00`) | `TIMESTAMP` | +| `io.debezium.data.VariableScaleDecimal` | `{scale, value}` record | `DECIMAL` | + +Notes: + +* Any timestamp type can be deserialized into either a `TIMESTAMP` or a + `TIMESTAMP WITH TIME ZONE` column; the connector stores the same instant in + both cases. Zoned values are normalized to UTC. +* The equivalent `org.apache.kafka.connect.data.Date`, `.Time`, and `.Timestamp` + types, emitted when Debezium's `time.precision.mode` is set to `connect`, are + also supported. +* `io.debezium.data.VariableScaleDecimal` represents decimals whose scale is not + fixed in the schema (for example, an unconstrained PostgreSQL `NUMERIC`). The + wire scale may differ from the SQL column's scale. + ### Configuration The following properties can be used to configure the Avro parser. All of these properties are optional. However, From 31e3528db3889567556d1edcf2cf5517abfd4eaa Mon Sep 17 00:00:00 2001 From: "release-feldera-feldera[bot]" Date: Tue, 14 Jul 2026 15:41:17 +0000 Subject: [PATCH 2/9] ci: Prepare for v0.322.0 --- Cargo.lock | 42 +++++++++++------------ Cargo.toml | 26 +++++++------- openapi.json | 2 +- python/dbt-feldera/pyproject.toml | 2 +- python/felderize/pyproject.toml | 2 +- python/pyproject.toml | 2 +- python/uv.lock | 4 +-- sql-to-dbsp-compiler/SQL-compiler/pom.xml | 2 +- 8 files changed, 41 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 337c2ee257a..d3c2156c043 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3976,7 +3976,7 @@ dependencies = [ [[package]] name = "dbsp" -version = "0.321.0" +version = "0.322.0" dependencies = [ "anyhow", "arc-swap", @@ -4066,7 +4066,7 @@ dependencies = [ [[package]] name = "dbsp_adapters" -version = "0.321.0" +version = "0.322.0" dependencies = [ "actix", "actix-codec", @@ -4210,7 +4210,7 @@ dependencies = [ [[package]] name = "dbsp_nexmark" -version = "0.321.0" +version = "0.322.0" dependencies = [ "anyhow", "ascii_table", @@ -5194,7 +5194,7 @@ dependencies = [ [[package]] name = "fda" -version = "0.321.0" +version = "0.322.0" dependencies = [ "anyhow", "arrow", @@ -5246,7 +5246,7 @@ dependencies = [ [[package]] name = "feldera-adapterlib" -version = "0.321.0" +version = "0.322.0" dependencies = [ "actix-web", "anyhow", @@ -5280,7 +5280,7 @@ dependencies = [ [[package]] name = "feldera-buffer-cache" -version = "0.321.0" +version = "0.322.0" dependencies = [ "crossbeam-utils", "enum-map", @@ -5308,7 +5308,7 @@ dependencies = [ [[package]] name = "feldera-datagen" -version = "0.321.0" +version = "0.322.0" dependencies = [ "anyhow", "async-channel 2.5.0", @@ -5334,7 +5334,7 @@ dependencies = [ [[package]] name = "feldera-fxp" -version = "0.321.0" +version = "0.322.0" dependencies = [ "bytecheck", "dbsp", @@ -5354,7 +5354,7 @@ dependencies = [ [[package]] name = "feldera-iceberg" -version = "0.321.0" +version = "0.322.0" dependencies = [ "anyhow", "chrono", @@ -5377,7 +5377,7 @@ dependencies = [ [[package]] name = "feldera-ir" -version = "0.321.0" +version = "0.322.0" dependencies = [ "proptest", "proptest-derive", @@ -5389,7 +5389,7 @@ dependencies = [ [[package]] name = "feldera-macros" -version = "0.321.0" +version = "0.322.0" dependencies = [ "prettyplease", "proc-macro2", @@ -5399,7 +5399,7 @@ dependencies = [ [[package]] name = "feldera-observability" -version = "0.321.0" +version = "0.322.0" dependencies = [ "actix-http", "awc", @@ -5414,7 +5414,7 @@ dependencies = [ [[package]] name = "feldera-rest-api" -version = "0.321.0" +version = "0.322.0" dependencies = [ "chrono", "feldera-observability", @@ -5433,7 +5433,7 @@ dependencies = [ [[package]] name = "feldera-samply" -version = "0.321.0" +version = "0.322.0" dependencies = [ "crossbeam", "feldera-size-of", @@ -5469,7 +5469,7 @@ dependencies = [ [[package]] name = "feldera-sqllib" -version = "0.321.0" +version = "0.322.0" dependencies = [ "arcstr", "base58", @@ -5513,7 +5513,7 @@ dependencies = [ [[package]] name = "feldera-storage" -version = "0.321.0" +version = "0.322.0" dependencies = [ "anyhow", "crossbeam", @@ -5536,7 +5536,7 @@ dependencies = [ [[package]] name = "feldera-types" -version = "0.321.0" +version = "0.322.0" dependencies = [ "actix-web", "anyhow", @@ -8813,7 +8813,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pipeline-manager" -version = "0.321.0" +version = "0.322.0" dependencies = [ "actix-cors", "actix-files", @@ -10023,7 +10023,7 @@ dependencies = [ [[package]] name = "readers" -version = "0.321.0" +version = "0.322.0" dependencies = [ "async-std", "csv", @@ -11722,7 +11722,7 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "sltsqlvalue" -version = "0.321.0" +version = "0.322.0" dependencies = [ "dbsp", "feldera-sqllib", @@ -12193,7 +12193,7 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "storage-test-compat" -version = "0.321.0" +version = "0.322.0" dependencies = [ "dbsp", "derive_more 1.0.0", diff --git a/Cargo.toml b/Cargo.toml index ffecd064b40..04dfef6851d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace.package] authors = ["Feldera Team "] -version = "0.321.0" +version = "0.322.0" license = "MIT OR Apache-2.0" homepage = "https://github.com/feldera/feldera" repository = "https://github.com/feldera/feldera" @@ -108,7 +108,7 @@ csv = "1.2.2" csv-core = "0.1.10" dashmap = "6.1.0" datafusion = "53.1" -dbsp = { path = "crates/dbsp", version = "0.321.0" } +dbsp = { path = "crates/dbsp", version = "0.322.0" } dbsp_nexmark = { path = "crates/nexmark" } deadpool-postgres = "0.14.1" # Feldera fork of delta-rs: upstream tag `rust-v0.32.3` + 3 patches, on branch @@ -136,20 +136,20 @@ erased-serde = "0.3.31" fake = "2.10" fastbloom = "0.14.0" fdlimit = "0.3.0" -feldera-buffer-cache = { version = "0.321.0", path = "crates/buffer-cache" } +feldera-buffer-cache = { version = "0.322.0", path = "crates/buffer-cache" } feldera-cloud1-client = "0.1.2" feldera-datagen = { path = "crates/datagen" } -feldera-fxp = { version = "0.321.0", path = "crates/fxp", features = ["dbsp"] } +feldera-fxp = { version = "0.322.0", path = "crates/fxp", features = ["dbsp"] } feldera-iceberg = { path = "crates/iceberg" } -feldera-observability = { version = "0.321.0", path = "crates/feldera-observability" } -feldera-macros = { version = "0.321.0", path = "crates/feldera-macros" } -feldera-sqllib = { version = "0.321.0", path = "crates/sqllib" } -feldera-storage = { version = "0.321.0", path = "crates/storage" } -feldera-types = { version = "0.321.0", path = "crates/feldera-types" } -feldera-rest-api = { version = "0.321.0", path = "crates/rest-api" } -feldera-ir = { version = "0.321.0", path = "crates/ir" } -feldera-adapterlib = { version = "0.321.0", path = "crates/adapterlib" } -feldera-samply = { version = "0.321.0", path = "crates/samply" } +feldera-observability = { version = "0.322.0", path = "crates/feldera-observability" } +feldera-macros = { version = "0.322.0", path = "crates/feldera-macros" } +feldera-sqllib = { version = "0.322.0", path = "crates/sqllib" } +feldera-storage = { version = "0.322.0", path = "crates/storage" } +feldera-types = { version = "0.322.0", path = "crates/feldera-types" } +feldera-rest-api = { version = "0.322.0", path = "crates/rest-api" } +feldera-ir = { version = "0.322.0", path = "crates/ir" } +feldera-adapterlib = { version = "0.322.0", path = "crates/adapterlib" } +feldera-samply = { version = "0.322.0", path = "crates/samply" } flate2 = "1.1.0" form_urlencoded = "1.2.0" fs_extra = "1.3.0" diff --git a/openapi.json b/openapi.json index d056cd59b89..15e8043bcee 100644 --- a/openapi.json +++ b/openapi.json @@ -10,7 +10,7 @@ "license": { "name": "MIT OR Apache-2.0" }, - "version": "0.321.0" + "version": "0.322.0" }, "paths": { "/config/authentication": { diff --git a/python/dbt-feldera/pyproject.toml b/python/dbt-feldera/pyproject.toml index 52b68ec121e..efb418810d6 100644 --- a/python/dbt-feldera/pyproject.toml +++ b/python/dbt-feldera/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" name = "dbt-feldera" readme = "README.md" description = "The dbt adapter for Feldera — DBSP-native incremental view maintenance" -version = "0.321.0" +version = "0.322.0" license = "MIT" requires-python = ">=3.10" authors = [ diff --git a/python/felderize/pyproject.toml b/python/felderize/pyproject.toml index b85605a12d3..e643bc850e2 100644 --- a/python/felderize/pyproject.toml +++ b/python/felderize/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" name = "felderize" readme = "README.md" description = "SQL dialect to Feldera SQL translator agent" -version = "0.321.0" +version = "0.322.0" license = "MIT" requires-python = ">=3.10" authors = [ diff --git a/python/pyproject.toml b/python/pyproject.toml index 616e8287b1c..18dcac354e2 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" name = "feldera" readme = "README.md" description = "The feldera python client" -version = "0.321.0" +version = "0.322.0" license = "MIT" requires-python = ">=3.10,<3.14" authors = [ diff --git a/python/uv.lock b/python/uv.lock index 3456b3766fd..984aaa814a3 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -9,7 +9,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-07-06T16:35:57.058623357Z" +exclude-newer = "2026-07-07T15:40:42.875333412Z" exclude-newer-span = "P1W" [[package]] @@ -273,7 +273,7 @@ wheels = [ [[package]] name = "feldera" -version = "0.321.0" +version = "0.322.0" source = { editable = "." } dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, diff --git a/sql-to-dbsp-compiler/SQL-compiler/pom.xml b/sql-to-dbsp-compiler/SQL-compiler/pom.xml index d0856c923bd..c9da7249b42 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/pom.xml +++ b/sql-to-dbsp-compiler/SQL-compiler/pom.xml @@ -19,7 +19,7 @@ 1.43.0 2.22.0 3.1.12 - 0.321.0 + 0.322.0 1.28.0 From af0a72ba7ed639718640cbf39ad49266f65527b5 Mon Sep 17 00:00:00 2001 From: Jyotshna Yaparla Date: Mon, 13 Jul 2026 13:32:32 -0500 Subject: [PATCH 3/9] deploy: prefer IPv4 in glibc DNS resolution Node.js-based GHA actions (e.g. setup-uv) hang and time out fetching raw.githubusercontent.com on our EKS runners, which have no IPv6 routing but still get an AAAA record back. glibc's getaddrinfo tries that IPv6 result first and only falls back to IPv4 after the OS-level timeout. Fixing this in the feldera-dev image itself (via /etc/gai.conf) applies to every environment that uses it, instead of patching NODE_OPTIONS per-cluster in each ArgoCD-managed runner/job-hook config. --- deploy/Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/deploy/Dockerfile b/deploy/Dockerfile index e30ae52f5d7..4dbde4cd202 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -35,6 +35,12 @@ RUN curl -sS https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \ -o /usr/local/share/ca-certificates/aws-global-rds-bundle.crt RUN update-ca-certificates +# Prefer IPv4 over IPv6 for dual-stack DNS results. Our EKS nodes have no IPv6 +# routing, so glibc's default preference for AAAA records makes any resolver +# (Node.js actions like setup-uv, curl, etc.) hang until the IPv6 attempt times +# out before falling back to IPv4. +RUN echo "precedence ::ffff:0:0/96 100" >> /etc/gai.conf + # Set UTF-8 locale. Needed for the Rust compiler to handle Unicode column names. RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ locale-gen From fdd8c9647a774f1df604ac746ef3d8c949fccb37 Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Tue, 14 Jul 2026 17:16:17 -0700 Subject: [PATCH 4/9] Revert "deploy: prefer IPv4 in glibc DNS resolution" This reverts commit af0a72ba7ed639718640cbf39ad49266f65527b5. --- deploy/Dockerfile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 4dbde4cd202..e30ae52f5d7 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -35,12 +35,6 @@ RUN curl -sS https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \ -o /usr/local/share/ca-certificates/aws-global-rds-bundle.crt RUN update-ca-certificates -# Prefer IPv4 over IPv6 for dual-stack DNS results. Our EKS nodes have no IPv6 -# routing, so glibc's default preference for AAAA records makes any resolver -# (Node.js actions like setup-uv, curl, etc.) hang until the IPv6 attempt times -# out before falling back to IPv4. -RUN echo "precedence ::ffff:0:0/96 100" >> /etc/gai.conf - # Set UTF-8 locale. Needed for the Rust compiler to handle Unicode column names. RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ locale-gen From 3f11627f327e425d2dbd7d3373e7255d70f4616d Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Tue, 14 Jul 2026 14:19:42 -0700 Subject: [PATCH 5/9] [connectors] Improve Avro key name validation Signed-off-by: Mihai Budiu --- crates/adapters/src/format/avro/schema.rs | 118 ++++++++++++++++++++-- 1 file changed, 112 insertions(+), 6 deletions(-) diff --git a/crates/adapters/src/format/avro/schema.rs b/crates/adapters/src/format/avro/schema.rs index d8092395346..75ca70ca440 100644 --- a/crates/adapters/src/format/avro/schema.rs +++ b/crates/adapters/src/format/avro/schema.rs @@ -458,6 +458,12 @@ pub fn is_valid_avro_identifier(ident: &str) -> bool { && ident.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') } +/// A valid Avro fullname is a sequence of dot-separated identifiers, such as +/// `namespace.record_name`. +pub fn is_valid_avro_fullname(fullname: &str) -> bool { + fullname.split('.').all(is_valid_avro_identifier) +} + #[derive(Default)] pub struct AvroSchemaBuilder { namespace: Option, @@ -497,10 +503,26 @@ impl AvroSchemaBuilder { top_level: bool, ) -> Result { let name = name.name(); - if !is_valid_avro_identifier(&name) { - return Err(format!("'{name}' is not a valid Avro identifier")); + if !is_valid_avro_fullname(&name) { + return Err(format!( + "'{name}' is not a valid Avro name: each dot-separated segment must start with a letter or underscore and contain only letters, digits, and underscores" + )); } + // Names such as 'view.key', which the catalog assigns to the key + // schema of an indexed view, are Avro fullnames: the segments before + // the last dot form a namespace, appended to the configured one. + let (namespace, name) = match name.rsplit_once('.') { + Some((prefix, base)) => ( + Some(match &self.namespace { + Some(ns) => format!("{ns}.{prefix}"), + None => prefix.to_string(), + }), + base.to_string(), + ), + None => (self.namespace.clone(), name), + }; + let mut fields = Vec::with_capacity(struct_fields.len()); let mut lookup = BTreeMap::new(); @@ -513,10 +535,7 @@ impl AvroSchemaBuilder { } Ok(RecordSchema { - name: Name { - name: name.to_string(), - namespace: self.namespace.clone(), - }, + name: Name { name, namespace }, aliases: None, doc: None, fields, @@ -647,3 +666,90 @@ impl AvroSchemaBuilder { }) } } + +#[cfg(test)] +mod test { + use super::{AvroSchemaBuilder, is_valid_avro_fullname, schema_json}; + use apache_avro::{Schema as AvroSchema, schema::Name}; + use feldera_types::program_schema::{ColumnType, Field, Relation, SqlIdentifier}; + use std::collections::BTreeMap; + + fn relation(name: &str) -> Relation { + Relation::new( + SqlIdentifier::new(name, false), + vec![Field::new( + SqlIdentifier::new("id", false), + ColumnType::boolean(false), + )], + false, + BTreeMap::new(), + ) + } + + fn record_name(schema: &AvroSchema) -> Name { + match schema { + AvroSchema::Record(record) => record.name.clone(), + _ => panic!("expected a record schema, got {schema:?}"), + } + } + + #[test] + fn fullname_validation() { + for valid in ["key", "v.key", "_a1.b2.c3"] { + assert!(is_valid_avro_fullname(valid), "'{valid}' should be valid"); + } + for invalid in ["", ".", "v..key", ".key", "key.", "1v.key", "v-1.key"] { + assert!( + !is_valid_avro_fullname(invalid), + "'{invalid}' should be invalid" + ); + } + } + + #[test] + fn dotted_relation_name_becomes_namespace() { + let schema = AvroSchemaBuilder::new() + .relation_to_avro_schema(&relation("v.key")) + .unwrap(); + let name = record_name(&schema); + assert_eq!(name.name, "key"); + assert_eq!(name.namespace.as_deref(), Some("v")); + + // The apache_avro parser must accept the generated schema. + AvroSchema::parse_str(&schema_json(&schema)).unwrap(); + } + + #[test] + fn configured_namespace_prefixes_derived_namespace() { + let schema = AvroSchemaBuilder::new() + .with_namespace(Some("com.acme")) + .relation_to_avro_schema(&relation("v.key")) + .unwrap(); + let name = record_name(&schema); + assert_eq!(name.name, "key"); + assert_eq!(name.namespace.as_deref(), Some("com.acme.v")); + } + + #[test] + fn plain_relation_name_keeps_configured_namespace() { + let schema = AvroSchemaBuilder::new() + .with_namespace(Some("com.acme")) + .relation_to_avro_schema(&relation("v")) + .unwrap(); + let name = record_name(&schema); + assert_eq!(name.name, "v"); + assert_eq!(name.namespace.as_deref(), Some("com.acme")); + } + + #[test] + fn invalid_relation_name_is_rejected() { + for invalid in ["v..key", ".key", "key.", "v-1"] { + assert!( + AvroSchemaBuilder::new() + .relation_to_avro_schema(&relation(invalid)) + .is_err(), + "'{invalid}' should be rejected" + ); + } + } +} From 53438ccb6b55894812efb3a9cd805cbb06554ccd Mon Sep 17 00:00:00 2001 From: Ben Pfaff Date: Tue, 14 Jul 2026 14:20:30 -0700 Subject: [PATCH 6/9] [dbsp] Don't issue warning for pidlock file. This suppresses a warning of the form ``` INFO dbsp::circuit::checkpointer: Keeping unexpected File { size: 0 } feldera.pidlock ``` when a pipeline restarts. Tested manually by starting a pipeline with nonempty storage with and without this commit. Fixes: https://github.com/feldera/cloud/issues/1726 Signed-off-by: Ben Pfaff --- crates/dbsp/src/circuit/checkpointer.rs | 2 ++ crates/dbsp/src/storage/dirlock.rs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/dbsp/src/circuit/checkpointer.rs b/crates/dbsp/src/circuit/checkpointer.rs index 04a3a255dba..36f0167d042 100644 --- a/crates/dbsp/src/circuit/checkpointer.rs +++ b/crates/dbsp/src/circuit/checkpointer.rs @@ -1,6 +1,7 @@ //! Logic to manage persistent checkpoints for a circuit. use crate::dynamic::{self, data::DataTyped}; +use crate::storage::dirlock::LockedDirectory; use crate::storage::file::SerializerInner; use crate::{Error, NumEntries, TypedBox}; use enum_map::{Enum, EnumMap}; @@ -120,6 +121,7 @@ impl Checkpointer { in_use_paths.insert(format!("{}.mut", STATUS_FILE).into(), SmallVec::new()); in_use_paths.insert(DATAFUSION_TEMP_DIR.into(), SmallVec::new()); in_use_paths.insert(ACTIVATION_MARKER_FILE.into(), SmallVec::new()); + in_use_paths.insert(LockedDirectory::LOCKFILE_NAME.into(), SmallVec::new()); for (checkpoint_index, cpm) in self.checkpoint_list.iter().enumerate() { in_use_paths .entry(cpm.uuid.to_string().into()) diff --git a/crates/dbsp/src/storage/dirlock.rs b/crates/dbsp/src/storage/dirlock.rs index 16091e4f9db..70a392359ad 100644 --- a/crates/dbsp/src/storage/dirlock.rs +++ b/crates/dbsp/src/storage/dirlock.rs @@ -92,7 +92,7 @@ fn get_lock(file: &File) -> Result, IoError> { } impl LockedDirectory { - const LOCKFILE_NAME: &'static str = "feldera.pidlock"; + pub const LOCKFILE_NAME: &'static str = "feldera.pidlock"; /// Attempts to create a new pidfile in the `base_path` directory, returning /// an error if the file was already created by a different process (and From 0d4fd2ea51113acbd0a7735b86b9ddee1dc6e070 Mon Sep 17 00:00:00 2001 From: Leonid Ryzhyk Date: Mon, 13 Jul 2026 19:02:27 -0700 Subject: [PATCH 7/9] [adapters] Widen test HTTP client request timeout to avoid CI flakes TestHttpSender relied on awc's 5s default request timeout, which is shorter than the test harness's own 20s async_wait/completion budgets. Under CI load, or a k8s CPU-quota freeze that stalls the in-process server past 5s, the client deadline fires and panics an otherwise healthy request (observed as test_silent_bootstrap failing with `Err(Timeout)` at test/http.rs). The /ingress POST is non-idempotent, so retrying on timeout could double-insert records; instead widen the client deadline to 120s and let the higher-level waits govern liveness. Signed-off-by: Leonid Ryzhyk --- crates/adapters/src/test/http.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crates/adapters/src/test/http.rs b/crates/adapters/src/test/http.rs index 73322882059..9625f998aa1 100644 --- a/crates/adapters/src/test/http.rs +++ b/crates/adapters/src/test/http.rs @@ -8,8 +8,19 @@ use csv::ReaderBuilder as CsvReaderBuilder; use csv::WriterBuilder as CsvWriterBuilder; use futures::{Stream, StreamExt}; use serde::Deserialize; +use std::time::Duration; use tracing::trace; +/// Per-request timeout for the test HTTP client. +/// +/// awc defaults to a 5s response timeout, shorter than the harness's own 20s +/// `async_wait`/completion budgets. Under CI load (or a k8s CPU-quota freeze) +/// the in-process server can stall past 5s and trip that deadline, panicking an +/// otherwise-healthy request. The `/ingress` POST is non-idempotent and cannot +/// be retried safely, so we widen the client deadline and let the higher-level +/// waits govern liveness. +const REQUEST_TIMEOUT: Duration = Duration::from_secs(120); + pub struct TestHttpSender; pub struct TestHttpReceiver; @@ -20,6 +31,7 @@ impl TestHttpSender { let data = data.to_vec(); let mut response = req + .timeout(REQUEST_TIMEOUT) .send_stream(stream! { for batch in data.iter() { let mut writer = CsvWriterBuilder::new() From ee7af2a2c9b70bac5611d70bd9f618b91ff56be9 Mon Sep 17 00:00:00 2001 From: Leonid Ryzhyk Date: Fri, 3 Jul 2026 03:00:38 -0400 Subject: [PATCH 8/9] [dbsp] Publish accumulated outputs atomically across workers The AccumulateOutput operator produces a single output per transaction per worker. The idea was that this would generate a single output batch per transaction for each output connector, giving the connector an opportunity to commit these changes in a single transaction to the data sink (e.g., the Delta Lake connector does that). However different workers can produce their per-transaction outputs in different steps, resulting in multiple non-empty outputs from the circuit. This was unlikely to happen before the introduction of streaming exchange, since the upstream `shard` operator contains a barrier that synchronizes transaction completion across all threads. This commit introduces a local (per-host) barrier that synchronizes transaction output across all workers. There is an alternative approach: allow workers to produce outputs at any step, but only handle them in the controller when the transaction commits. This may be an even cleaner solution. One advantage of my fix is that it allows views that are computed early to produce outputs before all other views are fully evaluated in the transaction. Some users rely on this behavior. We may want to revise this in the future, but for now I wanted to implement the least intrusive fix. Signed-off-by: Leonid Ryzhyk --- crates/dbsp/src/operator/output.rs | 251 ++++++++++++++++++++++++++++- 1 file changed, 245 insertions(+), 6 deletions(-) diff --git a/crates/dbsp/src/operator/output.rs b/crates/dbsp/src/operator/output.rs index ccdb5d6f119..98436ad8ef1 100644 --- a/crates/dbsp/src/operator/output.rs +++ b/crates/dbsp/src/operator/output.rs @@ -21,7 +21,8 @@ use std::{ hash::{Hash, Hasher}, marker::PhantomData, mem::transmute, - sync::Arc, + ops::Range, + sync::{Arc, Mutex}, }; use typedmap::TypedMapKey; @@ -200,6 +201,97 @@ where type Value = OutputHandle; } +/// `TypedMapKey` entry used to share the pending-cohort slots of an +/// [`AccumulateOutput`] operator across the workers of one host. +struct CohortId { + id: usize, + // `fn() -> T` keeps the key `Send + Sync` regardless of `T`: the key + // names a type but never holds a value of it. + _marker: PhantomData T>, +} + +// Implement `Hash`, `Eq` manually to avoid `T: Hash` type bound. +impl Hash for CohortId { + fn hash(&self, state: &mut H) + where + H: Hasher, + { + self.id.hash(state); + } +} + +impl PartialEq for CohortId { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + } +} + +impl Eq for CohortId {} + +impl CohortId { + fn new(id: usize) -> Self { + Self { + id, + _marker: PhantomData, + } + } +} + +impl TypedMapKey for CohortId +where + T: 'static, +{ + type Value = CohortSlots; +} + +/// Pending per-worker outputs for the transaction in progress. +/// +/// Slot `i` stores the output of the host's `i`-th worker until every worker +/// on this host has produced its output for the transaction. Remote workers +/// have no slots: they cannot reach this host's store and publish through +/// their own host's instance. +struct CohortSlots(Arc>>>); + +impl Clone for CohortSlots { + fn clone(&self) -> Self { + Self(self.0.clone()) + } +} + +impl CohortSlots { + fn new(num_workers: usize) -> Self { + assert_ne!(num_workers, 0); + Self(Arc::new(Mutex::new( + (0..num_workers).map(|_| None).collect(), + ))) + } + + /// Store `value` in `slot` as its worker's output for the transaction in + /// progress. + /// + /// Returns the complete cohort, one value per slot in slot order, when + /// `value` fills the last empty slot; returns `None` otherwise. + fn put(&self, slot: usize, value: T) -> Option> { + let mut pending = self.0.lock().unwrap(); + + // The upstream accumulator emits exactly once per transaction per + // worker, so the slot must be free. + debug_assert!(pending[slot].is_none()); + pending[slot] = Some(value); + + if pending.iter().all(Option::is_some) { + Some( + pending + .iter_mut() + .map(|slot| slot.take().unwrap()) + .collect(), + ) + } else { + None + } + } +} + struct OutputHandleInternal { mailbox: Vec>>, } @@ -479,12 +571,34 @@ where } } +/// Sink operator that publishes an accumulated stream through an +/// [`OutputHandle`]. +/// +/// Every worker's accumulator emits exactly once per transaction, but when a +/// transaction commit spans several steps, workers can emit in different +/// steps. Writing each emission to its mailbox immediately would let a reader +/// observe a partial cross-worker output for a transaction. Instead, the +/// operator parks emissions in [`CohortSlots`] shared by the workers of one +/// host and publishes the whole cohort to the mailboxes when the last of +/// these workers emits, so a reader sees either all of a transaction's +/// outputs or none. +/// +/// In a multihost runtime, the cohort covers one host: each host's workers +/// share host-local [`CohortSlots`] and publish to host-local mailboxes, +/// which remote workers cannot reach. A reader observes a host's outputs +/// atomically; distinct hosts publish independently. pub struct AccumulateOutput where B: Batch, { global_id: GlobalNodeId, - mailbox: Mailbox>>, + handle: OutputHandle>, + /// Global indices of this host's workers: cohort slot `i` belongs to + /// worker `local_workers.start + i`. + local_workers: Range, + /// This worker's cohort slot: its offset within `local_workers`. + slot: usize, + pending: CohortSlots>, output_batch_stats: BatchSizeStats, } @@ -494,17 +608,51 @@ where { pub fn new() -> (Self, OutputHandle>) { let handle = OutputHandle::new(); - let mailbox = handle.mailbox(Runtime::worker_index()).clone(); + + // The slots live in the host-local store, out of reach of remote + // workers, so the cohort covers only this host's workers. + let (local_workers, pending) = match Runtime::runtime() { + None => (0..1, CohortSlots::new(1)), + Some(runtime) => { + let local_workers = runtime.layout().local_workers(); + let cohort_id = runtime.sequence_next(); + + let pending = runtime + .local_store() + .entry(CohortId::new(cohort_id)) + .or_insert_with(|| CohortSlots::new(local_workers.len())) + .value() + .clone(); + + (local_workers, pending) + } + }; let output = Self { global_id: GlobalNodeId::root(), - mailbox, + handle: handle.clone(), + local_workers, + slot: Runtime::local_worker_offset(), + pending, output_batch_stats: BatchSizeStats::new(), }; (output, handle) } + /// Write this worker's output; publish the cohort if it is now complete. + /// + /// Only the worker that completes a cohort publishes it, so there is + /// never more than one publisher per cohort. Mailboxes are indexed by + /// global worker index, slots by local offset. + fn put_and_publish(&self, snapshot: SpineSnapshot) { + if let Some(cohort) = self.pending.put(self.slot, snapshot) { + for (worker, snapshot) in self.local_workers.clone().zip(cohort) { + self.handle.mailbox(worker).set(Some(snapshot)); + } + } + } + fn checkpoint_file(base: &StoragePath, persistent_id: &str) -> StoragePath { base.child(format!("accumulate-output-{}.dat", persistent_id)) } @@ -567,14 +715,16 @@ where async fn eval(&mut self, val: &Option>) { if let Some(val) = val { self.output_batch_stats.add_batch(val.len()); - self.mailbox.set(Some(val.ro_snapshot())); + // Park even empty outputs: cohort completion requires one + // emission per worker per transaction. + self.put_and_publish(val.ro_snapshot()); } } async fn eval_owned(&mut self, val: Option>) { if let Some(val) = val { self.output_batch_stats.add_batch(val.len()); - self.mailbox.set(Some(val.ro_snapshot())); + self.put_and_publish(val.ro_snapshot()); } } @@ -687,8 +837,97 @@ where #[cfg(test)] mod test { + use super::CohortSlots; use crate::{Runtime, typed_batch::OrdZSet, utils::Tup2}; + /// Parking releases a cohort only when every worker has contributed, and + /// the slots are reusable for the next cohort. + #[test] + fn test_cohort_slots() { + let slots = CohortSlots::new(3); + + // Partial cohort: nothing is released. + assert_eq!(slots.put(0, "a0"), None); + assert_eq!(slots.put(2, "a2"), None); + + // The last worker's output releases the whole cohort. + assert_eq!(slots.put(1, "a1"), Some(vec!["a0", "a1", "a2"])); + + // The slots are empty again and serve the next cohort. + assert_eq!(slots.put(1, "b1"), None); + assert_eq!(slots.put(2, "b2"), None); + assert_eq!(slots.put(0, "b0"), Some(vec!["b0", "b1", "b2"])); + } + + /// End-to-end: the accumulated output surfaces exactly once per + /// transaction, as a complete cohort with one output per worker, and + /// never mid-transaction or as a partial cohort mid-commit. + /// + /// Drives transactions with the manual `start_transaction` -> + /// `start_commit_transaction` -> `step` API so the handle can be + /// inspected after every step, including each step of a multi-step + /// commit; `DBSPHandle::transaction` would only return after the commit + /// completed, when the cohort is trivially complete. + #[test] + fn test_accumulate_output() { + use crate::trace::BatchReader; + + const WORKERS: usize = 8; + + let (mut dbsp, (input, output)) = Runtime::init_circuit(WORKERS, |circuit| { + let (zset, zset_handle) = circuit.add_input_zset::(); + let output = zset.accumulate_output(); + + Ok((zset_handle, output)) + }) + .unwrap(); + + for round in 0..20u64 { + // Odd rounds run an empty transaction, which must still publish + // a complete, empty cohort. + let expected = if round % 2 == 0 { 20_000 } else { 0 }; + + dbsp.start_transaction().unwrap(); + + if expected > 0 { + let mut tuples = (round * 100_000..round * 100_000 + 20_000) + .map(|k| Tup2(k, 1i64)) + .collect::>(); + input.append(&mut tuples); + } + + // Steps inside an open transaction publish nothing. + dbsp.step().unwrap(); + assert!(output.take_from_all().is_empty()); + dbsp.step().unwrap(); + assert!(output.take_from_all().is_empty()); + + dbsp.start_commit_transaction().unwrap(); + + // Every commit step yields either nothing or one complete + // cohort, and the whole transaction yields exactly one. + let mut cohorts = 0; + loop { + let committed = dbsp.step().unwrap(); + + let batches = output.take_from_all(); + if !batches.is_empty() { + assert_eq!(batches.len(), WORKERS); + assert_eq!(batches.iter().map(|b| b.len()).sum::(), expected); + cohorts += 1; + } + + if committed { + break; + } + } + assert_eq!(cohorts, 1); + assert!(output.take_from_all().is_empty()); + } + + dbsp.kill().unwrap(); + } + #[test] fn test_output_handle() { let (mut dbsp, (input, output)) = Runtime::init_circuit(4, |circuit| { From 975192311b2c5f5e949b3bd142d6c8ce78c0abb2 Mon Sep 17 00:00:00 2001 From: Simon Kassing Date: Mon, 13 Jul 2026 11:25:15 +0200 Subject: [PATCH 9/9] pipeline-manager: use runtime status details for connector stats Instead of the manager polling all pipelines every GET request for its list when using the `status_with_connectors` selector, instead incorporate the connector stats into the runtime status details. The details are regularly updated by the runner as it does its `/status` checks. As such, it will take some time for the latest connector stats to show up after this change (approximately between 1-15 seconds). At the cost of this slight staleness, the latency of the GET request for the list of pipelines with that selector will be significantly reduced (up to 10x faster is observed). `StorageStatusDetails` is now the type shown in the API for the `storage_status_details`, instead of the generic JSON value type. The same for `RuntimeStatusDetails`. Backward incompatibility notes: - Pipelines of older versions will continue to have their connector stats shown as before. After the user base has migrated past this change, the GET request fetching fallback can be removed. - Runtime status details were directly used for the approval diff. Now, if this is present (during the `AwaitingApproval` runtime status), it is located at: `deployment_runtime_status_details.approval_diff`. The API converts automatically the old JSON values into the JSON matching the strongly typed `RuntimeStatusDetails`. The Web Console and `fda` clients are adjusted to read the approval diff from the new nested location. Co-authored-by: Karakatiza666 Signed-off-by: Karakatiza666 Signed-off-by: Simon Kassing --- crates/adapters/src/server.rs | 72 +++-- crates/fda/src/main.rs | 9 +- crates/feldera-types/src/runtime_status.rs | 67 ++++- .../src/api/endpoints/pipeline_management.rs | 174 ++++++++--- crates/pipeline-manager/src/api/examples.rs | 21 +- crates/pipeline-manager/src/api/main.rs | 4 +- crates/pipeline-manager/src/db/test.rs | 22 +- .../src/runner/pipeline_automata.rs | 39 ++- docs.feldera.com/docs/changelog.md | 13 +- .../components/pipelines/Table.svelte.spec.ts | 1 + .../src/lib/compositions/duplicatePipeline.ts | 1 + .../functions/pipelines/pipelineDiff.spec.ts | 81 ++++++ .../lib/functions/pipelines/pipelineDiff.ts | 8 +- .../src/lib/services/manager/index.ts | 13 + .../src/lib/services/manager/sdk.gen.ts | 29 ++ .../src/lib/services/manager/types.gen.ts | 275 ++++++++++++++++-- openapi.json | 63 +++- python/tests/platform/test_bootstrapping.py | 14 +- .../tests/platform/test_pipeline_lifecycle.py | 34 +++ 19 files changed, 797 insertions(+), 143 deletions(-) create mode 100644 js-packages/web-console/src/lib/functions/pipelines/pipelineDiff.spec.ts diff --git a/crates/adapters/src/server.rs b/crates/adapters/src/server.rs index e6f821fcfa5..4d119019216 100644 --- a/crates/adapters/src/server.rs +++ b/crates/adapters/src/server.rs @@ -10,7 +10,7 @@ use crate::server::metrics::{ use crate::static_compile::catalog::OUTPUT_MAPPING; use crate::transport::http::HttpOutputFormat; use crate::util::{LongOperationWarning, RateLimitCheckResult, TokenBucketRateLimiter}; -use crate::{Catalog, dyn_event}; +use crate::{Catalog, ControllerStatus, dyn_event}; use crate::{ CircuitCatalog, Controller, ControllerError, FormatConfig, InputEndpointConfig, OutputEndpoint, OutputEndpointConfig, PipelineConfig, TransportInputEndpoint, @@ -74,8 +74,9 @@ use feldera_types::query_params::{ SamplyProfileParams, }; use feldera_types::runtime_status::{ - BootstrapConfig, BootstrapPolicy, ExtendedRuntimeStatus, ExtendedRuntimeStatusError, - RuntimeDesiredStatus, RuntimeStatus, StorageStatusDetails, + BootstrapConfig, BootstrapPolicy, ConnectorStats, ExtendedRuntimeStatus, + ExtendedRuntimeStatusError, RuntimeDesiredStatus, RuntimeStatus, RuntimeStatusDetails, + StorageStatusDetails, }; use feldera_types::suspend::{SuspendError, SuspendableResponse}; use feldera_types::time_series::TimeSeries; @@ -1469,7 +1470,7 @@ fn get_status(state: &ServerState) -> Result match Checkpointer::read_checkpoints(&**backend) { Ok(list_checkpoints) => Some(StorageStatusDetails { - checkpoints: list_checkpoints, + checkpoints: list_checkpoints.into(), }), Err(e) => { error!( @@ -1490,6 +1491,23 @@ fn get_status(state: &ServerState) -> Result, ) -> ExtendedRuntimeStatus { + // Error statistics across connectors + let (inputs, outputs) = retrieve_error_stats(controller.status()); + let mut total_errors = 0u64; + for endpoint in inputs { + total_errors = total_errors + .saturating_add(endpoint.metrics.num_transport_errors) + .saturating_add(endpoint.metrics.num_parse_errors); + } + for endpoint in outputs { + total_errors = total_errors + .saturating_add(endpoint.metrics.num_encode_errors) + .saturating_add(endpoint.metrics.num_transport_errors); + } + let connector_stats = ConnectorStats { + num_errors: total_errors, + }; + ExtendedRuntimeStatus { runtime_status: if controller.status().bootstrap_in_progress() { RuntimeStatus::Bootstrapping @@ -1498,7 +1516,11 @@ fn get_status(state: &ServerState) -> Result Result match inner { InitializationState::Starting => Ok(ExtendedRuntimeStatus { runtime_status: RuntimeStatus::Initializing, - runtime_status_details: json!(""), + runtime_status_details: RuntimeStatusDetails::default().serialize_guaranteed(), runtime_desired_status, storage_status_details, }), InitializationState::DownloadingCheckpoint => Ok(ExtendedRuntimeStatus { runtime_status: RuntimeStatus::Initializing, - runtime_status_details: json!("downloading checkpoint from object storage"), + runtime_status_details: RuntimeStatusDetails::new_only_reason( + "downloading checkpoint from object storage", + ) + .serialize_guaranteed(), runtime_desired_status, storage_status_details, }), InitializationState::Standby => Ok(ExtendedRuntimeStatus { runtime_status: RuntimeStatus::Standby, - runtime_status_details: json!(""), + runtime_status_details: RuntimeStatusDetails::default().serialize_guaranteed(), runtime_desired_status, storage_status_details, }), InitializationState::AwaitingApproval(diff) => Ok(ExtendedRuntimeStatus { runtime_status: RuntimeStatus::AwaitingApproval, - runtime_status_details: serde_json::to_value(&diff).unwrap_or_default(), + runtime_status_details: RuntimeStatusDetails { + approval_diff: Some(serde_json::to_value(&diff).unwrap_or_default()), + ..Default::default() + } + .serialize_guaranteed(), runtime_desired_status, storage_status_details, }), @@ -1582,7 +1611,7 @@ fn get_status(state: &ServerState) -> Result Result Ok(ExtendedRuntimeStatus { runtime_status: RuntimeStatus::Suspended, - runtime_status_details: json!(""), + runtime_status_details: RuntimeStatusDetails::default().serialize_guaranteed(), runtime_desired_status, storage_status_details, }), @@ -1678,12 +1707,13 @@ async fn stats( Ok(HttpResponse::Ok().json(state.controller()?.api_status(include_connector_errors))) } -/// This endpoint returns a subset of stats that don't need updating and so is more performant than /stats -#[get("/stats/errors")] -async fn error_stats(state: WebData) -> Result { - let controller = state.controller()?; - let controller_status = controller.status(); - +/// Retrieves the error statistics across all endpoints. +fn retrieve_error_stats( + controller_status: &ControllerStatus, +) -> ( + Vec>, + Vec>, +) { let inputs: Vec> = controller_status .input_status() .values() @@ -1707,7 +1737,15 @@ async fn error_stats(state: WebData) -> Result) -> Result { + let controller = state.controller()?; + let controller_status = controller.status(); + let (inputs, outputs) = retrieve_error_stats(controller_status); Ok(HttpResponse::Ok().json(PipelineStatsErrorsResponse { inputs, outputs })) } diff --git a/crates/fda/src/main.rs b/crates/fda/src/main.rs index ad62757f00c..0fc4f4048db 100644 --- a/crates/fda/src/main.rs +++ b/crates/fda/src/main.rs @@ -1076,7 +1076,7 @@ async fn pipeline(format: OutputFormat, action: PipelineAction, client: Client) ) .await; if status == CombinedStatus::AwaitingApproval { - let diff = client + let runtime_status_details = client .get_pipeline() .pipeline_name(name.clone()) .selector(PipelineFieldSelector::Status) @@ -1095,10 +1095,11 @@ async fn pipeline(format: OutputFormat, action: PipelineAction, client: Client) "Pipeline definition has changed since the last checkpoint. The pipeline is awaiting approval to proceed with bootstrapping the modified components. Run 'fda approve' to approve the changes or 'fda stop' to terminate the pipeline. Summary of changes:" ); - let diff_str = match diff { + let diff_str = match runtime_status_details.map(|details| details.approval_diff) + { // Normally shouldn't happen. - None => "".to_string(), - Some(diff) => match format { + None | Some(None) => "".to_string(), + Some(Some(diff)) => match format { OutputFormat::Text => json_to_table(&diff) .collapse() .into_pool_table() diff --git a/crates/feldera-types/src/runtime_status.rs b/crates/feldera-types/src/runtime_status.rs index d83ad2734b6..24b3c0d9880 100644 --- a/crates/feldera-types/src/runtime_status.rs +++ b/crates/feldera-types/src/runtime_status.rs @@ -6,7 +6,7 @@ use actix_web::{HttpRequest, HttpResponse, HttpResponseBuilder, Responder, Respo use bytemuck::NoUninit; use clap::ValueEnum; use serde::{Deserialize, Serialize}; -use std::collections::VecDeque; +use serde_json::json; use std::fmt; use std::fmt::Display; use utoipa::ToSchema; @@ -260,10 +260,10 @@ impl BootstrapConfig { /// Details about pipeline storage, which are returned as part of the regular runtime status polling /// by the runner. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, ToSchema)] pub struct StorageStatusDetails { /// Present checkpoints. - pub checkpoints: VecDeque, + pub checkpoints: Vec, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -348,3 +348,64 @@ impl ResponseError for ExtendedRuntimeStatusError { HttpResponseBuilder::new(self.status_code()).json(self.error.clone()) } } + +/// Details about the current runtime status. The fields in this struct should all be **optional** +/// and set only by a runtime status when they are known. Otherwise, they can just be set `None`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Default, Eq, ToSchema)] +pub struct RuntimeStatusDetails { + /// Free form text giving an explanation why it is currently in this runtime status. + /// + /// Specifically useful for: `Unavailable`, `Initializing`. + #[serde(skip_serializing_if = "Option::is_none")] + pub reason: Option, + + /// Statistics across all connectors. + /// + /// Specifically useful for: `Paused`, `Running`. + #[serde(skip_serializing_if = "Option::is_none")] + pub connector_stats: Option, + + /// The diff which is awaiting approval. + /// + /// Specifically useful for: `AwaitingApproval`. + #[serde(skip_serializing_if = "Option::is_none")] + pub approval_diff: Option, + // Backward compatibility: in older versions, the approval diff was the runtime status details + // value itself. To distinguish between the old and new version, clients can check if the + // `program_diff` field (one of the fields within the `approval_diff`) is present if they + // expect there to be a diff (i.e., when the runtime status is `AwaitingApproval`). As such, + // `program_diff` is a reserved field name that cannot be added here in the future. +} + +impl RuntimeStatusDetails { + pub fn new_only_reason(reason: &str) -> Self { + Self { + reason: Some(reason.to_string()), + ..Self::default() + } + } + + /// Serializes the runtime status details to JSON. If the serialization errors, an error JSON + /// is returned instead. This makes sure this method does not panic unexpectedly and that the + /// error bubbles up. The details are only supplementary information, and as such are not + /// critical to operation. + pub fn serialize_guaranteed(self) -> serde_json::Value { + serde_json::to_value(self).unwrap_or_else(|e| { + json!({ + "reason": format!("unable to serialize runtime status details due to: {e}") + }) + }) + } +} + +/// Statistics across all connectors. +#[derive(Serialize, Deserialize, ToSchema, Eq, PartialEq, Debug, Clone)] +pub struct ConnectorStats { + /// Total number of errors across all connectors. + /// + /// - `num_transport_errors` from all input connectors + /// - `num_parse_errors` from all input connectors + /// - `num_encode_errors` from all output connectors + /// - `num_transport_errors` from all output connectors + pub num_errors: u64, +} diff --git a/crates/pipeline-manager/src/api/endpoints/pipeline_management.rs b/crates/pipeline-manager/src/api/endpoints/pipeline_management.rs index ee66d6bf2a9..f2ae451e57b 100644 --- a/crates/pipeline-manager/src/api/endpoints/pipeline_management.rs +++ b/crates/pipeline-manager/src/api/endpoints/pipeline_management.rs @@ -32,7 +32,8 @@ use feldera_types::config::{InputEndpointConfig, OutputEndpointConfig, RuntimeCo use feldera_types::error::ErrorResponse; use feldera_types::program_schema::ProgramSchema; use feldera_types::runtime_status::{ - BootstrapConfig, BootstrapPolicy, RuntimeDesiredStatus, RuntimeStatus, + BootstrapConfig, BootstrapPolicy, ConnectorStats, RuntimeDesiredStatus, RuntimeStatus, + RuntimeStatusDetails, StorageStatusDetails, }; use futures_util::future::join_all; use serde::{Deserialize, Serialize}; @@ -74,22 +75,6 @@ fn remove_large_fields_from_program_info( program_info } -/// Aggregated connector error statistics. -/// -/// This structure contains the sum of all error counts across all input and output connectors -/// for a pipeline. -#[derive(Serialize, Deserialize, ToSchema, Eq, PartialEq, Debug, Clone)] -pub struct ConnectorStats { - /// Total number of errors across all connectors. - /// - /// This is the sum of: - /// - `num_transport_errors` from all input connectors - /// - `num_parse_errors` from all input connectors - /// - `num_encode_errors` from all output connectors - /// - `num_transport_errors` from all output connectors - pub num_errors: u64, -} - /// Pipeline information. /// It both includes fields which are user-provided and system-generated. #[derive(Serialize, ToSchema, PartialEq, Debug, Clone)] @@ -116,7 +101,7 @@ pub struct PipelineInfo { pub deployment_error: Option, pub refresh_version: Version, pub storage_status: StorageStatus, - pub storage_status_details: Option, + pub storage_status_details: Option, pub deployment_id: Option, pub deployment_initial: Option, pub deployment_status: CombinedStatus, @@ -129,7 +114,7 @@ pub struct PipelineInfo { pub deployment_resources_desired_status: ResourcesDesiredStatus, pub deployment_resources_desired_status_since: DateTime, pub deployment_runtime_status: Option, - pub deployment_runtime_status_details: Option, + pub deployment_runtime_status_details: Option, pub deployment_runtime_status_since: Option>, pub deployment_runtime_desired_status: Option, pub deployment_runtime_desired_status_since: Option>, @@ -139,8 +124,9 @@ pub struct PipelineInfo { /// /// This is the struct that is actually serialized when a response body type /// is [`PipelineInfo`] according to the OpenAPI specification. -/// The difference are the types of `runtime_config`, `program_config` and -/// `program_info` fields, which are JSON values rather than their actual ones. +/// The difference are the types of `runtime_config`, `program_config`, +/// `program_info`, `storage_status_details` and `deployment_runtime_status_details` fields, +/// which are JSON values rather than their actual ones. /// This ensures that even when a backward incompatible change occurred for /// any of these fields, the API still works (i.e., able to serialize them in /// order to return pipeline(s)). @@ -237,7 +223,9 @@ impl PipelineInfoInternal { deployment_resources_desired_status_since: extended_pipeline .deployment_resources_desired_status_since, deployment_runtime_status: extended_pipeline.deployment_runtime_status, - deployment_runtime_status_details: extended_pipeline.deployment_runtime_status_details, + deployment_runtime_status_details: backward_compatible_runtime_status_details( + extended_pipeline.deployment_runtime_status_details, + ), deployment_runtime_status_since: extended_pipeline.deployment_runtime_status_since, deployment_runtime_desired_status: extended_pipeline.deployment_runtime_desired_status, deployment_runtime_desired_status_since: extended_pipeline @@ -280,7 +268,7 @@ pub struct PipelineSelectedInfo { pub deployment_error: Option, pub refresh_version: Version, pub storage_status: StorageStatus, - pub storage_status_details: Option, + pub storage_status_details: Option, pub deployment_id: Option, pub deployment_initial: Option, pub deployment_status: CombinedStatus, @@ -294,7 +282,7 @@ pub struct PipelineSelectedInfo { pub deployment_resources_desired_status: ResourcesDesiredStatus, pub deployment_resources_desired_status_since: DateTime, pub deployment_runtime_status: Option, - pub deployment_runtime_status_details: Option, + pub deployment_runtime_status_details: Option, pub deployment_runtime_status_since: Option>, pub deployment_runtime_desired_status: Option, pub deployment_runtime_desired_status_since: Option>, @@ -417,7 +405,9 @@ impl PipelineSelectedInfoInternal { deployment_resources_desired_status_since: extended_pipeline .deployment_resources_desired_status_since, deployment_runtime_status: extended_pipeline.deployment_runtime_status, - deployment_runtime_status_details: extended_pipeline.deployment_runtime_status_details, + deployment_runtime_status_details: backward_compatible_runtime_status_details( + extended_pipeline.deployment_runtime_status_details, + ), deployment_runtime_status_since: extended_pipeline.deployment_runtime_status_since, deployment_runtime_desired_status: extended_pipeline.deployment_runtime_desired_status, deployment_runtime_desired_status_since: extended_pipeline @@ -477,7 +467,9 @@ impl PipelineSelectedInfoInternal { deployment_resources_desired_status_since: extended_pipeline .deployment_resources_desired_status_since, deployment_runtime_status: extended_pipeline.deployment_runtime_status, - deployment_runtime_status_details: extended_pipeline.deployment_runtime_status_details, + deployment_runtime_status_details: backward_compatible_runtime_status_details( + extended_pipeline.deployment_runtime_status_details, + ), deployment_runtime_status_since: extended_pipeline.deployment_runtime_status_since, deployment_runtime_desired_status: extended_pipeline.deployment_runtime_desired_status, deployment_runtime_desired_status_since: extended_pipeline @@ -739,6 +731,40 @@ pub struct PostStopPipelineParameters { force: bool, } +/// Converts old runtime status details to the new strongly typed one. +fn backward_compatible_runtime_status_details( + details: Option, +) -> Option { + details.map(|details| match details { + serde_json::Value::Null => RuntimeStatusDetails::default().serialize_guaranteed(), + serde_json::Value::Bool(b) => { + RuntimeStatusDetails::new_only_reason(&format!("Boolean: {b}")).serialize_guaranteed() + } + serde_json::Value::Number(n) => { + RuntimeStatusDetails::new_only_reason(&format!("Number: {n}")).serialize_guaranteed() + } + serde_json::Value::String(s) => { + RuntimeStatusDetails::new_only_reason(&s).serialize_guaranteed() + } + serde_json::Value::Array(a) => { + RuntimeStatusDetails::new_only_reason(&format!("Array: {a:?}")).serialize_guaranteed() + } + serde_json::Value::Object(obj) => { + if obj.get("program_diff").is_some() { + // Backward compatibility: the entire runtime status details is actually the approval + // diff, as such it must be restructured. + RuntimeStatusDetails { + approval_diff: Some(serde_json::Value::Object(obj)), + ..Default::default() + } + .serialize_guaranteed() + } else { + serde_json::Value::Object(obj) + } + } + }) +} + /// List Pipelines /// /// Retrieve the list of pipelines. @@ -798,13 +824,8 @@ pub(crate) async fn list_pipelines( let tenant_id = *tenant_id; let pipeline_name = pipeline.name.clone(); async move { - fetch_connector_error_stats( - &state, - tenant_id, - &pipeline_name, - pipeline.deployment_runtime_status, - ) - .await + fetch_connector_error_stats(&state, tenant_id, &pipeline_name, pipeline) + .await } }) .collect(); @@ -838,10 +859,25 @@ async fn fetch_connector_error_stats( state: &WebData, tenant_id: TenantId, pipeline_name: &str, - deployment_runtime_status: Option, + pipeline: &ExtendedPipelineDescrMonitoring, ) -> Option { + // First attempt to retrieve the connector statistics from the runtime status details if they + // are available there. The fetching afterward is added for backward compatibility. Once the + // runtime status details changes are sufficiently long present and the user base has migrated + // past it, then the HTTP fetching can be removed. + let details = backward_compatible_runtime_status_details( + pipeline.deployment_runtime_status_details.clone(), + ); + if let Some(value) = details { + if let Ok(details) = serde_json::from_value::(value) { + if let Some(connector_stats) = details.connector_stats { + return Some(connector_stats); + } + } + }; + // Only forward the request if the pipeline is in a valid runtime status - match deployment_runtime_status { + match pipeline.deployment_runtime_status { Some(RuntimeStatus::Bootstrapping) | Some(RuntimeStatus::Replaying) | Some(RuntimeStatus::Running) @@ -976,13 +1012,8 @@ pub(crate) async fn get_pipeline( .await .get_pipeline_for_monitoring(*tenant_id, &pipeline_name) .await?; - let connector_stats = fetch_connector_error_stats( - &state, - *tenant_id, - &pipeline_name, - pipeline.deployment_runtime_status, - ) - .await; + let connector_stats = + fetch_connector_error_stats(&state, *tenant_id, &pipeline_name, &pipeline).await; PipelineSelectedInfoInternal::new_status_with_connectors(pipeline, connector_stats) } }; @@ -1805,3 +1836,62 @@ pub(crate) async fn post_pipeline_testing( Ok(HttpResponse::Ok().finish()) } + +#[cfg(test)] +mod tests { + use crate::api::endpoints::pipeline_management::backward_compatible_runtime_status_details; + use feldera_types::runtime_status::{ConnectorStats, RuntimeStatusDetails}; + use serde_json::json; + + #[test] + fn test_backward_compatible_runtime_status_details() { + for (input, expected) in [ + (None, None), + (Some(json!(null)), Some(json!({}))), + ( + Some(json!(false)), + Some(json!({"reason": "Boolean: false"})), + ), + (Some(json!(123)), Some(json!({"reason": "Number: 123"}))), + (Some(json!("abc")), Some(json!({"reason": "abc"}))), + ( + Some(json!([1, 2, 3])), + Some(json!({"reason": "Array: [Number(1), Number(2), Number(3)]"})), + ), + (Some(json!({"abc": "def"})), Some(json!({"abc": "def"}))), + ( + Some(json!({"program_diff": "xyz"})), + Some(json!({"approval_diff": { "program_diff": "xyz" }})), + ), + ( + Some(RuntimeStatusDetails::default().serialize_guaranteed()), + Some(json!({})), + ), + ( + Some(RuntimeStatusDetails::new_only_reason("abc").serialize_guaranteed()), + Some(json!({"reason": "abc"})), + ), + ( + Some( + RuntimeStatusDetails { + reason: Some("abc".to_string()), + connector_stats: Some(ConnectorStats { num_errors: 123 }), + approval_diff: Some(json!({"a": "b"})), + } + .serialize_guaranteed(), + ), + Some(json!({ + "reason": "abc", + "connector_stats": { + "num_errors": 123 + }, + "approval_diff": { + "a": "b" + } + })), + ), + ] { + assert_eq!(backward_compatible_runtime_status_details(input), expected); + } + } +} diff --git a/crates/pipeline-manager/src/api/examples.rs b/crates/pipeline-manager/src/api/examples.rs index c965e7e44a6..7463b9e257d 100644 --- a/crates/pipeline-manager/src/api/examples.rs +++ b/crates/pipeline-manager/src/api/examples.rs @@ -23,6 +23,7 @@ use crate::runner::interaction::{ format_disconnected_error_message, format_timeout_error_message, RunnerInteraction, }; use feldera_types::config::{DevTweaks, FtConfig, ResourceConfig, StorageOptions}; +use feldera_types::runtime_status::{RuntimeStatusDetails, StorageStatusDetails}; use feldera_types::{config::RuntimeConfig, error::ErrorResponse}; use uuid::uuid; @@ -215,7 +216,10 @@ fn pipeline_info_internal_to_external(pipeline: PipelineInfoInternal) -> Pipelin deployment_error: pipeline.deployment_error, refresh_version: pipeline.refresh_version, storage_status: pipeline.storage_status, - storage_status_details: pipeline.storage_status_details, + storage_status_details: pipeline.storage_status_details.map(|v| { + serde_json::from_value::(v) + .expect("example should be deserializable") + }), deployment_id: pipeline.deployment_id, deployment_initial: pipeline.deployment_initial, deployment_status: pipeline.deployment_status, @@ -229,7 +233,10 @@ fn pipeline_info_internal_to_external(pipeline: PipelineInfoInternal) -> Pipelin deployment_resources_desired_status_since: pipeline .deployment_resources_desired_status_since, deployment_runtime_status: pipeline.deployment_runtime_status, - deployment_runtime_status_details: pipeline.deployment_runtime_status_details, + deployment_runtime_status_details: pipeline.deployment_runtime_status_details.map(|v| { + serde_json::from_value::(v) + .expect("example should be deserializable") + }), deployment_runtime_status_since: pipeline.deployment_runtime_status_since, deployment_runtime_desired_status: pipeline.deployment_runtime_desired_status, deployment_runtime_desired_status_since: pipeline.deployment_runtime_desired_status_since, @@ -278,7 +285,10 @@ fn pipeline_selected_info_internal_to_external( deployment_error: pipeline.deployment_error, refresh_version: pipeline.refresh_version, storage_status: pipeline.storage_status, - storage_status_details: pipeline.storage_status_details, + storage_status_details: pipeline.storage_status_details.map(|v| { + serde_json::from_value::(v) + .expect("example should be deserializable") + }), deployment_id: pipeline.deployment_id, deployment_initial: pipeline.deployment_initial, deployment_status: pipeline.deployment_status, @@ -292,7 +302,10 @@ fn pipeline_selected_info_internal_to_external( deployment_resources_desired_status_since: pipeline .deployment_resources_desired_status_since, deployment_runtime_status: pipeline.deployment_runtime_status, - deployment_runtime_status_details: pipeline.deployment_runtime_status_details, + deployment_runtime_status_details: pipeline.deployment_runtime_status_details.map(|v| { + serde_json::from_value::(v) + .expect("example should be deserializable") + }), deployment_runtime_status_since: pipeline.deployment_runtime_status_since, deployment_runtime_desired_status: pipeline.deployment_runtime_desired_status, deployment_runtime_desired_status_since: pipeline.deployment_runtime_desired_status_since, diff --git a/crates/pipeline-manager/src/api/main.rs b/crates/pipeline-manager/src/api/main.rs index 4121329c5c4..42e7f13c711 100644 --- a/crates/pipeline-manager/src/api/main.rs +++ b/crates/pipeline-manager/src/api/main.rs @@ -283,7 +283,9 @@ It contains the following fields: feldera_types::runtime_status::RuntimeStatus, feldera_types::runtime_status::RuntimeDesiredStatus, feldera_types::runtime_status::BootstrapPolicy, - crate::api::endpoints::pipeline_management::ConnectorStats, + feldera_types::runtime_status::RuntimeStatusDetails, + feldera_types::runtime_status::StorageStatusDetails, + feldera_types::runtime_status::ConnectorStats, crate::api::endpoints::pipeline_management::PipelineInfo, crate::api::endpoints::pipeline_management::PipelineSelectedInfo, crate::api::endpoints::pipeline_management::PipelineFieldSelector, diff --git a/crates/pipeline-manager/src/db/test.rs b/crates/pipeline-manager/src/db/test.rs index a094f2c74c2..fbbb8d59368 100644 --- a/crates/pipeline-manager/src/db/test.rs +++ b/crates/pipeline-manager/src/db/test.rs @@ -50,7 +50,7 @@ use proptest_derive::Arbitrary; use serde_json::json; use std::borrow::BorrowMut; use std::borrow::Cow; -use std::collections::{BTreeMap, VecDeque}; +use std::collections::BTreeMap; use std::fmt::Debug; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -617,8 +617,8 @@ fn limited_optional_storage_status_details() -> impl Strategy PipelineAutomaton { deployment_resources_status_details: details, extended_runtime_status: ExtendedRuntimeStatus { runtime_status: RuntimeStatus::Initializing, - runtime_status_details: json!(""), + runtime_status_details: RuntimeStatusDetails::new_only_reason( + "initializing set by the runner", + ) + .serialize_guaranteed(), runtime_desired_status: deployment_initial, storage_status_details: None, }, @@ -1506,19 +1510,19 @@ impl PipelineAutomaton { match response_str { "Paused" => Ok(ExtendedRuntimeStatus { runtime_status: RuntimeStatus::Paused, - runtime_status_details: json!(""), + runtime_status_details: RuntimeStatusDetails::default().serialize_guaranteed(), runtime_desired_status: RuntimeDesiredStatus::Paused, storage_status_details: None, }), "Running" => Ok(ExtendedRuntimeStatus { runtime_status: RuntimeStatus::Running, - runtime_status_details: json!(""), + runtime_status_details: RuntimeStatusDetails::default().serialize_guaranteed(), runtime_desired_status: RuntimeDesiredStatus::Running, storage_status_details: None, }), "Initializing" => Ok(ExtendedRuntimeStatus { // Backward compatibility: in anticipation of recent change of 503 to 200 runtime_status: RuntimeStatus::Initializing, - runtime_status_details: json!(""), + runtime_status_details: RuntimeStatusDetails::default().serialize_guaranteed(), runtime_desired_status: RuntimeDesiredStatus::Paused, storage_status_details: None, }), @@ -1528,7 +1532,7 @@ impl PipelineAutomaton { })), _ => Ok(ExtendedRuntimeStatus { runtime_status: RuntimeStatus::Unavailable, - runtime_status_details: json!(format!("Pipeline status response (200 OK) is an unexpected JSON string: '{response_str}'")), + runtime_status_details: RuntimeStatusDetails::new_only_reason(&format!("Pipeline status response (200 OK) is an unexpected JSON string: '{response_str}'")).serialize_guaranteed(), runtime_desired_status: RuntimeDesiredStatus::Unavailable, storage_status_details: None, }), @@ -1539,7 +1543,7 @@ impl PipelineAutomaton { Ok(response) => Ok(response), Err(e) => Ok(ExtendedRuntimeStatus { runtime_status: RuntimeStatus::Unavailable, - runtime_status_details: json!(format!("Pipeline status response (200 OK) cannot be deserialized due to: {e}")), + runtime_status_details: RuntimeStatusDetails::new_only_reason(&format!("Pipeline status response (200 OK) cannot be deserialized due to: {e}")).serialize_guaranteed(), runtime_desired_status: RuntimeDesiredStatus::Unavailable, storage_status_details: None, }), @@ -1548,7 +1552,7 @@ impl PipelineAutomaton { // JSON response must be either a string or an object. Ok(ExtendedRuntimeStatus { runtime_status: RuntimeStatus::Unavailable, - runtime_status_details: json!(format!("Pipeline status response (200 OK) is not a string or an object:\n{body:#}")), + runtime_status_details: RuntimeStatusDetails::new_only_reason(&format!("Pipeline status response (200 OK) is not a string or an object:\n{body:#}")).serialize_guaranteed(), runtime_desired_status: RuntimeDesiredStatus::Unavailable, storage_status_details: None, }) @@ -1559,13 +1563,13 @@ impl PipelineAutomaton { match error_response.error_code.as_ref() { "Initializing" => Ok(ExtendedRuntimeStatus { // For backward compatibility runtime_status: RuntimeStatus::Initializing, - runtime_status_details: json!(""), + runtime_status_details: RuntimeStatusDetails::default().serialize_guaranteed(), runtime_desired_status: RuntimeDesiredStatus::Paused, storage_status_details: None, }), "Suspended" => Ok(ExtendedRuntimeStatus { // For backward compatibility runtime_status: RuntimeStatus::Suspended, - runtime_status_details: json!(""), + runtime_status_details: RuntimeStatusDetails::default().serialize_guaranteed(), runtime_desired_status: RuntimeDesiredStatus::Suspended, storage_status_details: None, }), @@ -1577,16 +1581,16 @@ impl PipelineAutomaton { ); Ok(ExtendedRuntimeStatus { runtime_status: RuntimeStatus::Unavailable, - runtime_status_details: json!(format!("Pipeline status response (503 Service Unavailable) is an error:\n{error_response:?}")), + runtime_status_details: RuntimeStatusDetails::new_only_reason(&format!("Pipeline status response (503 Service Unavailable) is an error:\n{error_response:?}")).serialize_guaranteed(), runtime_desired_status: RuntimeDesiredStatus::Unavailable, - storage_status_details: None, + storage_status_details: None, }) }, } } Err(e) => Ok(ExtendedRuntimeStatus { runtime_status: RuntimeStatus::Unavailable, - runtime_status_details: json!(format!("Pipeline status response (503 Service Unavailable) cannot be deserialized due to: {e}. Response was:\n{body:#}")), + runtime_status_details: RuntimeStatusDetails::new_only_reason(&format!("Pipeline status response (503 Service Unavailable) cannot be deserialized due to: {e}. Response was:\n{body:#}")).serialize_guaranteed(), runtime_desired_status: RuntimeDesiredStatus::Unavailable, storage_status_details: None, }) @@ -1615,7 +1619,9 @@ impl PipelineAutomaton { { Ok(ExtendedRuntimeStatus { runtime_status: RuntimeStatus::Initializing, - runtime_status_details: json!(format!("Still in the grace period for initializing. Pipeline status endpoint cannot yet be reached due to: {e}")), + runtime_status_details: RuntimeStatusDetails::new_only_reason( + &format!("Still in the grace period for initializing. Pipeline status endpoint cannot yet be reached due to: {e}") + ).serialize_guaranteed(), runtime_desired_status: deployment_initial, storage_status_details: None, }) @@ -1627,9 +1633,10 @@ impl PipelineAutomaton { ); Ok(ExtendedRuntimeStatus { runtime_status: RuntimeStatus::Unavailable, - runtime_status_details: json!(format!( + runtime_status_details: RuntimeStatusDetails::new_only_reason(&format!( "Pipeline status endpoint could not be reached: {e}" - )), + )) + .serialize_guaranteed(), runtime_desired_status: RuntimeDesiredStatus::Unavailable, storage_status_details: None, }) diff --git a/docs.feldera.com/docs/changelog.md b/docs.feldera.com/docs/changelog.md index 34f641ac25b..880d5c23063 100644 --- a/docs.feldera.com/docs/changelog.md +++ b/docs.feldera.com/docs/changelog.md @@ -12,7 +12,16 @@ import TabItem from '@theme/TabItem'; - ## Unreleased + ## Unreleased + + - Pipeline API field `deployment_runtime_status_details` is now strongly typed, + whereas before it was just a generic JSON value type. While `AwaitingApproval`, + the diff is now located at `deployment_runtime_status_details.approval_diff` + instead of being the whole details itself. + + - Pipelines from this latest version onward will have their GET selector + `status_with_connectors` connector stats cached, which are now updated along + with the runtime status details within roughly 1-15s. - Cluster monitor events with information on the backing (Kubernetes) resources is no longer gated behind unstable feature `cluster_monitor_resources` (deprecated). @@ -22,7 +31,7 @@ import TabItem from '@theme/TabItem'; The cluster monitoring of resources can still be disabled by setting in the Helm chart `disableClusterMonitorResources` to `true`. - - A bug fix introduced a backward incompatible change to the replay journal format. + - A bug fix introduced a backward incompatible change to the replay journal format. This only affects pipelines configured with exactly-once fault tolerance. Such pipelines should not be upgraded to the new Feldera runtime if they are in a failed state with non-empty replay journal. Upgrade is possible once the pipeline has been diff --git a/js-packages/web-console/src/lib/components/pipelines/Table.svelte.spec.ts b/js-packages/web-console/src/lib/components/pipelines/Table.svelte.spec.ts index 9193a3b05ed..ff72e078672 100644 --- a/js-packages/web-console/src/lib/components/pipelines/Table.svelte.spec.ts +++ b/js-packages/web-console/src/lib/components/pipelines/Table.svelte.spec.ts @@ -58,6 +58,7 @@ const thumb = (name: string): PipelineThumb => programConfig: { runtime_version: null }, deploymentResourcesStatus: 'Stopped', deploymentResourcesStatusSince: new Date(lastChange[name]), + deploymentRuntimeStatusDetails: { connector_stats: { num_errors: 0 } }, connectors: { numErrors: 0 } }) as unknown as PipelineThumb diff --git a/js-packages/web-console/src/lib/compositions/duplicatePipeline.ts b/js-packages/web-console/src/lib/compositions/duplicatePipeline.ts index 68d9f0f63a5..4aacb1c598b 100644 --- a/js-packages/web-console/src/lib/compositions/duplicatePipeline.ts +++ b/js-packages/web-console/src/lib/compositions/duplicatePipeline.ts @@ -40,6 +40,7 @@ export const optimisticDuplicatePipelineThumb = ( programStatusSince: now.toISOString(), deploymentResourcesStatus: 'Stopped', deploymentResourcesStatusSince: now, + deploymentRuntimeStatusDetails: undefined, connectors: undefined } } diff --git a/js-packages/web-console/src/lib/functions/pipelines/pipelineDiff.spec.ts b/js-packages/web-console/src/lib/functions/pipelines/pipelineDiff.spec.ts new file mode 100644 index 00000000000..a5a7a3654a8 --- /dev/null +++ b/js-packages/web-console/src/lib/functions/pipelines/pipelineDiff.spec.ts @@ -0,0 +1,81 @@ +/** + * Unit tests for `parsePipelineDiff`, which reads the approval diff from + * `deployment_runtime_status_details.approval_diff`. Older pipelines stored the + * diff as the whole runtime status details value; the manager rewrites those + * into the `approval_diff` shape, so the parser only handles the new location. + */ +import { describe, expect, it } from 'vitest' +import type { ExtendedPipeline } from '$lib/services/pipelineManager' +import { parsePipelineDiff } from './pipelineDiff' + +const approvalDiff = { + program_diff: { + added_tables: ['t_added'], + removed_tables: [], + modified_tables: ['t_modified'], + added_views: [], + removed_views: ['v_removed'], + modified_views: [] + }, + program_diff_error: null, + added_input_connectors: ['in_added'], + modified_input_connectors: [], + removed_input_connectors: [], + added_output_connectors: [], + modified_output_connectors: ['out_modified'], + removed_output_connectors: [] +} + +const pipeline = ( + details: unknown +): Pick => + ({ + status: 'AwaitingApproval', + deploymentRuntimeStatusDetails: details + }) as Pick + +describe('parsePipelineDiff', () => { + it('parses the diff nested under approval_diff', () => { + const diff = parsePipelineDiff(pipeline({ approval_diff: approvalDiff })) + expect(diff.tables).toEqual({ added: ['t_added'], removed: [], modified: ['t_modified'] }) + expect(diff.views).toEqual({ added: [], removed: ['v_removed'], modified: [] }) + expect(diff.inputConnectors.added).toEqual(['in_added']) + expect(diff.outputConnectors.modified).toEqual(['out_modified']) + expect(diff.error).toBeUndefined() + }) + + it('defaults program table/view diffs when program_diff is null', () => { + const diff = parsePipelineDiff( + pipeline({ approval_diff: { ...approvalDiff, program_diff: null } }) + ) + expect(diff.tables).toEqual({ added: [], removed: [], modified: [] }) + expect(diff.views).toEqual({ added: [], removed: [], modified: [] }) + }) + + it('surfaces program_diff_error', () => { + const diff = parsePipelineDiff( + pipeline({ approval_diff: { ...approvalDiff, program_diff_error: 'boom' } }) + ) + expect(diff.error).toBe('boom') + }) + + it('throws when the expected approval info is not there', () => { + expect(() => + parsePipelineDiff({ + status: 'AwaitingApproval', + deploymentRuntimeStatusDetails: {} + } as Pick) + ).toThrow('data is not available') + }) + + it('throws when approval_diff is absent', () => { + // A running-connector runtime status may set only `connector_stats`. + expect(() => parsePipelineDiff(pipeline({ connector_stats: { num_errors: 0 } }))).toThrow( + 'not available' + ) + }) + + it('throws when there are no runtime status details at all', () => { + expect(() => parsePipelineDiff(pipeline(undefined))).toThrow('not available') + }) +}) diff --git a/js-packages/web-console/src/lib/functions/pipelines/pipelineDiff.ts b/js-packages/web-console/src/lib/functions/pipelines/pipelineDiff.ts index 7ebc91234bb..ac711c6c4c0 100644 --- a/js-packages/web-console/src/lib/functions/pipelines/pipelineDiff.ts +++ b/js-packages/web-console/src/lib/functions/pipelines/pipelineDiff.ts @@ -32,12 +32,16 @@ export const parsePipelineDiff = ( throw new Error('Pipeline is not awaiting approval') } - if (!pipeline.deploymentRuntimeStatusDetails) { + // The approval diff lives at `deployment_runtime_status_details.approval_diff`. + // The manager rewrites older pipelines, whose entire runtime status details + // were the diff, into this shape, so a single path covers all versions. + const approvalDiff = pipeline.deploymentRuntimeStatusDetails?.approval_diff + if (!approvalDiff) { throw new Error('Pipeline diff data is not available') } try { - const rawDiff = va.parse(pipelineDiffSchema, pipeline.deploymentRuntimeStatusDetails) + const rawDiff = va.parse(pipelineDiffSchema, approvalDiff) return { tables: rawDiff.program_diff ? { diff --git a/js-packages/web-console/src/lib/services/manager/index.ts b/js-packages/web-console/src/lib/services/manager/index.ts index 1555ae309bd..0b973bbd576 100644 --- a/js-packages/web-console/src/lib/services/manager/index.ts +++ b/js-packages/web-console/src/lib/services/manager/index.ts @@ -34,6 +34,7 @@ export { getPipelineSupportBundle, getPipelineTimeSeries, getPipelineTimeSeriesStream, + getRemoteCheckpoints, httpInput, httpOutput, listApiKeys, @@ -89,6 +90,7 @@ export type { CheckpointResponse, CheckpointStatus, CheckpointSyncFailure, + CheckpointSyncResponse, CheckpointSyncStatus, Chunk, ClientMetadata, @@ -161,6 +163,8 @@ export type { Demo, DevTweaks, DisplaySchedule, + DynamoDbWriteMode, + DynamoDbWriterConfig, ErrorResponse, ExtendedClusterMonitorEvent, Field, @@ -302,6 +306,11 @@ export type { GetPipelineTimeSeriesStreamErrors, GetPipelineTimeSeriesStreamResponse, GetPipelineTimeSeriesStreamResponses, + GetRemoteCheckpointsData, + GetRemoteCheckpointsError, + GetRemoteCheckpointsErrors, + GetRemoteCheckpointsResponse, + GetRemoteCheckpointsResponses, GlobalControllerMetrics, GlueCatalogConfig, HealthStatus, @@ -467,6 +476,7 @@ export type { PostPipelineTestingErrors, PostPipelineTestingResponses, PostPutPipeline, + PostprocessorConfig, PostStopPipelineParameters, PostUpdateRuntimeData, PostUpdateRuntimeError, @@ -493,6 +503,7 @@ export type { RedisOutputConfig, Rel, Relation, + RemoteCheckpoint, ReplayPolicy, ResourceConfig, ResourcesDesiredStatus, @@ -502,6 +513,7 @@ export type { RuntimeConfig, RuntimeDesiredStatus, RuntimeStatus, + RuntimeStatusDetails, RustCompilationInfo, S3InputConfig, SampleStatistics, @@ -530,6 +542,7 @@ export type { StorageConfig, StorageOptions, StorageStatus, + StorageStatusDetails, SuspendError, SyncCheckpointData, SyncCheckpointError, diff --git a/js-packages/web-console/src/lib/services/manager/sdk.gen.ts b/js-packages/web-console/src/lib/services/manager/sdk.gen.ts index 8f4ecab02cd..fd4a1ec0bf4 100644 --- a/js-packages/web-console/src/lib/services/manager/sdk.gen.ts +++ b/js-packages/web-console/src/lib/services/manager/sdk.gen.ts @@ -101,6 +101,9 @@ import type { GetPipelineTimeSeriesStreamData, GetPipelineTimeSeriesStreamErrors, GetPipelineTimeSeriesStreamResponses, + GetRemoteCheckpointsData, + GetRemoteCheckpointsErrors, + GetRemoteCheckpointsResponses, HttpInputData, HttpInputErrors, HttpInputResponses, @@ -696,6 +699,10 @@ export const getCheckpointStatus = ( * Get the checkpoints for a pipeline * * Retrieve the current checkpoints made by a pipeline. + * + * **Stability note**: for multihost pipelines, this endpoint returns the + * combined checkpoint list from all hosts. The shape of this response may + * change in a future release. */ export const getCheckpoints = ( options: Options @@ -712,6 +719,28 @@ export const getCheckpoints = ( ...options }) +/** + * List checkpoints in remote object storage + * + * Retrieve the list of checkpoints available in the configured remote object + * storage (e.g., S3). Requires the pipeline to be running with a sync + * storage configuration. + */ +export const getRemoteCheckpoints = ( + options: Options +) => + (options.client ?? client).get< + GetRemoteCheckpointsResponses, + GetRemoteCheckpointsErrors, + ThrowOnError, + 'data' + >({ + responseStyle: 'data', + security: [{ scheme: 'bearer', type: 'http' }], + url: '/v0/pipelines/{pipeline_name}/checkpoints/remote', + ...options + }) + /** * Performance Profile JSON * diff --git a/js-packages/web-console/src/lib/services/manager/types.gen.ts b/js-packages/web-console/src/lib/services/manager/types.gen.ts index 14fb6b841be..c3d591b3267 100644 --- a/js-packages/web-console/src/lib/services/manager/types.gen.ts +++ b/js-packages/web-console/src/lib/services/manager/types.gen.ts @@ -266,6 +266,13 @@ export type CheckpointSyncFailure = { uuid: string } +/** + * Response to a sync checkpoint request. + */ +export type CheckpointSyncResponse = { + checkpoint_uuid: string +} + /** * Checkpoint status returned by the `/checkpoint/sync_status` endpoint. */ @@ -780,6 +787,13 @@ export type ConnectorConfig = OutputBufferConfig & { * The default is `false`. */ paused?: boolean + /** + * Optional postprocessor configuration + */ + postprocessor?: Array | null + /** + * Optional preprocessor configuration + */ preprocessor?: Array | null /** * Send a full snapshot of a materialized view when the connector first @@ -840,16 +854,12 @@ export type ConnectorHealth = { export type ConnectorHealthStatus = 'Healthy' | 'Unhealthy' /** - * Aggregated connector error statistics. - * - * This structure contains the sum of all error counts across all input and output connectors - * for a pipeline. + * Statistics across all connectors. */ export type ConnectorStats = { /** * Total number of errors across all connectors. * - * This is the sum of: * - `num_transport_errors` from all input connectors * - `num_parse_errors` from all input connectors * - `num_encode_errors` from all output connectors @@ -1770,6 +1780,93 @@ export type DisplaySchedule = } | 'Always' +/** + * DynamoDB write API used by the output connector. + */ +export type DynamoDbWriteMode = 'batch' | 'transactional' + +/** + * DynamoDB output connector configuration. + */ +export type DynamoDbWriterConfig = { + /** + * AWS access key ID. + * + * If both `aws_access_key_id` and `aws_secret_access_key` are specified, + * the connector uses these static credentials. Otherwise it uses the + * default AWS credential provider chain, including IAM Roles for Service + * Accounts (IRSA) in EKS. + * + * Static credentials are treated as long-lived IAM keys: no session token + * is sent, so STS-issued temporary credentials are not supported via these + * fields. To use temporary credentials, rely on the default provider chain + * instead (leave these unset). + */ + aws_access_key_id?: string | null + /** + * AWS secret access key. + */ + aws_secret_access_key?: string | null + /** + * Maximum number of write requests in one DynamoDB write call. + * + * DynamoDB supports at most 100 for `TransactWriteItems` and at most 25 + * for `BatchWriteItem`. If omitted, the connector uses the maximum for + * the selected `write_mode`. + */ + batch_size?: number | null + /** + * Optional endpoint URL, for example when using a local + * DynamoDB-compatible service. + */ + endpoint_url?: string | null + /** + * Maximum number of bytes buffered by each worker before flushing writes. + * + * This is an approximate size based on encoded DynamoDB attributes. + */ + max_buffer_size_bytes?: number + /** + * Maximum number of DynamoDB write requests in flight per worker thread. + * + * The total in-flight request count across the connector is + * `threads × max_concurrent_requests`. Size this accordingly when + * tuning against a provisioned-throughput table to avoid excessive + * throttling. + */ + max_concurrent_requests?: number + /** + * Maximum number of retries for a failed or partially-applied DynamoDB write chunk. + * + * For `batch` writes, `BatchWriteItem` may return some items as "unprocessed" in a + * successful 200 response; those are re-submitted and counted as retries. For + * `transactional` writes, a failed `TransactWriteItems` call is retried in full. + * + * Transient errors (throttling, network failures) are first handled transparently by the + * AWS SDK. Only attempts that reach this connector's retry loop count against this limit. + * Each retry waits longer than the previous one (exponential backoff), up to a ceiling + * of roughly 13 seconds. + * + * Set to `null` to retry indefinitely. After the backoff ceiling is reached the connector + * keeps retrying at that interval, providing backpressure until DynamoDB recovers. + * Defaults to `10`. + */ + max_retries?: number | null + /** + * AWS region. + */ + region: string + /** + * Name of the DynamoDB table to write to. + */ + table: string + /** + * Number of worker threads used to encode and write disjoint key ranges. + */ + threads?: number + write_mode?: DynamoDbWriteMode +} + /** * Information returned by REST API endpoints on error. */ @@ -3149,8 +3246,7 @@ export type OutputBufferConfig = { * total number of updates output by the pipeline. Updates to the * same record can overwrite or cancel previous updates. * - * By default, the buffer can grow indefinitely until one of - * the other output conditions is satisfied. + * The default is 10,000,000. * * NOTE: this configuration option requires the `enable_output_buffer` flag * to be set. @@ -3634,7 +3730,7 @@ export type PipelineInfo = ClientMetadata & { deployment_runtime_desired_status?: RuntimeDesiredStatus | null deployment_runtime_desired_status_since?: string | null deployment_runtime_status?: RuntimeStatus | null - deployment_runtime_status_details?: unknown + deployment_runtime_status_details?: RuntimeStatusDetails | null deployment_runtime_status_since?: string | null deployment_status: CombinedStatus deployment_status_since: string @@ -3651,7 +3747,7 @@ export type PipelineInfo = ClientMetadata & { refresh_version: Version runtime_config: RuntimeConfig storage_status: StorageStatus - storage_status_details?: unknown + storage_status_details?: StorageStatusDetails | null udf_rust: string udf_toml: string version: Version @@ -3705,7 +3801,7 @@ export type PipelineSelectedInfo = ClientMetadata & { deployment_runtime_desired_status?: RuntimeDesiredStatus | null deployment_runtime_desired_status_since?: string | null deployment_runtime_status?: RuntimeStatus | null - deployment_runtime_status_details?: unknown + deployment_runtime_status_details?: RuntimeStatusDetails | null deployment_runtime_status_since?: string | null deployment_status: CombinedStatus deployment_status_since: string @@ -3722,7 +3818,7 @@ export type PipelineSelectedInfo = ClientMetadata & { refresh_version: Version runtime_config?: RuntimeConfig | null storage_status: StorageStatus - storage_status_details?: unknown + storage_status_details?: StorageStatusDetails | null udf_rust?: string | null udf_toml?: string | null version: Version @@ -4073,6 +4169,22 @@ export type PostgresWriterConfig = { uri: string } +/** + * Configuration for describing a postprocessor + */ +export type PostprocessorConfig = { + /** + * Arbitrary additional configuration expected by the postprocessor + * encoded as a JSON Value. + */ + config: unknown + /** + * Name of the postprocessor. + * All postprocessors with the same name will perform the same task. + */ + name: string +} + /** * Configuration for describing a preprocessor */ @@ -4130,6 +4242,24 @@ export type ProgramConfig = { * If not set (null), the runtime version will be the same as the platform version. */ runtime_version?: string | null + /** + * Use the platform SQL compiler when a non-platform `runtime_version` is specified. + * + * Warning: This setting is experimental and may change in the future. + * Requires the platform to run with the unstable feature `runtime_version` enabled. + * + * When `false` (default), the SQL compiler matching the `runtime_version` is + * downloaded and used. When `true`, the platform's SQL compiler is used instead. + * + * Setting this to `true` avoids downloading the runtime-version-specific SQL + * compiler JAR (e.g., when network access is unavailable or slow), at the cost + * of potentially using a mismatched SQL compiler. The Rust runtime sources are + * still checked out and compiled from the requested `runtime_version`. + * + * Has no effect when `runtime_version` is not set or the platform does not have + * the unstable feature `runtime_version` enabled. + */ + use_platform_compiler?: boolean } /** @@ -4391,6 +4521,16 @@ export type Relation = SqlIdentifier & { } } +/** + * A checkpoint that exists in remote object storage. + */ +export type RemoteCheckpoint = { + /** + * UUID of the checkpoint. + */ + uuid: string +} + export type ReplayPolicy = 'Instant' | 'Original' export type ResourceConfig = { @@ -4843,6 +4983,26 @@ export type RuntimeStatus = | 'Running' | 'Suspended' +/** + * Details about the current runtime status. The fields in this struct should all be **optional** + * and set only by a runtime status when they are known. Otherwise, they can just be set `None`. + */ +export type RuntimeStatusDetails = { + /** + * The diff which is awaiting approval. + * + * Specifically useful for: `AwaitingApproval`. + */ + approval_diff?: unknown + connector_stats?: ConnectorStats | null + /** + * Free form text giving an explanation why it is currently in this runtime status. + * + * Specifically useful for: `Unavailable`, `Initializing`. + */ + reason?: string | null +} + /** * Rust compilation information. */ @@ -5025,7 +5185,7 @@ export type SqlIdentifier = { } /** - * The available SQL types as specified in `CREATE` statements. + * The available SQL column type names. Each value is the platform's wire encoding of the type (e.g. `BIGINT`, `INTEGER`, `INTERVAL_DAY`), not valid SQL type syntax. */ export type SqlType = | 'BOOLEAN' @@ -5180,6 +5340,17 @@ export type StorageOptions = { */ export type StorageStatus = 'Cleared' | 'InUse' | 'Clearing' +/** + * Details about pipeline storage, which are returned as part of the regular runtime status polling + * by the runner. + */ +export type StorageStatusDetails = { + /** + * Present checkpoints. + */ + checkpoints: Array +} + /** * Whether a pipeline supports checkpointing and suspend-and-resume. */ @@ -5268,6 +5439,19 @@ export type SyncConfig = { * Default: 10 */ multi_thread_streams?: number | null + /** + * When true, checkpoint downloads use the maximum resources available on + * the host: `transfers` and `checkers` are scaled to the number of CPUs, + * and the download buffer is allowed to grow up to most of the available + * memory. This maximizes download throughput at the cost of higher CPU and + * memory usage during a pull. + * + * When false, downloads use the values configured via `transfers`, + * `checkers`, and the rclone defaults instead. + * + * Default: true + */ + optimize_download_resources?: boolean /** * The name of the cloud storage provider (e.g., `"AWS"`, `"Minio"`). * @@ -5334,19 +5518,9 @@ export type SyncConfig = { */ secret_key?: string | null /** - * When `true`, the pipeline starts in **standby** mode; processing doesn't - * start until activation (`POST /activate`). - * If this pipeline was previously activated and the storage has not been - * cleared, the pipeline will auto activate, no newer checkpoints will be - * fetched. - * - * Standby behavior depends on `start_from_checkpoint`: - * - If `latest`, pipeline continuously fetches the latest available - * checkpoint until activated. - * - If checkpoint UUID, pipeline fetches this checkpoint once and waits - * in standby until activated. + * **Deprecated.** Use `initial=standby` when starting the pipeline instead. * - * Default: `false` + * @deprecated */ standby?: boolean start_from_checkpoint?: StartFromCheckpoint | null @@ -5465,6 +5639,10 @@ export type TransportConfig = config: DeltaTableWriterConfig name: 'delta_table_output' } + | { + config: DynamoDbWriterConfig + name: 'dynamodb_output' + } | { config: RedisOutputConfig name: 'redis_output' @@ -5690,7 +5868,7 @@ export type GetApiKeyResponses = { /** * API key retrieved successfully */ - 200: ApiKeyDescr + 200: Array } export type GetApiKeyResponse = GetApiKeyResponses[keyof GetApiKeyResponses] @@ -6095,7 +6273,7 @@ export type PostPipelineActivateResponses = { /** * Pipeline activation initiated */ - 202: CheckpointResponse + 202: string } export type PostPipelineActivateResponse = @@ -6131,9 +6309,9 @@ export type PostPipelineApproveError = PostPipelineApproveErrors[keyof PostPipel export type PostPipelineApproveResponses = { /** - * Pipeline activation initiated + * Bootstrap approved */ - 202: CheckpointResponse + 200: string } export type PostPipelineApproveResponse = @@ -6197,9 +6375,9 @@ export type SyncCheckpointError = SyncCheckpointErrors[keyof SyncCheckpointError export type SyncCheckpointResponses = { /** - * Checkpoint synced to object store + * Checkpoint sync to object store initiated */ - 200: CheckpointResponse + 202: CheckpointSyncResponse } export type SyncCheckpointResponse = SyncCheckpointResponses[keyof SyncCheckpointResponses] @@ -6296,13 +6474,46 @@ export type GetCheckpointsError = GetCheckpointsErrors[keyof GetCheckpointsError export type GetCheckpointsResponses = { /** - * Checkpoints retrieved successfully + * Checkpoints retrieved successfully. For multihost pipelines the list contains entries from all hosts; the shape of this response may change in a future release. */ - 200: CheckpointMetadata + 200: Array } export type GetCheckpointsResponse = GetCheckpointsResponses[keyof GetCheckpointsResponses] +export type GetRemoteCheckpointsData = { + body?: never + path: { + /** + * Unique pipeline name + */ + pipeline_name: string + } + query?: never + url: '/v0/pipelines/{pipeline_name}/checkpoints/remote' +} + +export type GetRemoteCheckpointsErrors = { + /** + * Pipeline with that name does not exist + */ + 404: ErrorResponse + 500: ErrorResponse + 503: ErrorResponse +} + +export type GetRemoteCheckpointsError = GetRemoteCheckpointsErrors[keyof GetRemoteCheckpointsErrors] + +export type GetRemoteCheckpointsResponses = { + /** + * Remote checkpoints retrieved successfully. + */ + 200: Array +} + +export type GetRemoteCheckpointsResponse = + GetRemoteCheckpointsResponses[keyof GetRemoteCheckpointsResponses] + export type GetPipelineCircuitJsonProfileData = { body?: never path: { diff --git a/openapi.json b/openapi.json index 15e8043bcee..ae463787026 100644 --- a/openapi.json +++ b/openapi.json @@ -8189,7 +8189,7 @@ }, "ConnectorStats": { "type": "object", - "description": "Aggregated connector error statistics.\n\nThis structure contains the sum of all error counts across all input and output connectors\nfor a pipeline.", + "description": "Statistics across all connectors.", "required": [ "num_errors" ], @@ -8197,7 +8197,7 @@ "num_errors": { "type": "integer", "format": "int64", - "description": "Total number of errors across all connectors.\n\nThis is the sum of:\n- `num_transport_errors` from all input connectors\n- `num_parse_errors` from all input connectors\n- `num_encode_errors` from all output connectors\n- `num_transport_errors` from all output connectors", + "description": "Total number of errors across all connectors.\n\n- `num_transport_errors` from all input connectors\n- `num_parse_errors` from all input connectors\n- `num_encode_errors` from all output connectors\n- `num_transport_errors` from all output connectors", "minimum": 0 } } @@ -11425,6 +11425,11 @@ "nullable": true }, "deployment_runtime_status_details": { + "allOf": [ + { + "$ref": "#/components/schemas/RuntimeStatusDetails" + } + ], "nullable": true }, "deployment_runtime_status_since": { @@ -11485,6 +11490,11 @@ "$ref": "#/components/schemas/StorageStatus" }, "storage_status_details": { + "allOf": [ + { + "$ref": "#/components/schemas/StorageStatusDetails" + } + ], "nullable": true }, "udf_rust": { @@ -11690,6 +11700,11 @@ "nullable": true }, "deployment_runtime_status_details": { + "allOf": [ + { + "$ref": "#/components/schemas/RuntimeStatusDetails" + } + ], "nullable": true }, "deployment_runtime_status_since": { @@ -11766,6 +11781,11 @@ "$ref": "#/components/schemas/StorageStatus" }, "storage_status_details": { + "allOf": [ + { + "$ref": "#/components/schemas/StorageStatusDetails" + } + ], "nullable": true }, "udf_rust": { @@ -13145,6 +13165,29 @@ "Suspended" ] }, + "RuntimeStatusDetails": { + "type": "object", + "description": "Details about the current runtime status. The fields in this struct should all be **optional**\nand set only by a runtime status when they are known. Otherwise, they can just be set `None`.", + "properties": { + "approval_diff": { + "description": "The diff which is awaiting approval.\n\nSpecifically useful for: `AwaitingApproval`.", + "nullable": true + }, + "connector_stats": { + "allOf": [ + { + "$ref": "#/components/schemas/ConnectorStats" + } + ], + "nullable": true + }, + "reason": { + "type": "string", + "description": "Free form text giving an explanation why it is currently in this runtime status.\n\nSpecifically useful for: `Unavailable`, `Initializing`.", + "nullable": true + } + } + }, "RustCompilationInfo": { "type": "object", "description": "Rust compilation information.", @@ -13685,6 +13728,22 @@ "Clearing" ] }, + "StorageStatusDetails": { + "type": "object", + "description": "Details about pipeline storage, which are returned as part of the regular runtime status polling\nby the runner.", + "required": [ + "checkpoints" + ], + "properties": { + "checkpoints": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CheckpointMetadata" + }, + "description": "Present checkpoints." + } + } + }, "SuspendError": { "oneOf": [ { diff --git a/python/tests/platform/test_bootstrapping.py b/python/tests/platform/test_bootstrapping.py index 325e7dad9e9..3c232405a1a 100644 --- a/python/tests/platform/test_bootstrapping.py +++ b/python/tests/platform/test_bootstrapping.py @@ -91,7 +91,7 @@ def test_bootstrap_enterprise(pipeline_name): pipeline.start(bootstrap_policy=BootstrapPolicy.AWAIT_APPROVAL) assert pipeline.status() == PipelineStatus.AWAITINGAPPROVAL - diff = pipeline.deployment_runtime_status_details() + diff = pipeline.deployment_runtime_status_details()["approval_diff"] print(f"Pipeline diff: {diff}") assert diff["program_diff"] == { "added_tables": [], @@ -132,7 +132,7 @@ def test_bootstrap_enterprise(pipeline_name): pipeline.start(bootstrap_policy=BootstrapPolicy.AWAIT_APPROVAL) assert pipeline.status() == PipelineStatus.AWAITINGAPPROVAL - diff = pipeline.deployment_runtime_status_details() + diff = pipeline.deployment_runtime_status_details()["approval_diff"] print(f"Pipeline diff: {diff}") assert diff["program_diff"] == { "added_tables": ["t2"], @@ -173,7 +173,7 @@ def test_bootstrap_enterprise(pipeline_name): pipeline.start(bootstrap_policy=BootstrapPolicy.AWAIT_APPROVAL) assert pipeline.status() == PipelineStatus.AWAITINGAPPROVAL - diff = pipeline.deployment_runtime_status_details() + diff = pipeline.deployment_runtime_status_details()["approval_diff"] print(f"Pipeline diff: {diff}") assert diff["program_diff"] == { "added_tables": [], @@ -219,7 +219,7 @@ def test_bootstrap_enterprise(pipeline_name): pipeline.start(bootstrap_policy=BootstrapPolicy.AWAIT_APPROVAL) assert pipeline.status() == PipelineStatus.AWAITINGAPPROVAL - diff = pipeline.deployment_runtime_status_details() + diff = pipeline.deployment_runtime_status_details()["approval_diff"] print(f"Pipeline diff: {diff}") assert diff["program_diff"] == { "added_tables": [], @@ -253,7 +253,7 @@ def test_bootstrap_enterprise(pipeline_name): pipeline.start(bootstrap_policy=BootstrapPolicy.AWAIT_APPROVAL) assert pipeline.status() == PipelineStatus.AWAITINGAPPROVAL - diff = pipeline.deployment_runtime_status_details() + diff = pipeline.deployment_runtime_status_details()["approval_diff"] print(f"Pipeline diff: {diff}") assert diff["program_diff"] == { "added_tables": [], @@ -624,7 +624,7 @@ def gen_sql(connectors): pipeline.start(bootstrap_policy=BootstrapPolicy.AWAIT_APPROVAL) assert pipeline.status() == PipelineStatus.AWAITINGAPPROVAL - diff = pipeline.deployment_runtime_status_details() + diff = pipeline.deployment_runtime_status_details()["approval_diff"] print(f"Pipeline diff: {diff}") assert diff == { "added_input_connectors": ["t1.unnamed-0", "t1.unnamed-1"], @@ -673,7 +673,7 @@ def gen_sql(connectors): pipeline.start(bootstrap_policy=BootstrapPolicy.AWAIT_APPROVAL) assert pipeline.status() == PipelineStatus.AWAITINGAPPROVAL - diff = pipeline.deployment_runtime_status_details() + diff = pipeline.deployment_runtime_status_details()["approval_diff"] print(f"Pipeline diff: {diff}") assert diff == { "added_input_connectors": [], diff --git a/python/tests/platform/test_pipeline_lifecycle.py b/python/tests/platform/test_pipeline_lifecycle.py index 2fb70487eb1..a57bd22457d 100644 --- a/python/tests/platform/test_pipeline_lifecycle.py +++ b/python/tests/platform/test_pipeline_lifecycle.py @@ -850,3 +850,37 @@ def test_start_failed_compilation(pipeline_name): except FelderaAPIError as e: error = e assert error is not None and error.error_code == "CannotStartWithCompilationError" + + +@gen_pipeline_name +def test_connector_stats_errors_count(pipeline_name): + """ + Tests that the connector statistics number of errors is set. + """ + pipeline = PipelineBuilder( + TEST_CLIENT, pipeline_name, "CREATE TABLE t1 (v1 INT);" + ).create_or_replace() + pipeline.start() + assert _ingress(pipeline_name, "t1", "1\n2\n3\n").status_code == HTTPStatus.OK + assert ( + _ingress(pipeline_name, "t1", "a\nb\nc\nd\n").status_code + == HTTPStatus.BAD_REQUEST + ) + assert _ingress(pipeline_name, "t1", "4\n5\n6\n").status_code == HTTPStatus.OK + start_s = time.monotonic() + while True: + num_errors = pipeline.deployment_runtime_status_details()["connector_stats"][ + "num_errors" + ] + if num_errors == 4: + break + elif time.monotonic() - start_s >= 30.0 or num_errors > 4: + raise ValueError(f"Number of errors is not 4 but {num_errors}") + time.sleep(0.5) + assert pipeline.stats().global_metrics.total_completed_records == 6 + assert ( + TEST_CLIENT.http.get( + f"/pipelines/{pipeline_name}?selector=status_with_connectors" + )["connectors"]["num_errors"] + == 4 + )