[adapters] Add verbose config setting for delta.

ryzhyk · ryzhyk · commit abdd71bb1185 · 2026-01-18T22:57:06.000Z
The new `verbose` flag can be used to log detailed progress at the INFO level
for a specific connector.

Currently we only log add/delete actions in follow and cdc modes.

Signed-off-by: Leonid Ryzhyk &lt;ryzhyk@gmail.com&gt;
diff --git a/crates/adapters/src/integrated/delta_table/input.rs b/crates/adapters/src/integrated/delta_table/input.rs
@@ -1625,6 +1625,31 @@ impl DeltaTableInputEndpointInner {
         input_stream: &mut dyn ArrowStream,
         receiver: &mut Receiver<PipelineState>,
     ) {
+        if self.config.verbose > 0 {
+            // Don't log actions we ignore to limit spurious logging. E.g., delta lake
+            // optimization passes can generate thousand of noop actions.
+            let data_change_actions = actions
+                .iter()
+                .filter(|action| match action {
+                    Action::Add(add) if add.data_change => true,
+                    Action::Remove(remove) if remove.data_change => true,
+                    _ => false,
+                })
+                .collect::<Vec<_>>();
+            info!(
+                "delta_table {}: log entry for table version {new_version}: {data_change_actions:?}{}",
+                &self.endpoint_name,
+                if actions.len() > data_change_actions.len() {
+                    format!(
+                        " ({} other actions)",
+                        actions.len() - data_change_actions.len()
+                    )
+                } else {
+                    "".to_string()
+                }
+            );
+        }
+
         // Use the time when we _started_ reading transaction data as the ingestion timestamp.
         let timestamp = Utc::now();
 
diff --git a/crates/feldera-types/src/transport/delta_table.rs b/crates/feldera-types/src/transport/delta_table.rs
@@ -296,6 +296,17 @@ pub struct DeltaTableReaderConfig {
     /// The default value is 6.
     pub max_concurrent_readers: Option<u32>,
 
+    /// Enable verbose logging.
+    ///
+    /// When enabled, the connector will log detailed information at INFO level.
+    ///
+    /// Supported values:
+    /// * 0 - no verbose logging
+    /// * 1 - log all Delta log entries in follow and cdc modes.
+    /// * >1 - reserved for future use
+    #[serde(default)]
+    pub verbose: u32,
+
     /// Storage options for configuring backend object store.
     ///
     /// For specific options available for different storage backends, see:
@@ -324,7 +335,7 @@ fn test_delta_reader_config_serde() {
     let config = serde_json::from_str::<DeltaTableReaderConfig>(config_str).unwrap();
 
     let serialized_config = serde_json::to_string(&config).unwrap();
-    let expected = r#"{"uri":"protocol:/path/to/somewhere","mode":"follow","transaction_mode":"none","timestamp_column":"ts","filter":null,"skip_unused_columns":false,"snapshot_filter":"ts BETWEEN '2005-01-01 00:00:00' AND '2010-12-31 23:59:59'","version":null,"datetime":"2010-12-31 00:00:00Z","end_version":null,"cdc_delete_filter":null,"cdc_order_by":null,"num_parsers":4,"max_concurrent_readers":null,"customoption1":"val1","customoption2":"val2"}"#;
+    let expected = r#"{"uri":"protocol:/path/to/somewhere","mode":"follow","transaction_mode":"none","timestamp_column":"ts","filter":null,"skip_unused_columns":false,"snapshot_filter":"ts BETWEEN '2005-01-01 00:00:00' AND '2010-12-31 23:59:59'","version":null,"datetime":"2010-12-31 00:00:00Z","end_version":null,"cdc_delete_filter":null,"cdc_order_by":null,"num_parsers":4,"max_concurrent_readers":null,"customoption1":"val1","customoption2":"val2","verbose":0}"#;
     assert_eq!(
         serde_json::from_str::<serde_json::Value>(&serialized_config).unwrap(),
         serde_json::from_str::<serde_json::Value>(expected).unwrap()
diff --git a/openapi.json b/openapi.json
@@ -7431,6 +7431,12 @@
             "type": "string",
             "description": "Table URI.\n\nExample: \"s3://feldera-fraud-detection-data/demographics_train\""
           },
+          "verbose": {
+            "type": "integer",
+            "format": "int32",
+            "description": "Enable verbose logging.\n\nWhen enabled, the connector will log detailed information at INFO level.\n\nSupported values:\n* 0 - no verbose logging\n* 1 - log all Delta log entries in follow and cdc modes.\n* >1 - reserved for future use",
+            "minimum": 0
+          },
           "version": {
             "type": "integer",
             "format": "int64",
