From d018dec5aee5e67a68e1a0dc0d3fa2b54e4c8529 Mon Sep 17 00:00:00 2001 From: Zhenyu Luo Date: Sat, 16 May 2026 22:24:41 +0800 Subject: [PATCH 01/16] Update lib (#229) Co-authored-by: Cursor --- .gitignore | 4 + Cargo.lock | 123 ++++++++- Cargo.toml | 99 ++----- Makefile | 7 +- cli/cli/Cargo.toml | 4 +- cli/cli/src/repl.rs | 44 ++-- examples/examples-validator/Cargo.toml | 2 +- protocol/Cargo.toml | 4 +- src/catalog/Cargo.toml | 11 + src/catalog/src/error.rs | 4 + src/catalog/src/lib.rs | 9 + src/catalog/src/meta_store.rs | 23 ++ src/catalog/src/stream_catalog.rs | 69 +++++ src/catalog_storage/Cargo.toml | 14 + src/catalog_storage/src/lib.rs | 10 + src/catalog_storage/src/memory.rs | 63 +++++ src/catalog_storage/src/rocksdb.rs | 81 ++++++ .../src}/stream_catalog/codec.rs | 0 .../src}/stream_catalog/manager.rs | 4 +- .../src}/stream_catalog/meta_store.rs | 0 .../src}/stream_catalog/mod.rs | 0 .../src}/stream_catalog/rocksdb_meta_store.rs | 2 +- .../src}/task/factory.rs | 0 .../src}/task/function_info.rs | 0 .../src}/task/mod.rs | 0 .../src}/task/proto_codec.rs | 2 +- .../src}/task/rocksdb_storage.rs | 2 +- .../src}/task/storage.rs | 2 +- src/common/Cargo.toml | 8 + src/common/{ => src}/fs_schema.rs | 0 src/common/{mod.rs => src/legacy_mod.rs} | 0 src/common/src/lib.rs | 3 + src/config/Cargo.toml | 17 ++ src/config/{ => src}/global_config.rs | 0 src/config/{mod.rs => src/lib.rs} | 17 +- src/config/{ => src}/loader.rs | 0 src/config/{ => src}/log_config.rs | 0 src/config/{ => src}/paths.rs | 0 src/config/{ => src}/python_config.rs | 0 src/config/{ => src}/service_config.rs | 0 src/config/{ => src}/storage.rs | 0 src/config/{ => src}/streaming_job.rs | 0 src/config/{ => src}/system.rs | 0 src/config/{ => src}/wasm_config.rs | 0 src/coordinator/Cargo.toml | 8 + src/coordinator/{ => src}/analyze/analysis.rs | 0 src/coordinator/{ => src}/analyze/analyzer.rs | 0 src/coordinator/{ => src}/analyze/mod.rs | 0 src/coordinator/{ => src}/coordinator.rs | 0 .../{mod.rs => src/coordinator_body.rs} | 2 + src/coordinator/{ => src}/dataset/data_set.rs | 0 .../{ => src}/dataset/execute_result.rs | 0 src/coordinator/{ => src}/dataset/mod.rs | 0 .../dataset/show_catalog_tables_result.rs | 0 .../show_create_streaming_table_result.rs | 0 .../dataset/show_create_table_result.rs | 0 .../dataset/show_functions_result.rs | 2 +- .../dataset/show_streaming_tables_result.rs | 2 +- .../{ => src}/execution/executor.rs | 8 +- src/coordinator/{ => src}/execution/mod.rs | 0 .../{ => src}/execution_context.rs | 0 src/coordinator/src/lib.rs | 3 + .../{ => src}/plan/compile_error_plan.rs | 0 .../{ => src}/plan/create_function_plan.rs | 0 .../plan/create_python_function_plan.rs | 0 .../{ => src}/plan/create_table_plan.rs | 0 .../{ => src}/plan/drop_function_plan.rs | 0 .../plan/drop_streaming_table_plan.rs | 0 .../{ => src}/plan/drop_table_plan.rs | 0 .../{ => src}/plan/logical_plan_visitor.rs | 17 +- .../{ => src}/plan/lookup_table_plan.rs | 0 src/coordinator/{ => src}/plan/mod.rs | 2 - src/coordinator/{ => src}/plan/optimizer.rs | 0 .../plan/show_catalog_tables_plan.rs | 0 .../plan/show_create_streaming_table_plan.rs | 0 .../{ => src}/plan/show_create_table_plan.rs | 0 .../{ => src}/plan/show_functions_plan.rs | 0 .../plan/show_streaming_tables_plan.rs | 0 .../{ => src}/plan/start_function_plan.rs | 0 .../{ => src}/plan/stop_function_plan.rs | 0 .../{ => src}/plan/streaming_compiler.rs | 0 .../plan/streaming_table_connector_plan.rs | 0 .../{ => src}/plan/streaming_table_plan.rs | 0 src/coordinator/{ => src}/plan/visitor.rs | 0 src/coordinator/{ => src}/runtime_context.rs | 6 +- .../src/sql_classify.rs} | 244 ++++++------------ .../{ => src}/statement/create_function.rs | 0 .../statement/create_python_function.rs | 0 .../{ => src}/statement/create_table.rs | 0 .../{ => src}/statement/drop_function.rs | 0 .../statement/drop_streaming_table.rs | 0 .../{ => src}/statement/drop_table.rs | 0 src/coordinator/{ => src}/statement/mod.rs | 0 .../statement/show_catalog_tables.rs | 0 .../statement/show_create_streaming_table.rs | 0 .../{ => src}/statement/show_create_table.rs | 0 .../{ => src}/statement/show_functions.rs | 0 .../statement/show_streaming_tables.rs | 0 .../{ => src}/statement/start_function.rs | 0 .../{ => src}/statement/stop_function.rs | 0 .../{ => src}/statement/streaming_table.rs | 0 .../{ => src}/statement/visitor.rs | 0 .../{ => src}/streaming_table_options.rs | 0 src/coordinator/{ => src}/tool/mod.rs | 0 src/function-stream/Cargo.toml | 86 ++++++ .../mod.rs => function-stream/src/lib.rs} | 31 +++ src/{ => function-stream/src}/main.rs | 71 ++++- src/logger/Cargo.toml | 15 ++ src/logger/src/lib.rs | 9 + src/{logging/mod.rs => logger/src/logging.rs} | 0 src/runtime_common/Cargo.toml | 16 ++ .../src}/common/component_state.rs | 0 .../src}/common/mod.rs | 0 .../src}/common/task_completion.rs | 0 .../mod.rs => runtime_common/src/lib.rs} | 10 +- .../src}/memory/block.rs | 0 .../src}/memory/error.rs | 0 .../src}/memory/global.rs | 0 .../src}/memory/mod.rs | 0 .../src}/memory/pool.rs | 0 .../src}/memory/ticket.rs | 0 .../src/streaming_protocol.rs} | 45 +--- src/servicer/Cargo.toml | 8 + src/{server => servicer/src}/handler.rs | 12 +- src/{server => servicer/src}/initializer.rs | 28 +- src/servicer/src/lib.rs | 3 + .../src}/memory_service.rs | 2 +- src/{server => servicer/src}/service.rs | 0 .../mod.rs => servicer/src/servicer_body.rs} | 0 src/sql/logical_node/mod.rs | 42 --- src/sqlparser/Cargo.toml | 8 + src/sqlparser/src/lib.rs | 3 + src/streaming_planner/Cargo.toml | 49 ++++ .../src}/analysis/aggregate_rewriter.rs | 14 +- .../src}/analysis/async_udf_rewriter.rs | 8 +- .../src}/analysis/join_rewriter.rs | 16 +- .../src}/analysis/mod.rs | 26 +- .../src}/analysis/row_time_rewriter.rs | 4 +- .../src}/analysis/sink_input_rewriter.rs | 8 +- .../src}/analysis/source_metadata_visitor.rs | 12 +- .../src}/analysis/source_rewriter.rs | 20 +- .../src}/analysis/stream_rewriter.rs | 14 +- .../src}/analysis/streaming_window_analzer.rs | 12 +- .../src}/analysis/time_window.rs | 0 .../src}/analysis/udafs.rs | 0 .../src}/analysis/unnest_rewriter.rs | 4 +- .../src}/analysis/window_function_rewriter.rs | 12 +- .../src}/api/checkpoints.rs | 2 +- .../src}/api/connections.rs | 4 +- .../src}/api/metrics.rs | 0 src/{sql => streaming_planner/src}/api/mod.rs | 0 .../src}/api/pipelines.rs | 2 +- .../src}/api/public_ids.rs | 0 .../src}/api/schema_resolver.rs | 0 .../src}/api/udfs.rs | 0 .../src}/api/var_str.rs | 0 .../src}/common/arrow_ext.rs | 0 .../src/common}/ast_utils.rs | 2 +- .../src}/common/connector_options.rs | 0 .../src}/common/constants.rs | 0 .../src}/common/control.rs | 2 +- .../src}/common/converter.rs | 0 .../src}/common/date.rs | 0 .../src}/common/debezium.rs | 0 .../src}/common/errors.rs | 2 +- .../src}/common/format_from_opts.rs | 0 .../src}/common/formats.rs | 0 .../src}/common/fs_schema.rs | 2 +- .../src}/common/kafka_catalog.rs | 0 .../src}/common/mod.rs | 6 +- .../src}/common/operator_config.rs | 0 .../src}/common/time_utils.rs | 0 .../src}/common/topology.rs | 0 .../src}/common/with_option_keys.rs | 0 .../src}/connector/config.rs | 0 .../src}/connector/factory.rs | 6 +- .../src}/connector/mod.rs | 0 .../src}/connector/provider.rs | 4 +- .../src}/connector/registry.rs | 0 .../src}/connector/sink/delta.rs | 14 +- .../src}/connector/sink/filesystem.rs | 14 +- .../src}/connector/sink/iceberg.rs | 14 +- .../src}/connector/sink/kafka.rs | 14 +- .../src}/connector/sink/lancedb.rs | 14 +- .../src}/connector/sink/mod.rs | 0 .../src}/connector/sink/runtime_config.rs | 8 +- .../src}/connector/sink/s3.rs | 16 +- .../src}/connector/sink/utils.rs | 8 +- .../src}/connector/source/kafka.rs | 12 +- .../src/connector}/source/mod.rs | 0 .../src}/functions/mod.rs | 10 +- .../mod.rs => streaming_planner/src/lib.rs} | 9 +- .../src}/logical_node/aggregate.rs | 20 +- .../src}/logical_node/async_udf.rs | 20 +- .../src}/logical_node/debezium.rs | 22 +- .../src}/logical_node/extension_try_from.rs | 28 +- .../src}/logical_node/is_retract.rs | 10 +- .../src}/logical_node/join.rs | 14 +- .../src}/logical_node/key_calculation.rs | 18 +- .../logical_node/logical/dylib_udf_config.rs | 0 .../logical/fs_program_convert.rs | 4 +- .../src}/logical_node/logical/logical_edge.rs | 4 +- .../logical_node/logical/logical_graph.rs | 0 .../src}/logical_node/logical/logical_node.rs | 0 .../logical_node/logical/logical_program.rs | 0 .../src}/logical_node/logical/mod.rs | 0 .../logical_node/logical/operator_chain.rs | 2 +- .../logical_node/logical/operator_name.rs | 2 +- .../logical_node/logical/program_config.rs | 0 .../logical_node/logical/python_udf_config.rs | 0 .../src}/logical_node/lookup.rs | 14 +- .../src}/logical_node/macros.rs | 0 src/streaming_planner/src/logical_node/mod.rs | 42 +++ .../src}/logical_node/projection.rs | 20 +- .../src}/logical_node/remote_table.rs | 16 +- .../src}/logical_node/sink.rs | 18 +- .../streaming_operator_blueprint.rs | 10 +- .../src}/logical_node/table_source.rs | 20 +- .../src}/logical_node/timestamp_append.rs | 10 +- .../src}/logical_node/updating_aggregate.rs | 20 +- .../src}/logical_node/watermark_node.rs | 22 +- .../src}/logical_node/windows_function.rs | 16 +- .../src}/logical_planner/mod.rs | 4 +- .../logical_planner/optimizers/chaining.rs | 6 +- .../src}/logical_planner/optimizers/mod.rs | 0 .../optimizers/optimized_plan.rs | 2 +- .../src}/logical_planner/streaming_planner.rs | 53 ++-- src/streaming_planner/src/parse.rs | 82 ++++++ .../src}/physical/cdc/encode.rs | 8 +- .../src}/physical/cdc/mod.rs | 0 .../src}/physical/cdc/unroll.rs | 12 +- .../src}/physical/codec.rs | 10 +- .../src}/physical/meta.rs | 4 +- .../src}/physical/mod.rs | 0 .../src}/physical/source_exec.rs | 18 +- .../src}/physical/udfs.rs | 4 +- .../src}/planning_runtime.rs | 8 +- .../src}/schema/catalog.rs | 8 +- .../src}/schema/column_descriptor.rs | 0 .../src}/schema/connection_type.rs | 0 .../src}/schema/data_encoding_format.rs | 4 +- .../src/schema/introspection}/ddl_compiler.rs | 48 +++- .../schema/introspection/ddl_formatter.rs | 2 +- .../src}/schema/introspection/mod.rs | 2 + .../schema/introspection/show_formatter.rs | 6 +- .../schema/introspection/stream_formatter.rs | 6 +- .../src}/schema/mod.rs | 3 +- .../src}/schema/schema_provider.rs | 16 +- .../src}/schema/table.rs | 12 +- .../src}/schema/table_role.rs | 4 +- .../src}/schema/temporal_pipeline_config.rs | 2 +- .../src}/schema/utils.rs | 4 +- .../src}/types/data_type.rs | 4 +- .../src}/types/df_field.rs | 0 .../src}/types/mod.rs | 12 +- .../src}/types/placeholder_udf.rs | 2 +- .../src}/types/stream_schema.rs | 0 .../src}/types/window.rs | 4 +- src/streaming_runtime/Cargo.toml | 8 + src/{ => streaming_runtime/src}/lib.rs | 17 +- .../src}/streaming/api/context.rs | 10 +- .../src}/streaming/api/mod.rs | 0 .../src}/streaming/api/operator.rs | 6 +- .../src}/streaming/api/source.rs | 2 +- .../src}/streaming/error.rs | 0 .../src}/streaming/execution/mod.rs | 0 .../streaming/execution/operator_chain.rs | 10 +- .../src}/streaming/execution/pipeline.rs | 16 +- .../src}/streaming/execution/source_driver.rs | 12 +- .../execution/tracker/barrier_aligner.rs | 0 .../src}/streaming/execution/tracker/mod.rs | 0 .../execution/tracker/watermark_tracker.rs | 2 +- .../src}/streaming/factory/connector/delta.rs | 16 +- .../factory/connector/dispatchers.rs | 6 +- .../streaming/factory/connector/filesystem.rs | 14 +- .../streaming/factory/connector/iceberg.rs | 16 +- .../src}/streaming/factory/connector/kafka.rs | 18 +- .../streaming/factory/connector/lancedb.rs | 14 +- .../src}/streaming/factory/connector/mod.rs | 0 .../src}/streaming/factory/connector/s3.rs | 16 +- .../factory/connector/sink_props_codec.rs | 0 .../src}/streaming/factory/global/mod.rs | 0 .../factory/global/session_registry.rs | 0 .../src}/streaming/factory/mod.rs | 0 .../streaming/factory/operator_constructor.rs | 4 +- .../streaming/factory/operator_factory.rs | 22 +- .../src}/streaming/format/config.rs | 0 .../src}/streaming/format/deserializer.rs | 0 .../src}/streaming/format/encoder.rs | 0 .../src}/streaming/format/json_encoder.rs | 0 .../src}/streaming/format/mod.rs | 0 .../src}/streaming/format/serializer.rs | 0 .../src}/streaming/job/edge_manager.rs | 2 +- .../src}/streaming/job/job_manager.rs | 34 +-- .../src}/streaming/job/mod.rs | 0 .../src}/streaming/job/models.rs | 2 +- .../src}/streaming/mod.rs | 0 .../src}/streaming/network/endpoint.rs | 4 +- .../src}/streaming/network/environment.rs | 0 .../src}/streaming/network/mod.rs | 0 .../grouping/incremental_aggregate.rs | 14 +- .../src}/streaming/operators/grouping/mod.rs | 0 .../operators/grouping/updating_cache.rs | 0 .../operators/joins/join_instance.rs | 10 +- .../operators/joins/join_with_expiration.rs | 10 +- .../src}/streaming/operators/joins/mod.rs | 0 .../src}/streaming/operators/key_by.rs | 6 +- .../src}/streaming/operators/key_operator.rs | 10 +- .../src}/streaming/operators/mod.rs | 0 .../src}/streaming/operators/projection.rs | 8 +- .../streaming/operators/sink/delta/mod.rs | 10 +- .../operators/sink/filesystem/mod.rs | 10 +- .../streaming/operators/sink/iceberg/mod.rs | 10 +- .../streaming/operators/sink/kafka/mod.rs | 8 +- .../streaming/operators/sink/lancedb/mod.rs | 6 +- .../src}/streaming/operators/sink/mod.rs | 0 .../src}/streaming/operators/sink/s3/mod.rs | 6 +- .../streaming/operators/source/kafka/mod.rs | 10 +- .../src/streaming/operators/source}/mod.rs | 0 .../operators/stateless_physical_executor.rs | 2 +- .../streaming/operators/value_execution.rs | 8 +- .../src}/streaming/operators/watermark/mod.rs | 0 .../watermark/watermark_generator.rs | 8 +- .../src}/streaming/operators/windows/mod.rs | 0 .../windows/session_aggregating_window.rs | 10 +- .../windows/sliding_aggregating_window.rs | 10 +- .../windows/tumbling_aggregating_window.rs | 10 +- .../operators/windows/window_function.rs | 10 +- .../src}/streaming/protocol/control.rs | 0 .../src/streaming/protocol/event.rs | 59 +++++ .../src}/streaming/protocol/mod.rs | 1 + .../src}/streaming/state/error.rs | 2 +- .../src}/streaming/state/io_manager.rs | 0 .../src}/streaming/state/metrics.rs | 0 .../src}/streaming/state/mod.rs | 0 .../src}/streaming/state/operator_state.rs | 6 +- .../src}/util/mod.rs | 0 .../src}/util/physical_aggregate.rs | 0 src/wasm_runtime/Cargo.toml | 8 + src/wasm_runtime/src/lib.rs | 22 ++ .../src}/state_backend/error.rs | 0 .../src}/state_backend/factory.rs | 13 +- .../src}/state_backend/key_builder.rs | 0 .../src}/state_backend/memory/factory.rs | 6 +- .../src}/state_backend/memory/mod.rs | 0 .../src}/state_backend/memory/store.rs | 9 +- .../src}/state_backend/mod.rs | 0 .../src}/state_backend/rocksdb/factory.rs | 10 +- .../src}/state_backend/rocksdb/mod.rs | 0 .../src}/state_backend/rocksdb/store.rs | 6 +- .../src}/state_backend/server.rs | 8 +- .../src}/state_backend/store.rs | 37 +-- .../wasm/buffer_and_event/buffer_or_event.rs | 0 .../src}/wasm/buffer_and_event/mod.rs | 0 .../buffer_and_event/stream_element/mod.rs | 0 .../stream_element/stream_element.rs | 0 .../src}/wasm/input/input_protocol.rs | 2 +- .../src}/wasm/input/input_provider.rs | 8 +- .../src}/wasm/input/input_runner.rs | 18 +- .../src}/wasm/input/interface.rs | 6 +- .../src}/wasm/input/mod.rs | 0 .../src}/wasm/input/protocol/kafka/config.rs | 0 .../input/protocol/kafka/kafka_protocol.rs | 4 +- .../src}/wasm/input/protocol/kafka/mod.rs | 0 .../src/wasm/input/protocol}/mod.rs | 0 src/{runtime => wasm_runtime/src}/wasm/mod.rs | 0 .../src}/wasm/output/interface.rs | 4 +- .../src}/wasm/output/mod.rs | 0 .../src}/wasm/output/output_protocol.rs | 2 +- .../src}/wasm/output/output_provider.rs | 10 +- .../src}/wasm/output/output_runner.rs | 18 +- .../output/protocol/kafka/kafka_protocol.rs | 4 +- .../src}/wasm/output/protocol/kafka/mod.rs | 0 .../output/protocol/kafka/producer_config.rs | 0 .../src}/wasm/output/protocol/mod.rs | 0 .../src}/wasm/processor/function_error.rs | 0 .../src}/wasm/processor/mod.rs | 0 .../src}/wasm/processor/python/mod.rs | 0 .../src}/wasm/processor/python/python_host.rs | 0 .../wasm/processor/python/python_service.rs | 0 .../wasm/processor/wasm/input_strategy.rs | 0 .../src}/wasm/processor/wasm/mod.rs | 0 .../src}/wasm/processor/wasm/thread_pool.rs | 4 +- .../src}/wasm/processor/wasm/wasm_cache.rs | 0 .../src}/wasm/processor/wasm/wasm_host.rs | 28 +- .../wasm/processor/wasm/wasm_processor.rs | 6 +- .../processor/wasm/wasm_processor_trait.rs | 4 +- .../src}/wasm/processor/wasm/wasm_task.rs | 24 +- .../src}/wasm/task/builder/mod.rs | 0 .../src}/wasm/task/builder/processor/mod.rs | 14 +- .../src}/wasm/task/builder/python/mod.rs | 19 +- .../src}/wasm/task/builder/sink/mod.rs | 4 +- .../src}/wasm/task/builder/source/mod.rs | 4 +- .../src}/wasm/task/builder/task_builder.rs | 14 +- .../src}/wasm/task/control_mailbox.rs | 4 +- .../src}/wasm/task/lifecycle.rs | 8 +- .../src}/wasm/task/mod.rs | 0 .../src}/wasm/task/processor_config.rs | 2 +- .../src}/wasm/task/yaml_keys.rs | 0 .../src}/wasm/taskexecutor/init_context.rs | 8 +- .../src}/wasm/taskexecutor/mod.rs | 0 .../src}/wasm/taskexecutor/task_manager.rs | 14 +- 402 files changed, 1967 insertions(+), 1257 deletions(-) create mode 100644 src/catalog/Cargo.toml create mode 100644 src/catalog/src/error.rs create mode 100644 src/catalog/src/lib.rs create mode 100644 src/catalog/src/meta_store.rs create mode 100644 src/catalog/src/stream_catalog.rs create mode 100644 src/catalog_storage/Cargo.toml create mode 100644 src/catalog_storage/src/lib.rs create mode 100644 src/catalog_storage/src/memory.rs create mode 100644 src/catalog_storage/src/rocksdb.rs rename src/{storage => catalog_storage/src}/stream_catalog/codec.rs (100%) rename src/{storage => catalog_storage/src}/stream_catalog/manager.rs (99%) rename src/{storage => catalog_storage/src}/stream_catalog/meta_store.rs (100%) rename src/{storage => catalog_storage/src}/stream_catalog/mod.rs (100%) rename src/{storage => catalog_storage/src}/stream_catalog/rocksdb_meta_store.rs (98%) rename src/{storage => catalog_storage/src}/task/factory.rs (100%) rename src/{storage => catalog_storage/src}/task/function_info.rs (100%) rename src/{storage => catalog_storage/src}/task/mod.rs (100%) rename src/{storage => catalog_storage/src}/task/proto_codec.rs (99%) rename src/{storage => catalog_storage/src}/task/rocksdb_storage.rs (99%) rename src/{storage => catalog_storage/src}/task/storage.rs (97%) create mode 100644 src/common/Cargo.toml rename src/common/{ => src}/fs_schema.rs (100%) rename src/common/{mod.rs => src/legacy_mod.rs} (100%) create mode 100644 src/common/src/lib.rs create mode 100644 src/config/Cargo.toml rename src/config/{ => src}/global_config.rs (100%) rename src/config/{mod.rs => src/lib.rs} (61%) rename src/config/{ => src}/loader.rs (100%) rename src/config/{ => src}/log_config.rs (100%) rename src/config/{ => src}/paths.rs (100%) rename src/config/{ => src}/python_config.rs (100%) rename src/config/{ => src}/service_config.rs (100%) rename src/config/{ => src}/storage.rs (100%) rename src/config/{ => src}/streaming_job.rs (100%) rename src/config/{ => src}/system.rs (100%) rename src/config/{ => src}/wasm_config.rs (100%) create mode 100644 src/coordinator/Cargo.toml rename src/coordinator/{ => src}/analyze/analysis.rs (100%) rename src/coordinator/{ => src}/analyze/analyzer.rs (100%) rename src/coordinator/{ => src}/analyze/mod.rs (100%) rename src/coordinator/{ => src}/coordinator.rs (100%) rename src/coordinator/{mod.rs => src/coordinator_body.rs} (95%) rename src/coordinator/{ => src}/dataset/data_set.rs (100%) rename src/coordinator/{ => src}/dataset/execute_result.rs (100%) rename src/coordinator/{ => src}/dataset/mod.rs (100%) rename src/coordinator/{ => src}/dataset/show_catalog_tables_result.rs (100%) rename src/coordinator/{ => src}/dataset/show_create_streaming_table_result.rs (100%) rename src/coordinator/{ => src}/dataset/show_create_table_result.rs (100%) rename src/coordinator/{ => src}/dataset/show_functions_result.rs (98%) rename src/coordinator/{ => src}/dataset/show_streaming_tables_result.rs (97%) rename src/coordinator/{ => src}/execution/executor.rs (98%) rename src/coordinator/{ => src}/execution/mod.rs (100%) rename src/coordinator/{ => src}/execution_context.rs (100%) create mode 100644 src/coordinator/src/lib.rs rename src/coordinator/{ => src}/plan/compile_error_plan.rs (100%) rename src/coordinator/{ => src}/plan/create_function_plan.rs (100%) rename src/coordinator/{ => src}/plan/create_python_function_plan.rs (100%) rename src/coordinator/{ => src}/plan/create_table_plan.rs (100%) rename src/coordinator/{ => src}/plan/drop_function_plan.rs (100%) rename src/coordinator/{ => src}/plan/drop_streaming_table_plan.rs (100%) rename src/coordinator/{ => src}/plan/drop_table_plan.rs (100%) rename src/coordinator/{ => src}/plan/logical_plan_visitor.rs (92%) rename src/coordinator/{ => src}/plan/lookup_table_plan.rs (100%) rename src/coordinator/{ => src}/plan/mod.rs (98%) rename src/coordinator/{ => src}/plan/optimizer.rs (100%) rename src/coordinator/{ => src}/plan/show_catalog_tables_plan.rs (100%) rename src/coordinator/{ => src}/plan/show_create_streaming_table_plan.rs (100%) rename src/coordinator/{ => src}/plan/show_create_table_plan.rs (100%) rename src/coordinator/{ => src}/plan/show_functions_plan.rs (100%) rename src/coordinator/{ => src}/plan/show_streaming_tables_plan.rs (100%) rename src/coordinator/{ => src}/plan/start_function_plan.rs (100%) rename src/coordinator/{ => src}/plan/stop_function_plan.rs (100%) rename src/coordinator/{ => src}/plan/streaming_compiler.rs (100%) rename src/coordinator/{ => src}/plan/streaming_table_connector_plan.rs (100%) rename src/coordinator/{ => src}/plan/streaming_table_plan.rs (100%) rename src/coordinator/{ => src}/plan/visitor.rs (100%) rename src/coordinator/{ => src}/runtime_context.rs (93%) rename src/{sql/parse.rs => coordinator/src/sql_classify.rs} (51%) rename src/coordinator/{ => src}/statement/create_function.rs (100%) rename src/coordinator/{ => src}/statement/create_python_function.rs (100%) rename src/coordinator/{ => src}/statement/create_table.rs (100%) rename src/coordinator/{ => src}/statement/drop_function.rs (100%) rename src/coordinator/{ => src}/statement/drop_streaming_table.rs (100%) rename src/coordinator/{ => src}/statement/drop_table.rs (100%) rename src/coordinator/{ => src}/statement/mod.rs (100%) rename src/coordinator/{ => src}/statement/show_catalog_tables.rs (100%) rename src/coordinator/{ => src}/statement/show_create_streaming_table.rs (100%) rename src/coordinator/{ => src}/statement/show_create_table.rs (100%) rename src/coordinator/{ => src}/statement/show_functions.rs (100%) rename src/coordinator/{ => src}/statement/show_streaming_tables.rs (100%) rename src/coordinator/{ => src}/statement/start_function.rs (100%) rename src/coordinator/{ => src}/statement/stop_function.rs (100%) rename src/coordinator/{ => src}/statement/streaming_table.rs (100%) rename src/coordinator/{ => src}/statement/visitor.rs (100%) rename src/coordinator/{ => src}/streaming_table_options.rs (100%) rename src/coordinator/{ => src}/tool/mod.rs (100%) create mode 100644 src/function-stream/Cargo.toml rename src/{storage/mod.rs => function-stream/src/lib.rs} (70%) rename src/{ => function-stream/src}/main.rs (77%) create mode 100644 src/logger/Cargo.toml create mode 100644 src/logger/src/lib.rs rename src/{logging/mod.rs => logger/src/logging.rs} (100%) create mode 100644 src/runtime_common/Cargo.toml rename src/{runtime => runtime_common/src}/common/component_state.rs (100%) rename src/{runtime => runtime_common/src}/common/mod.rs (100%) rename src/{runtime => runtime_common/src}/common/task_completion.rs (100%) rename src/{runtime/mod.rs => runtime_common/src/lib.rs} (81%) rename src/{runtime => runtime_common/src}/memory/block.rs (100%) rename src/{runtime => runtime_common/src}/memory/error.rs (100%) rename src/{runtime => runtime_common/src}/memory/global.rs (100%) rename src/{runtime => runtime_common/src}/memory/mod.rs (100%) rename src/{runtime => runtime_common/src}/memory/pool.rs (100%) rename src/{runtime => runtime_common/src}/memory/ticket.rs (100%) rename src/{runtime/streaming/protocol/event.rs => runtime_common/src/streaming_protocol.rs} (79%) create mode 100644 src/servicer/Cargo.toml rename src/{server => servicer/src}/handler.rs (96%) rename src/{server => servicer/src}/initializer.rs (87%) create mode 100644 src/servicer/src/lib.rs rename src/{server => servicer/src}/memory_service.rs (97%) rename src/{server => servicer/src}/service.rs (100%) rename src/{server/mod.rs => servicer/src/servicer_body.rs} (100%) delete mode 100644 src/sql/logical_node/mod.rs create mode 100644 src/sqlparser/Cargo.toml create mode 100644 src/sqlparser/src/lib.rs create mode 100644 src/streaming_planner/Cargo.toml rename src/{sql => streaming_planner/src}/analysis/aggregate_rewriter.rs (96%) rename src/{sql => streaming_planner/src}/analysis/async_udf_rewriter.rs (95%) rename src/{sql => streaming_planner/src}/analysis/join_rewriter.rs (94%) rename src/{sql => streaming_planner/src}/analysis/mod.rs (90%) rename src/{sql => streaming_planner/src}/analysis/row_time_rewriter.rs (95%) rename src/{sql => streaming_planner/src}/analysis/sink_input_rewriter.rs (89%) rename src/{sql => streaming_planner/src}/analysis/source_metadata_visitor.rs (82%) rename src/{sql => streaming_planner/src}/analysis/source_rewriter.rs (95%) rename src/{sql => streaming_planner/src}/analysis/stream_rewriter.rs (96%) rename src/{sql => streaming_planner/src}/analysis/streaming_window_analzer.rs (95%) rename src/{sql => streaming_planner/src}/analysis/time_window.rs (100%) rename src/{sql => streaming_planner/src}/analysis/udafs.rs (100%) rename src/{sql => streaming_planner/src}/analysis/unnest_rewriter.rs (97%) rename src/{sql => streaming_planner/src}/analysis/window_function_rewriter.rs (93%) rename src/{sql => streaming_planner/src}/api/checkpoints.rs (98%) rename src/{sql => streaming_planner/src}/api/connections.rs (99%) rename src/{sql => streaming_planner/src}/api/metrics.rs (100%) rename src/{sql => streaming_planner/src}/api/mod.rs (100%) rename src/{sql => streaming_planner/src}/api/pipelines.rs (98%) rename src/{sql => streaming_planner/src}/api/public_ids.rs (100%) rename src/{sql => streaming_planner/src}/api/schema_resolver.rs (100%) rename src/{sql => streaming_planner/src}/api/udfs.rs (100%) rename src/{sql => streaming_planner/src}/api/var_str.rs (100%) rename src/{sql => streaming_planner/src}/common/arrow_ext.rs (100%) rename src/{coordinator/plan => streaming_planner/src/common}/ast_utils.rs (98%) rename src/{sql => streaming_planner/src}/common/connector_options.rs (100%) rename src/{sql => streaming_planner/src}/common/constants.rs (100%) rename src/{sql => streaming_planner/src}/common/control.rs (98%) rename src/{sql => streaming_planner/src}/common/converter.rs (100%) rename src/{sql => streaming_planner/src}/common/date.rs (100%) rename src/{sql => streaming_planner/src}/common/debezium.rs (100%) rename src/{sql => streaming_planner/src}/common/errors.rs (97%) rename src/{sql => streaming_planner/src}/common/format_from_opts.rs (100%) rename src/{sql => streaming_planner/src}/common/formats.rs (100%) rename src/{sql => streaming_planner/src}/common/fs_schema.rs (99%) rename src/{sql => streaming_planner/src}/common/kafka_catalog.rs (100%) rename src/{sql => streaming_planner/src}/common/mod.rs (90%) rename src/{sql => streaming_planner/src}/common/operator_config.rs (100%) rename src/{sql => streaming_planner/src}/common/time_utils.rs (100%) rename src/{sql => streaming_planner/src}/common/topology.rs (100%) rename src/{sql => streaming_planner/src}/common/with_option_keys.rs (100%) rename src/{sql => streaming_planner/src}/connector/config.rs (100%) rename src/{sql => streaming_planner/src}/connector/factory.rs (93%) rename src/{sql => streaming_planner/src}/connector/mod.rs (100%) rename src/{sql => streaming_planner/src}/connector/provider.rs (93%) rename src/{sql => streaming_planner/src}/connector/registry.rs (100%) rename src/{sql => streaming_planner/src}/connector/sink/delta.rs (83%) rename src/{sql => streaming_planner/src}/connector/sink/filesystem.rs (83%) rename src/{sql => streaming_planner/src}/connector/sink/iceberg.rs (82%) rename src/{sql => streaming_planner/src}/connector/sink/kafka.rs (94%) rename src/{sql => streaming_planner/src}/connector/sink/lancedb.rs (84%) rename src/{sql => streaming_planner/src}/connector/sink/mod.rs (100%) rename src/{sql => streaming_planner/src}/connector/sink/runtime_config.rs (95%) rename src/{sql => streaming_planner/src}/connector/sink/s3.rs (84%) rename src/{sql => streaming_planner/src}/connector/sink/utils.rs (94%) rename src/{sql => streaming_planner/src}/connector/source/kafka.rs (95%) rename src/{runtime/streaming/operators => streaming_planner/src/connector}/source/mod.rs (100%) rename src/{sql => streaming_planner/src}/functions/mod.rs (98%) rename src/{sql/mod.rs => streaming_planner/src/lib.rs} (68%) rename src/{sql => streaming_planner/src}/logical_node/aggregate.rs (97%) rename src/{sql => streaming_planner/src}/logical_node/async_udf.rs (93%) rename src/{sql => streaming_planner/src}/logical_node/debezium.rs (95%) rename src/{sql => streaming_planner/src}/logical_node/extension_try_from.rs (72%) rename src/{sql => streaming_planner/src}/logical_node/is_retract.rs (89%) rename src/{sql => streaming_planner/src}/logical_node/join.rs (93%) rename src/{sql => streaming_planner/src}/logical_node/key_calculation.rs (94%) rename src/{sql => streaming_planner/src}/logical_node/logical/dylib_udf_config.rs (100%) rename src/{sql => streaming_planner/src}/logical_node/logical/fs_program_convert.rs (98%) rename src/{sql => streaming_planner/src}/logical_node/logical/logical_edge.rs (96%) rename src/{sql => streaming_planner/src}/logical_node/logical/logical_graph.rs (100%) rename src/{sql => streaming_planner/src}/logical_node/logical/logical_node.rs (100%) rename src/{sql => streaming_planner/src}/logical_node/logical/logical_program.rs (100%) rename src/{sql => streaming_planner/src}/logical_node/logical/mod.rs (100%) rename src/{sql => streaming_planner/src}/logical_node/logical/operator_chain.rs (99%) rename src/{sql => streaming_planner/src}/logical_node/logical/operator_name.rs (98%) rename src/{sql => streaming_planner/src}/logical_node/logical/program_config.rs (100%) rename src/{sql => streaming_planner/src}/logical_node/logical/python_udf_config.rs (100%) rename src/{sql => streaming_planner/src}/logical_node/lookup.rs (95%) rename src/{sql => streaming_planner/src}/logical_node/macros.rs (100%) create mode 100644 src/streaming_planner/src/logical_node/mod.rs rename src/{sql => streaming_planner/src}/logical_node/projection.rs (92%) rename src/{sql => streaming_planner/src}/logical_node/remote_table.rs (92%) rename src/{sql => streaming_planner/src}/logical_node/sink.rs (94%) rename src/{sql => streaming_planner/src}/logical_node/streaming_operator_blueprint.rs (88%) rename src/{sql => streaming_planner/src}/logical_node/table_source.rs (91%) rename src/{sql => streaming_planner/src}/logical_node/timestamp_append.rs (92%) rename src/{sql => streaming_planner/src}/logical_node/updating_aggregate.rs (92%) rename src/{sql => streaming_planner/src}/logical_node/watermark_node.rs (91%) rename src/{sql => streaming_planner/src}/logical_node/windows_function.rs (92%) rename src/{sql => streaming_planner/src}/logical_planner/mod.rs (87%) rename src/{sql => streaming_planner/src}/logical_planner/optimizers/chaining.rs (97%) rename src/{sql => streaming_planner/src}/logical_planner/optimizers/mod.rs (100%) rename src/{sql => streaming_planner/src}/logical_planner/optimizers/optimized_plan.rs (98%) rename src/{sql => streaming_planner/src}/logical_planner/streaming_planner.rs (90%) create mode 100644 src/streaming_planner/src/parse.rs rename src/{sql => streaming_planner/src}/physical/cdc/encode.rs (97%) rename src/{sql => streaming_planner/src}/physical/cdc/mod.rs (100%) rename src/{sql => streaming_planner/src}/physical/cdc/unroll.rs (96%) rename src/{sql => streaming_planner/src}/physical/codec.rs (97%) rename src/{sql => streaming_planner/src}/physical/meta.rs (93%) rename src/{sql => streaming_planner/src}/physical/mod.rs (100%) rename src/{sql => streaming_planner/src}/physical/source_exec.rs (96%) rename src/{sql => streaming_planner/src}/physical/udfs.rs (97%) rename src/{sql => streaming_planner/src}/planning_runtime.rs (85%) rename src/{sql => streaming_planner/src}/schema/catalog.rs (99%) rename src/{sql => streaming_planner/src}/schema/column_descriptor.rs (100%) rename src/{sql => streaming_planner/src}/schema/connection_type.rs (100%) rename src/{sql => streaming_planner/src}/schema/data_encoding_format.rs (97%) rename src/{coordinator/plan => streaming_planner/src/schema/introspection}/ddl_compiler.rs (89%) rename src/{sql => streaming_planner/src}/schema/introspection/ddl_formatter.rs (99%) rename src/{sql => streaming_planner/src}/schema/introspection/mod.rs (90%) rename src/{sql => streaming_planner/src}/schema/introspection/show_formatter.rs (96%) rename src/{sql => streaming_planner/src}/schema/introspection/stream_formatter.rs (95%) rename src/{sql => streaming_planner/src}/schema/mod.rs (94%) rename src/{sql => streaming_planner/src}/schema/schema_provider.rs (96%) rename src/{sql => streaming_planner/src}/schema/table.rs (94%) rename src/{sql => streaming_planner/src}/schema/table_role.rs (97%) rename src/{sql => streaming_planner/src}/schema/temporal_pipeline_config.rs (97%) rename src/{sql => streaming_planner/src}/schema/utils.rs (95%) rename src/{sql => streaming_planner/src}/types/data_type.rs (98%) rename src/{sql => streaming_planner/src}/types/df_field.rs (100%) rename src/{sql => streaming_planner/src}/types/mod.rs (83%) rename src/{sql => streaming_planner/src}/types/placeholder_udf.rs (98%) rename src/{sql => streaming_planner/src}/types/stream_schema.rs (100%) rename src/{sql => streaming_planner/src}/types/window.rs (98%) create mode 100644 src/streaming_runtime/Cargo.toml rename src/{ => streaming_runtime/src}/lib.rs (54%) rename src/{runtime => streaming_runtime/src}/streaming/api/context.rs (96%) rename src/{runtime => streaming_runtime/src}/streaming/api/mod.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/api/operator.rs (93%) rename src/{runtime => streaming_runtime/src}/streaming/api/source.rs (98%) rename src/{runtime => streaming_runtime/src}/streaming/error.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/execution/mod.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/execution/operator_chain.rs (98%) rename src/{runtime => streaming_runtime/src}/streaming/execution/pipeline.rs (94%) rename src/{runtime => streaming_runtime/src}/streaming/execution/source_driver.rs (95%) rename src/{runtime => streaming_runtime/src}/streaming/execution/tracker/barrier_aligner.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/execution/tracker/mod.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/execution/tracker/watermark_tracker.rs (97%) rename src/{runtime => streaming_runtime/src}/streaming/factory/connector/delta.rs (87%) rename src/{runtime => streaming_runtime/src}/streaming/factory/connector/dispatchers.rs (92%) rename src/{runtime => streaming_runtime/src}/streaming/factory/connector/filesystem.rs (89%) rename src/{runtime => streaming_runtime/src}/streaming/factory/connector/iceberg.rs (86%) rename src/{runtime => streaming_runtime/src}/streaming/factory/connector/kafka.rs (94%) rename src/{runtime => streaming_runtime/src}/streaming/factory/connector/lancedb.rs (93%) rename src/{runtime => streaming_runtime/src}/streaming/factory/connector/mod.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/factory/connector/s3.rs (89%) rename src/{runtime => streaming_runtime/src}/streaming/factory/connector/sink_props_codec.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/factory/global/mod.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/factory/global/session_registry.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/factory/mod.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/factory/operator_constructor.rs (86%) rename src/{runtime => streaming_runtime/src}/streaming/factory/operator_factory.rs (93%) rename src/{runtime => streaming_runtime/src}/streaming/format/config.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/format/deserializer.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/format/encoder.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/format/json_encoder.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/format/mod.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/format/serializer.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/job/edge_manager.rs (98%) rename src/{runtime => streaming_runtime/src}/streaming/job/job_manager.rs (96%) rename src/{runtime => streaming_runtime/src}/streaming/job/mod.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/job/models.rs (98%) rename src/{runtime => streaming_runtime/src}/streaming/mod.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/network/endpoint.rs (94%) rename src/{runtime => streaming_runtime/src}/streaming/network/environment.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/network/mod.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/operators/grouping/incremental_aggregate.rs (99%) rename src/{runtime => streaming_runtime/src}/streaming/operators/grouping/mod.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/operators/grouping/updating_cache.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/operators/joins/join_instance.rs (98%) rename src/{runtime => streaming_runtime/src}/streaming/operators/joins/join_with_expiration.rs (97%) rename src/{runtime => streaming_runtime/src}/streaming/operators/joins/mod.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/operators/key_by.rs (96%) rename src/{runtime => streaming_runtime/src}/streaming/operators/key_operator.rs (94%) rename src/{runtime => streaming_runtime/src}/streaming/operators/mod.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/operators/projection.rs (94%) rename src/{runtime => streaming_runtime/src}/streaming/operators/sink/delta/mod.rs (96%) rename src/{runtime => streaming_runtime/src}/streaming/operators/sink/filesystem/mod.rs (96%) rename src/{runtime => streaming_runtime/src}/streaming/operators/sink/iceberg/mod.rs (96%) rename src/{runtime => streaming_runtime/src}/streaming/operators/sink/kafka/mod.rs (98%) rename src/{runtime => streaming_runtime/src}/streaming/operators/sink/lancedb/mod.rs (96%) rename src/{runtime => streaming_runtime/src}/streaming/operators/sink/mod.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/operators/sink/s3/mod.rs (97%) rename src/{runtime => streaming_runtime/src}/streaming/operators/source/kafka/mod.rs (98%) rename src/{runtime/wasm/input/protocol => streaming_runtime/src/streaming/operators/source}/mod.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/operators/stateless_physical_executor.rs (98%) rename src/{runtime => streaming_runtime/src}/streaming/operators/value_execution.rs (89%) rename src/{runtime => streaming_runtime/src}/streaming/operators/watermark/mod.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/operators/watermark/watermark_generator.rs (96%) rename src/{runtime => streaming_runtime/src}/streaming/operators/windows/mod.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/operators/windows/session_aggregating_window.rs (99%) rename src/{runtime => streaming_runtime/src}/streaming/operators/windows/sliding_aggregating_window.rs (98%) rename src/{runtime => streaming_runtime/src}/streaming/operators/windows/tumbling_aggregating_window.rs (98%) rename src/{runtime => streaming_runtime/src}/streaming/operators/windows/window_function.rs (97%) rename src/{runtime => streaming_runtime/src}/streaming/protocol/control.rs (100%) create mode 100644 src/streaming_runtime/src/streaming/protocol/event.rs rename src/{runtime => streaming_runtime/src}/streaming/protocol/mod.rs (96%) rename src/{runtime => streaming_runtime/src}/streaming/state/error.rs (97%) rename src/{runtime => streaming_runtime/src}/streaming/state/io_manager.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/state/metrics.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/state/mod.rs (100%) rename src/{runtime => streaming_runtime/src}/streaming/state/operator_state.rs (99%) rename src/{runtime => streaming_runtime/src}/util/mod.rs (100%) rename src/{runtime => streaming_runtime/src}/util/physical_aggregate.rs (100%) create mode 100644 src/wasm_runtime/Cargo.toml create mode 100644 src/wasm_runtime/src/lib.rs rename src/{storage => wasm_runtime/src}/state_backend/error.rs (100%) rename src/{storage => wasm_runtime/src}/state_backend/factory.rs (82%) rename src/{storage => wasm_runtime/src}/state_backend/key_builder.rs (100%) rename src/{storage => wasm_runtime/src}/state_backend/memory/factory.rs (86%) rename src/{storage => wasm_runtime/src}/state_backend/memory/mod.rs (100%) rename src/{storage => wasm_runtime/src}/state_backend/memory/store.rs (95%) rename src/{storage => wasm_runtime/src}/state_backend/mod.rs (100%) rename src/{storage => wasm_runtime/src}/state_backend/rocksdb/factory.rs (94%) rename src/{storage => wasm_runtime/src}/state_backend/rocksdb/mod.rs (100%) rename src/{storage => wasm_runtime/src}/state_backend/rocksdb/store.rs (97%) rename src/{storage => wasm_runtime/src}/state_backend/server.rs (92%) rename src/{storage => wasm_runtime/src}/state_backend/store.rs (88%) rename src/{runtime => wasm_runtime/src}/wasm/buffer_and_event/buffer_or_event.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/buffer_and_event/mod.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/buffer_and_event/stream_element/mod.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/buffer_and_event/stream_element/stream_element.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/input/input_protocol.rs (95%) rename src/{runtime => wasm_runtime/src}/wasm/input/input_provider.rs (94%) rename src/{runtime => wasm_runtime/src}/wasm/input/input_runner.rs (96%) rename src/{runtime => wasm_runtime/src}/wasm/input/interface.rs (89%) rename src/{runtime => wasm_runtime/src}/wasm/input/mod.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/input/protocol/kafka/config.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/input/protocol/kafka/kafka_protocol.rs (97%) rename src/{runtime => wasm_runtime/src}/wasm/input/protocol/kafka/mod.rs (100%) rename src/{sql/connector/source => wasm_runtime/src/wasm/input/protocol}/mod.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/mod.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/output/interface.rs (93%) rename src/{runtime => wasm_runtime/src}/wasm/output/mod.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/output/output_protocol.rs (95%) rename src/{runtime => wasm_runtime/src}/wasm/output/output_provider.rs (91%) rename src/{runtime => wasm_runtime/src}/wasm/output/output_runner.rs (96%) rename src/{runtime => wasm_runtime/src}/wasm/output/protocol/kafka/kafka_protocol.rs (96%) rename src/{runtime => wasm_runtime/src}/wasm/output/protocol/kafka/mod.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/output/protocol/kafka/producer_config.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/output/protocol/mod.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/processor/function_error.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/processor/mod.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/processor/python/mod.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/processor/python/python_host.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/processor/python/python_service.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/processor/wasm/input_strategy.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/processor/wasm/mod.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/processor/wasm/thread_pool.rs (99%) rename src/{runtime => wasm_runtime/src}/wasm/processor/wasm/wasm_cache.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/processor/wasm/wasm_host.rs (94%) rename src/{runtime => wasm_runtime/src}/wasm/processor/wasm/wasm_processor.rs (99%) rename src/{runtime => wasm_runtime/src}/wasm/processor/wasm/wasm_processor_trait.rs (97%) rename src/{runtime => wasm_runtime/src}/wasm/processor/wasm/wasm_task.rs (97%) rename src/{runtime => wasm_runtime/src}/wasm/task/builder/mod.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/task/builder/processor/mod.rs (92%) rename src/{runtime => wasm_runtime/src}/wasm/task/builder/python/mod.rs (89%) rename src/{runtime => wasm_runtime/src}/wasm/task/builder/sink/mod.rs (95%) rename src/{runtime => wasm_runtime/src}/wasm/task/builder/source/mod.rs (95%) rename src/{runtime => wasm_runtime/src}/wasm/task/builder/task_builder.rs (93%) rename src/{runtime => wasm_runtime/src}/wasm/task/control_mailbox.rs (97%) rename src/{runtime => wasm_runtime/src}/wasm/task/lifecycle.rs (96%) rename src/{runtime => wasm_runtime/src}/wasm/task/mod.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/task/processor_config.rs (99%) rename src/{runtime => wasm_runtime/src}/wasm/task/yaml_keys.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/taskexecutor/init_context.rs (90%) rename src/{runtime => wasm_runtime/src}/wasm/taskexecutor/mod.rs (100%) rename src/{runtime => wasm_runtime/src}/wasm/taskexecutor/task_manager.rs (96%) diff --git a/.gitignore b/.gitignore index 5dffeb0f..62f1dfb1 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,7 @@ python/**/target/ tests/integration/target/ tests/integration/.venv/ tests/integration/install + +# Python bytecode (do not commit) +**/__pycache__/ +*.py[cod] diff --git a/Cargo.lock b/Cargo.lock index 18b01c0d..a1176666 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3800,6 +3800,10 @@ dependencies = [ "datafusion-expr 48.0.1", "datafusion-physical-expr 48.0.1", "datafusion-proto", + "function-stream-config", + "function-stream-logger", + "function-stream-runtime-common", + "function-stream-streaming-planner", "futures", "governor", "itertools 0.14.0", @@ -3838,9 +3842,26 @@ dependencies = [ "xxhash-rust", ] +[[package]] +name = "function-stream-catalog" +version = "0.6.0" +dependencies = [ + "protocol", +] + +[[package]] +name = "function-stream-catalog-storage" +version = "0.6.0" +dependencies = [ + "anyhow", + "function-stream-catalog", + "parking_lot", + "rocksdb", +] + [[package]] name = "function-stream-cli" -version = "0.1.0" +version = "0.6.0" dependencies = [ "arrow-array 52.2.0", "arrow-ipc 52.2.0", @@ -3853,6 +3874,104 @@ dependencies = [ "tonic", ] +[[package]] +name = "function-stream-common" +version = "0.6.0" + +[[package]] +name = "function-stream-config" +version = "0.6.0" +dependencies = [ + "serde", + "serde_yaml", + "uuid", +] + +[[package]] +name = "function-stream-coordinator" +version = "0.6.0" + +[[package]] +name = "function-stream-logger" +version = "0.6.0" +dependencies = [ + "anyhow", + "function-stream-config", + "tracing", + "tracing-appender", + "tracing-subscriber", +] + +[[package]] +name = "function-stream-runtime-common" +version = "0.6.0" +dependencies = [ + "arrow-array 55.2.0", + "bincode", + "parking_lot", + "serde", + "tokio", + "tracing", +] + +[[package]] +name = "function-stream-servicer" +version = "0.6.0" + +[[package]] +name = "function-stream-sqlparser" +version = "0.6.0" + +[[package]] +name = "function-stream-streaming-planner" +version = "0.6.0" +dependencies = [ + "ahash", + "anyhow", + "apache-avro", + "arrow 55.2.0", + "arrow-array 55.2.0", + "arrow-json 55.2.0", + "arrow-schema 55.2.0", + "async-trait", + "bincode", + "bytes", + "chrono", + "datafusion 48.0.1", + "datafusion-common 48.0.1", + "datafusion-execution 48.0.1", + "datafusion-expr 48.0.1", + "datafusion-physical-expr 48.0.1", + "datafusion-proto", + "function-stream-config", + "function-stream-runtime-common", + "futures", + "itertools 0.14.0", + "petgraph 0.7.1", + "prost 0.13.5", + "protocol", + "rand 0.8.5", + "serde", + "serde_json", + "serde_json_path", + "sqlparser 0.55.0", + "strum 0.26.3", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tracing", + "unicase", + "xxhash-rust", +] + +[[package]] +name = "function-stream-streaming-runtime" +version = "0.6.0" + +[[package]] +name = "function-stream-wasm-runtime" +version = "0.6.0" + [[package]] name = "funty" version = "2.0.0" @@ -6431,7 +6550,7 @@ dependencies = [ [[package]] name = "protocol" -version = "0.1.0" +version = "0.6.0" dependencies = [ "env_logger", "log", diff --git a/Cargo.toml b/Cargo.toml index eebf2a6c..324c882b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,90 +1,25 @@ [workspace] members = [ - ".", + "src/function-stream", "protocol", "cli/cli", + "src/catalog", + "src/catalog_storage", + "src/common", + "src/config", + "src/coordinator", + "src/logger", + "src/runtime_common", + "src/servicer", + "src/sqlparser", + "src/streaming_runtime", + "src/streaming_planner", + "src/wasm_runtime", ] +resolver = "2" +default-members = ["src/function-stream"] -[package] -name = "function-stream" +# Shared crate metadata for workspace members (`version.workspace = true`, etc.). +[workspace.package] version = "0.6.0" edition = "2024" - -[lib] -name = "function_stream" -path = "src/lib.rs" - -[[bin]] -name = "function-stream" -path = "src/main.rs" - - -[dependencies] -tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "sync", "time", "net", "signal"] } -serde = { version = "1.0", features = ["derive"] } -serde_yaml = "0.9" -serde_json = "1.0" -uuid = { version = "1.0", features = ["v4", "v7"] } -log = "0.4" -tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } -tracing-appender = "0.2" -anyhow = "1.0" -thiserror = "2" -tonic = { version = "0.12", features = ["default"] } -async-trait = "0.1" -num_cpus = "1.0" -protocol = { path = "./protocol" } -prost = "0.13" -rdkafka = { version = "0.38", features = ["cmake-build", "ssl", "gssapi", "curl"] } -crossbeam-channel = "0.5" -wasmtime = { version = "41.0.3", features = ["component-model", "async"] } -base64 = "0.22" -wasmtime-wasi = "41.0.3" -rocksdb = { version = "0.21", features = ["multi-threaded-cf", "lz4"] } -bincode = { version = "2", features = ["serde"] } -chrono = "0.4" -tokio-stream = "0.1.18" -lru = "0.12" -parking_lot = "0.12" -arrow = { version = "55", default-features = false } -arrow-array = "55" -arrow-ipc = "55" -arrow-schema = { version = "55", features = ["serde"] } -parquet = "55" -object_store = { version = "0.12.5", features = ["aws"] } -bytes = "1" -futures = "0.3" -serde_json_path = "0.7" -xxhash-rust = { version = "0.8", features = ["xxh3"] } -proctitle = "0.1" -unicase = "2.7" -petgraph = "0.7" -rand = { version = "0.8", features = ["small_rng"] } -itertools = "0.14" -strum = { version = "0.26", features = ["derive"] } - -arrow-json = {version = '55.2.0'} -apache-avro = "0.21" -datafusion = {git = 'https://github.com/FunctionStream/datafusion', branch = '48.0.1/fs'} -datafusion-common = {git = 'https://github.com/FunctionStream/datafusion', branch = '48.0.1/fs'} -datafusion-execution = {git = 'https://github.com/FunctionStream/datafusion', branch = '48.0.1/fs'} -datafusion-expr = {git = 'https://github.com/FunctionStream/datafusion', branch = '48.0.1/fs'} -datafusion-physical-expr = {git = 'https://github.com/FunctionStream/datafusion', branch = '48.0.1/fs'} -datafusion-proto = {git = 'https://github.com/FunctionStream/datafusion', branch = '48.0.1/fs'} - -sqlparser = { git = "https://github.com/FunctionStream/sqlparser-rs", branch = "0.58.0/fs" } - -ahash = "0.8" -governor = "0.8.0" -lance = { version = "4.0.0", default-features = false, features = ["aws"] } -arrow-array-lance = { package = "arrow-array", version = "57.3.0" } -arrow-ipc-lance = { package = "arrow-ipc", version = "57.3.0" } - -[features] -default = ["incremental-cache", "python"] -incremental-cache = ["wasmtime/incremental-cache"] -python = [] - -[dev-dependencies] -tempfile = "3.27.0" diff --git a/Makefile b/Makefile index 78138dae..d6340d9d 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. APP_NAME := function-stream +# Version from root `[workspace.package]` (single source of truth). VERSION := $(shell grep '^version' Cargo.toml | head -1 | awk -F '"' '{print $$2}') DATE := $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") @@ -106,6 +107,7 @@ build: .check-env .ensure-target .build-wasm @RUSTFLAGS="$(OPTIMIZE_FLAGS)" \ cargo build --release \ --target $(TRIPLE) \ + -p $(APP_NAME) \ --features python \ --quiet $(call log,BUILD,CLI) @@ -118,14 +120,15 @@ build: .check-env .ensure-target .build-wasm build-lite: .check-env .ensure-target $(call log,BUILD,Rust Lite [$(OS_NAME) / $(TRIPLE)]) - @RUSTFLAGS="$(INDUSTRIAL_RUSTFLAGS)" \ + @RUSTFLAGS="$(OPTIMIZE_FLAGS)" \ cargo build --release \ --target $(TRIPLE) \ + -p $(APP_NAME) \ --no-default-features \ --features incremental-cache \ --quiet $(call log,BUILD,CLI for dist) - @RUSTFLAGS="$(INDUSTRIAL_RUSTFLAGS)" \ + @RUSTFLAGS="$(OPTIMIZE_FLAGS)" \ cargo build --release \ --target $(TRIPLE) \ -p function-stream-cli \ diff --git a/cli/cli/Cargo.toml b/cli/cli/Cargo.toml index 49c0a881..75bd6cb4 100644 --- a/cli/cli/Cargo.toml +++ b/cli/cli/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "function-stream-cli" -version = "0.1.0" -edition = "2021" +version.workspace = true +edition.workspace = true [[bin]] name = "cli" diff --git a/cli/cli/src/repl.rs b/cli/cli/src/repl.rs index b442bd07..01209ba1 100644 --- a/cli/cli/src/repl.rs +++ b/cli/cli/src/repl.rs @@ -17,7 +17,7 @@ use arrow_ipc::reader::StreamReader; use arrow_schema::DataType; use comfy_table::presets::UTF8_FULL; use comfy_table::{Attribute, Cell, Color, ContentArrangement, Table, TableComponent}; -use protocol::cli::{function_stream_service_client::FunctionStreamServiceClient, SqlRequest}; +use protocol::cli::{SqlRequest, function_stream_service_client::FunctionStreamServiceClient}; use rustyline::error::ReadlineError; use rustyline::{Config, DefaultEditor, EditMode}; use std::fmt; @@ -158,17 +158,17 @@ impl Repl { } // 3. Strict Data Check: Only proceed if data is explicitly present and non-empty - if let Some(bytes) = response.data { - if !bytes.is_empty() { - // format_arrow_data returns Ok(Some(Table)) ONLY if row_count > 0 - match self.format_arrow_data(&bytes) { - Ok(Some(table)) => println!("{}", table), - Ok(None) => { - // Data was present but contained 0 rows (e.g., empty result set) - // We print nothing here to keep output clean as requested - } - Err(e) => eprintln!("Failed to parse result data: {}", e), + if let Some(bytes) = response.data + && !bytes.is_empty() + { + // format_arrow_data returns Ok(Some(Table)) ONLY if row_count > 0 + match self.format_arrow_data(&bytes) { + Ok(Some(table)) => println!("{}", table), + Ok(None) => { + // Data was present but contained 0 rows (e.g., empty result set) + // We print nothing here to keep output clean as requested } + Err(e) => eprintln!("Failed to parse result data: {}", e), } } @@ -243,11 +243,7 @@ impl Repl { } } - if has_rows { - Ok(Some(table)) - } else { - Ok(None) - } + if has_rows { Ok(Some(table)) } else { Ok(None) } } fn extract_value(&self, column: &dyn Array, row: usize) -> String { @@ -317,7 +313,7 @@ impl Repl { #[cfg(unix)] let mut sigterm = { - use tokio::signal::unix::{signal, SignalKind}; + use tokio::signal::unix::{SignalKind, signal}; signal(SignalKind::terminate()).expect("failed to register SIGTERM handler") }; @@ -403,10 +399,8 @@ impl Repl { println!(); } - if !skip_save_history { - if let Some(ref mut ed) = repl.lock().await.editor { - let _ = ed.save_history(".function-stream-cli-history"); - } + if !skip_save_history && let Some(ref mut ed) = repl.lock().await.editor { + let _ = ed.save_history(".function-stream-cli-history"); } Ok(()) } @@ -448,10 +442,10 @@ impl Repl { } fn add_history_entry(&mut self, entry: &str) { - if let Some(ed) = self.editor.as_mut() { - if !entry.trim().is_empty() { - let _ = ed.add_history_entry(entry.trim()); - } + if let Some(ed) = self.editor.as_mut() + && !entry.trim().is_empty() + { + let _ = ed.add_history_entry(entry.trim()); } } diff --git a/examples/examples-validator/Cargo.toml b/examples/examples-validator/Cargo.toml index de87dbb3..9c204935 100644 --- a/examples/examples-validator/Cargo.toml +++ b/examples/examples-validator/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "examples-validator" version = "0.1.0" -edition = "2021" +edition = "2024" [workspace] diff --git a/protocol/Cargo.toml b/protocol/Cargo.toml index 51b1f3c1..96501c30 100644 --- a/protocol/Cargo.toml +++ b/protocol/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "protocol" -version = "0.1.0" -edition = "2024" +version.workspace = true +edition.workspace = true description = "Protocol Buffers protocol definitions for function stream" license = "MIT OR Apache-2.0" repository = "https://github.com/your-username/rust-function-stream" diff --git a/src/catalog/Cargo.toml b/src/catalog/Cargo.toml new file mode 100644 index 00000000..2a50735c --- /dev/null +++ b/src/catalog/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "function-stream-catalog" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_catalog" +path = "src/lib.rs" + +[dependencies] +protocol = { path = "../../protocol" } diff --git a/src/catalog/src/error.rs b/src/catalog/src/error.rs new file mode 100644 index 00000000..f5a9378e --- /dev/null +++ b/src/catalog/src/error.rs @@ -0,0 +1,4 @@ +use std::error::Error; + +pub type CatalogError = Box; +pub type CatalogResult = Result; diff --git a/src/catalog/src/lib.rs b/src/catalog/src/lib.rs new file mode 100644 index 00000000..33fb2743 --- /dev/null +++ b/src/catalog/src/lib.rs @@ -0,0 +1,9 @@ +//! Catalog domain types and APIs. + +pub mod error; +pub mod meta_store; +pub mod stream_catalog; + +pub use error::{CatalogError, CatalogResult}; +pub use meta_store::MetaStore; +pub use stream_catalog::{GlobalStreamCatalog, StoredStreamingJob, StreamCatalog}; diff --git a/src/catalog/src/meta_store.rs b/src/catalog/src/meta_store.rs new file mode 100644 index 00000000..b361e03c --- /dev/null +++ b/src/catalog/src/meta_store.rs @@ -0,0 +1,23 @@ +use crate::CatalogResult; + +/// Synchronous metadata key-value backend for catalog records. +pub trait MetaStore: Send + Sync { + fn put(&self, key: &str, value: Vec) -> CatalogResult<()>; + fn get(&self, key: &str) -> CatalogResult>>; + fn delete(&self, key: &str) -> CatalogResult<()>; + fn scan_prefix(&self, prefix: &str) -> CatalogResult)>>; + + /// Atomic apply of many puts (`Some(value)`) and deletes (`None`). + /// + /// Backends should override this with a single transaction or write batch + /// when the storage engine supports it. + fn write_batch(&self, batch: Vec<(String, Option>)>) -> CatalogResult<()> { + for (key, value) in batch { + match value { + Some(value) => self.put(&key, value)?, + None => self.delete(&key)?, + } + } + Ok(()) + } +} diff --git a/src/catalog/src/stream_catalog.rs b/src/catalog/src/stream_catalog.rs new file mode 100644 index 00000000..a5641a34 --- /dev/null +++ b/src/catalog/src/stream_catalog.rs @@ -0,0 +1,69 @@ +use std::sync::Arc; + +use protocol::function_stream_graph::FsProgram; +use protocol::storage as pb; + +use crate::CatalogResult; + +/// One persisted streaming job row from catalog storage. +/// +/// This is intentionally storage-agnostic: the catalog keeps source checkpoint +/// payloads as protocol oneof envelopes and does not inspect source-specific +/// checkpoint data. +#[derive(Debug, Clone)] +pub struct StoredStreamingJob { + pub table_name: String, + pub program: FsProgram, + pub checkpoint_interval_ms: u64, + pub latest_checkpoint_epoch: u64, + pub source_checkpoints: Vec, +} + +/// Interface exposed by the stream catalog manager. +/// +/// The concrete table and planning types are generic so this crate can define +/// the catalog boundary without depending on the monolithic SQL/runtime crates. +pub trait StreamCatalog: Send + Sync { + fn persist_streaming_job( + &self, + table_name: &str, + fs_program: &FsProgram, + comment: &str, + checkpoint_interval_ms: u64, + ) -> CatalogResult<()>; + + fn remove_streaming_job(&self, table_name: &str) -> CatalogResult<()>; + + fn commit_job_checkpoint( + &self, + table_name: &str, + epoch: u64, + source_checkpoints: Vec, + ) -> CatalogResult<()>; + + fn load_streaming_job_definitions(&self) -> CatalogResult>; + + fn add_catalog_table(&self, table: Table) -> CatalogResult<()>; + fn has_catalog_table(&self, name: &str) -> bool; + fn drop_catalog_table(&self, table_name: &str, if_exists: bool) -> CatalogResult<()>; + fn restore_from_store(&self) -> CatalogResult<()>; + fn acquire_planning_context(&self) -> PlanningContext; + fn list_catalog_tables(&self) -> CatalogResult>>; + fn get_catalog_table(&self, name: &str) -> CatalogResult>>; + + fn add_table(&self, table: StreamTable) -> CatalogResult<()>; + fn has_stream_table(&self, name: &str) -> bool; + fn drop_table(&self, table_name: &str, if_exists: bool) -> CatalogResult<()>; + fn list_stream_tables(&self) -> Vec>; + fn get_stream_table(&self, name: &str) -> Option>; +} + +/// Process-global catalog access boundary. +pub trait GlobalStreamCatalog: Send + Sync { + fn init_global(manager: Arc) -> CatalogResult<()>; + fn try_global() -> Option>; + + fn global() -> CatalogResult> { + Self::try_global().ok_or_else(|| "CatalogManager not initialized".into()) + } +} diff --git a/src/catalog_storage/Cargo.toml b/src/catalog_storage/Cargo.toml new file mode 100644 index 00000000..3bb9fed8 --- /dev/null +++ b/src/catalog_storage/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "function-stream-catalog-storage" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_catalog_storage" +path = "src/lib.rs" + +[dependencies] +anyhow = "1.0" +function-stream-catalog = { path = "../catalog" } +parking_lot = "0.12" +rocksdb = { version = "0.21", features = ["multi-threaded-cf", "lz4"] } diff --git a/src/catalog_storage/src/lib.rs b/src/catalog_storage/src/lib.rs new file mode 100644 index 00000000..8fdca0bf --- /dev/null +++ b/src/catalog_storage/src/lib.rs @@ -0,0 +1,10 @@ +//! Persistent catalog storage implementations. +//! +//! The stream catalog manager and task persistence (`stream_catalog/`, `task/`) live in this +//! package and are compiled as part of `function-stream` via `#[path]` in `src/lib.rs` / `src/main.rs`. + +pub mod memory; +pub mod rocksdb; + +pub use memory::InMemoryMetaStore; +pub use rocksdb::RocksDbMetaStore; diff --git a/src/catalog_storage/src/memory.rs b/src/catalog_storage/src/memory.rs new file mode 100644 index 00000000..df6fd0c0 --- /dev/null +++ b/src/catalog_storage/src/memory.rs @@ -0,0 +1,63 @@ +use std::collections::HashMap; + +use function_stream_catalog::{CatalogResult, MetaStore}; +use parking_lot::RwLock; + +/// In-process KV store for single-node deployments and tests. +pub struct InMemoryMetaStore { + db: RwLock>>, +} + +impl InMemoryMetaStore { + pub fn new() -> Self { + Self { + db: RwLock::new(HashMap::new()), + } + } +} + +impl Default for InMemoryMetaStore { + fn default() -> Self { + Self::new() + } +} + +impl MetaStore for InMemoryMetaStore { + fn put(&self, key: &str, value: Vec) -> CatalogResult<()> { + self.db.write().insert(key.to_string(), value); + Ok(()) + } + + fn get(&self, key: &str) -> CatalogResult>> { + Ok(self.db.read().get(key).cloned()) + } + + fn delete(&self, key: &str) -> CatalogResult<()> { + self.db.write().remove(key); + Ok(()) + } + + fn scan_prefix(&self, prefix: &str) -> CatalogResult)>> { + let db = self.db.read(); + Ok(db + .iter() + .filter(|(key, _)| key.starts_with(prefix)) + .map(|(key, value)| (key.clone(), value.clone())) + .collect()) + } + + fn write_batch(&self, batch: Vec<(String, Option>)>) -> CatalogResult<()> { + let mut db = self.db.write(); + for (key, value) in batch { + match value { + Some(value) => { + db.insert(key, value); + } + None => { + db.remove(&key); + } + } + } + Ok(()) + } +} diff --git a/src/catalog_storage/src/rocksdb.rs b/src/catalog_storage/src/rocksdb.rs new file mode 100644 index 00000000..192b9d80 --- /dev/null +++ b/src/catalog_storage/src/rocksdb.rs @@ -0,0 +1,81 @@ +use std::path::Path; +use std::sync::Arc; + +use anyhow::Context; +use function_stream_catalog::{CatalogResult, MetaStore}; +use rocksdb::{DB, Direction, IteratorMode, Options, WriteBatch}; + +/// RocksDB-backed catalog metadata store. +pub struct RocksDbMetaStore { + db: Arc, +} + +impl RocksDbMetaStore { + pub fn open>(path: P) -> anyhow::Result { + let path = path.as_ref(); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("stream catalog: create parent directory {parent:?}"))?; + } + let mut opts = Options::default(); + opts.create_if_missing(true); + let db = DB::open(&opts, path) + .with_context(|| format!("stream catalog: open RocksDB at {}", path.display()))?; + Ok(Self { db: Arc::new(db) }) + } +} + +impl MetaStore for RocksDbMetaStore { + fn put(&self, key: &str, value: Vec) -> CatalogResult<()> { + self.db + .put(key.as_bytes(), value.as_slice()) + .map_err(|e| format!("stream catalog store put: {e}").into()) + } + + fn get(&self, key: &str) -> CatalogResult>> { + self.db + .get(key.as_bytes()) + .map_err(|e| format!("stream catalog store get: {e}").into()) + } + + fn delete(&self, key: &str) -> CatalogResult<()> { + self.db + .delete(key.as_bytes()) + .map_err(|e| format!("stream catalog store delete: {e}").into()) + } + + fn scan_prefix(&self, prefix: &str) -> CatalogResult)>> { + let mut out = Vec::new(); + let iter = self + .db + .iterator(IteratorMode::From(prefix.as_bytes(), Direction::Forward)); + for item in iter { + let (key, value) = item.map_err(|e| format!("stream catalog store scan: {e}"))?; + let key = String::from_utf8(key.to_vec()) + .map_err(|e| format!("stream catalog store: invalid utf8 key: {e}"))?; + if !key.starts_with(prefix) { + break; + } + out.push((key, value.to_vec())); + } + Ok(out) + } + + fn write_batch(&self, batch: Vec<(String, Option>)>) -> CatalogResult<()> { + if batch.is_empty() { + return Ok(()); + } + + let mut write_batch = WriteBatch::default(); + for (key, value) in batch { + match value { + Some(value) => write_batch.put(key.as_bytes(), value.as_slice()), + None => write_batch.delete(key.as_bytes()), + } + } + + self.db + .write(write_batch) + .map_err(|e| format!("stream catalog store write_batch: {e}").into()) + } +} diff --git a/src/storage/stream_catalog/codec.rs b/src/catalog_storage/src/stream_catalog/codec.rs similarity index 100% rename from src/storage/stream_catalog/codec.rs rename to src/catalog_storage/src/stream_catalog/codec.rs diff --git a/src/storage/stream_catalog/manager.rs b/src/catalog_storage/src/stream_catalog/manager.rs similarity index 99% rename from src/storage/stream_catalog/manager.rs rename to src/catalog_storage/src/stream_catalog/manager.rs index fa810e8d..cbdf30b3 100644 --- a/src/storage/stream_catalog/manager.rs +++ b/src/catalog_storage/src/stream_catalog/manager.rs @@ -627,7 +627,7 @@ pub fn restore_global_catalog_from_store() { } pub fn restore_streaming_jobs_from_store() { - use crate::runtime::streaming::job::JobManager; + use crate::streaming::job::JobManager; let Some(catalog) = CatalogManager::try_global() else { warn!("CatalogManager not available; skipping streaming job restore"); @@ -742,7 +742,7 @@ mod tests { use crate::sql::schema::column_descriptor::ColumnDescriptor; use crate::sql::schema::table::CatalogEntity; use crate::sql::schema::temporal_pipeline_config::TemporalPipelineConfig; - use crate::storage::stream_catalog::InMemoryMetaStore; + use crate::stream_catalog::InMemoryMetaStore; use super::CatalogManager; diff --git a/src/storage/stream_catalog/meta_store.rs b/src/catalog_storage/src/stream_catalog/meta_store.rs similarity index 100% rename from src/storage/stream_catalog/meta_store.rs rename to src/catalog_storage/src/stream_catalog/meta_store.rs diff --git a/src/storage/stream_catalog/mod.rs b/src/catalog_storage/src/stream_catalog/mod.rs similarity index 100% rename from src/storage/stream_catalog/mod.rs rename to src/catalog_storage/src/stream_catalog/mod.rs diff --git a/src/storage/stream_catalog/rocksdb_meta_store.rs b/src/catalog_storage/src/stream_catalog/rocksdb_meta_store.rs similarity index 98% rename from src/storage/stream_catalog/rocksdb_meta_store.rs rename to src/catalog_storage/src/stream_catalog/rocksdb_meta_store.rs index 1537f278..58c0b35c 100644 --- a/src/storage/stream_catalog/rocksdb_meta_store.rs +++ b/src/catalog_storage/src/stream_catalog/rocksdb_meta_store.rs @@ -20,7 +20,7 @@ use rocksdb::{DB, Direction, IteratorMode, Options, WriteBatch}; use super::MetaStore; -/// Single-node durable KV used by [`crate::storage::stream_catalog::CatalogManager`]. +/// Single-node durable KV used by [`crate::stream_catalog::CatalogManager`]. pub struct RocksDbMetaStore { db: Arc, } diff --git a/src/storage/task/factory.rs b/src/catalog_storage/src/task/factory.rs similarity index 100% rename from src/storage/task/factory.rs rename to src/catalog_storage/src/task/factory.rs diff --git a/src/storage/task/function_info.rs b/src/catalog_storage/src/task/function_info.rs similarity index 100% rename from src/storage/task/function_info.rs rename to src/catalog_storage/src/task/function_info.rs diff --git a/src/storage/task/mod.rs b/src/catalog_storage/src/task/mod.rs similarity index 100% rename from src/storage/task/mod.rs rename to src/catalog_storage/src/task/mod.rs diff --git a/src/storage/task/proto_codec.rs b/src/catalog_storage/src/task/proto_codec.rs similarity index 99% rename from src/storage/task/proto_codec.rs rename to src/catalog_storage/src/task/proto_codec.rs index 6c1bc8df..78a0426d 100644 --- a/src/storage/task/proto_codec.rs +++ b/src/catalog_storage/src/task/proto_codec.rs @@ -20,7 +20,7 @@ use protocol::storage::{ }; use serde::{Deserialize, Serialize}; -use crate::runtime::common::ComponentState; +use crate::common::ComponentState; use super::storage::TaskModuleBytes; diff --git a/src/storage/task/rocksdb_storage.rs b/src/catalog_storage/src/task/rocksdb_storage.rs similarity index 99% rename from src/storage/task/rocksdb_storage.rs rename to src/catalog_storage/src/task/rocksdb_storage.rs index cea0ceb9..a3927515 100644 --- a/src/storage/task/rocksdb_storage.rs +++ b/src/catalog_storage/src/task/rocksdb_storage.rs @@ -19,8 +19,8 @@ use super::proto_codec::{ encode_task_module_bytes, }; use super::storage::{StoredTaskInfo, TaskStorage}; +use crate::common::ComponentState; use crate::config::storage::RocksDBStorageConfig; -use crate::runtime::common::ComponentState; use anyhow::{Context, Result, anyhow}; use rocksdb::{ColumnFamilyDescriptor, DB, IteratorMode, Options, WriteBatch}; use std::path::Path; diff --git a/src/storage/task/storage.rs b/src/catalog_storage/src/task/storage.rs similarity index 97% rename from src/storage/task/storage.rs rename to src/catalog_storage/src/task/storage.rs index 156ee5d8..6cbec9ea 100644 --- a/src/storage/task/storage.rs +++ b/src/catalog_storage/src/task/storage.rs @@ -10,7 +10,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::common::ComponentState; +use crate::common::ComponentState; use anyhow::Result; use serde::{Deserialize, Serialize}; diff --git a/src/common/Cargo.toml b/src/common/Cargo.toml new file mode 100644 index 00000000..4c155c5e --- /dev/null +++ b/src/common/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "function-stream-common" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_common" +path = "src/lib.rs" diff --git a/src/common/fs_schema.rs b/src/common/src/fs_schema.rs similarity index 100% rename from src/common/fs_schema.rs rename to src/common/src/fs_schema.rs diff --git a/src/common/mod.rs b/src/common/src/legacy_mod.rs similarity index 100% rename from src/common/mod.rs rename to src/common/src/legacy_mod.rs diff --git a/src/common/src/lib.rs b/src/common/src/lib.rs new file mode 100644 index 00000000..0356f6de --- /dev/null +++ b/src/common/src/lib.rs @@ -0,0 +1,3 @@ +//! Shared types and utilities for FunctionStream crates. + +pub const CRATE_NAME: &str = "function-stream-common"; diff --git a/src/config/Cargo.toml b/src/config/Cargo.toml new file mode 100644 index 00000000..d1e437cd --- /dev/null +++ b/src/config/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "function-stream-config" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_config" +path = "src/lib.rs" + +[features] +default = ["python"] +python = [] + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_yaml = "0.9" +uuid = { version = "1.0", features = ["v4", "v7"] } diff --git a/src/config/global_config.rs b/src/config/src/global_config.rs similarity index 100% rename from src/config/global_config.rs rename to src/config/src/global_config.rs diff --git a/src/config/mod.rs b/src/config/src/lib.rs similarity index 61% rename from src/config/mod.rs rename to src/config/src/lib.rs index e60dcfde..c07c676c 100644 --- a/src/config/mod.rs +++ b/src/config/src/lib.rs @@ -1,14 +1,4 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//! Configuration loading and validation. pub mod global_config; pub mod loader; @@ -21,6 +11,11 @@ pub mod streaming_job; pub mod system; pub mod wasm_config; +// Compatibility shim for files that still reference `crate::config::*`. +pub mod config { + pub use crate::*; +} + pub use global_config::{ DEFAULT_OPERATOR_STATE_STORE_MEMORY_BYTES, DEFAULT_STREAMING_RUNTIME_MEMORY_BYTES, GlobalConfig, }; diff --git a/src/config/loader.rs b/src/config/src/loader.rs similarity index 100% rename from src/config/loader.rs rename to src/config/src/loader.rs diff --git a/src/config/log_config.rs b/src/config/src/log_config.rs similarity index 100% rename from src/config/log_config.rs rename to src/config/src/log_config.rs diff --git a/src/config/paths.rs b/src/config/src/paths.rs similarity index 100% rename from src/config/paths.rs rename to src/config/src/paths.rs diff --git a/src/config/python_config.rs b/src/config/src/python_config.rs similarity index 100% rename from src/config/python_config.rs rename to src/config/src/python_config.rs diff --git a/src/config/service_config.rs b/src/config/src/service_config.rs similarity index 100% rename from src/config/service_config.rs rename to src/config/src/service_config.rs diff --git a/src/config/storage.rs b/src/config/src/storage.rs similarity index 100% rename from src/config/storage.rs rename to src/config/src/storage.rs diff --git a/src/config/streaming_job.rs b/src/config/src/streaming_job.rs similarity index 100% rename from src/config/streaming_job.rs rename to src/config/src/streaming_job.rs diff --git a/src/config/system.rs b/src/config/src/system.rs similarity index 100% rename from src/config/system.rs rename to src/config/src/system.rs diff --git a/src/config/wasm_config.rs b/src/config/src/wasm_config.rs similarity index 100% rename from src/config/wasm_config.rs rename to src/config/src/wasm_config.rs diff --git a/src/coordinator/Cargo.toml b/src/coordinator/Cargo.toml new file mode 100644 index 00000000..f8a6e0b7 --- /dev/null +++ b/src/coordinator/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "function-stream-coordinator" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_coordinator" +path = "src/lib.rs" diff --git a/src/coordinator/analyze/analysis.rs b/src/coordinator/src/analyze/analysis.rs similarity index 100% rename from src/coordinator/analyze/analysis.rs rename to src/coordinator/src/analyze/analysis.rs diff --git a/src/coordinator/analyze/analyzer.rs b/src/coordinator/src/analyze/analyzer.rs similarity index 100% rename from src/coordinator/analyze/analyzer.rs rename to src/coordinator/src/analyze/analyzer.rs diff --git a/src/coordinator/analyze/mod.rs b/src/coordinator/src/analyze/mod.rs similarity index 100% rename from src/coordinator/analyze/mod.rs rename to src/coordinator/src/analyze/mod.rs diff --git a/src/coordinator/coordinator.rs b/src/coordinator/src/coordinator.rs similarity index 100% rename from src/coordinator/coordinator.rs rename to src/coordinator/src/coordinator.rs diff --git a/src/coordinator/mod.rs b/src/coordinator/src/coordinator_body.rs similarity index 95% rename from src/coordinator/mod.rs rename to src/coordinator/src/coordinator_body.rs index 86598bc5..0b5b7135 100644 --- a/src/coordinator/mod.rs +++ b/src/coordinator/src/coordinator_body.rs @@ -18,12 +18,14 @@ mod execution; mod execution_context; mod plan; mod runtime_context; +mod sql_classify; mod statement; mod streaming_table_options; mod tool; pub use coordinator::Coordinator; pub use dataset::{DataSet, ShowFunctionsResult}; +pub use sql_classify::classify_statement; pub use statement::{ CreateFunction, CreatePythonFunction, CreateTable, DropFunction, DropStreamingTableStatement, DropTableStatement, PythonModule, ShowCatalogTables, ShowCreateStreamingTable, ShowCreateTable, diff --git a/src/coordinator/dataset/data_set.rs b/src/coordinator/src/dataset/data_set.rs similarity index 100% rename from src/coordinator/dataset/data_set.rs rename to src/coordinator/src/dataset/data_set.rs diff --git a/src/coordinator/dataset/execute_result.rs b/src/coordinator/src/dataset/execute_result.rs similarity index 100% rename from src/coordinator/dataset/execute_result.rs rename to src/coordinator/src/dataset/execute_result.rs diff --git a/src/coordinator/dataset/mod.rs b/src/coordinator/src/dataset/mod.rs similarity index 100% rename from src/coordinator/dataset/mod.rs rename to src/coordinator/src/dataset/mod.rs diff --git a/src/coordinator/dataset/show_catalog_tables_result.rs b/src/coordinator/src/dataset/show_catalog_tables_result.rs similarity index 100% rename from src/coordinator/dataset/show_catalog_tables_result.rs rename to src/coordinator/src/dataset/show_catalog_tables_result.rs diff --git a/src/coordinator/dataset/show_create_streaming_table_result.rs b/src/coordinator/src/dataset/show_create_streaming_table_result.rs similarity index 100% rename from src/coordinator/dataset/show_create_streaming_table_result.rs rename to src/coordinator/src/dataset/show_create_streaming_table_result.rs diff --git a/src/coordinator/dataset/show_create_table_result.rs b/src/coordinator/src/dataset/show_create_table_result.rs similarity index 100% rename from src/coordinator/dataset/show_create_table_result.rs rename to src/coordinator/src/dataset/show_create_table_result.rs diff --git a/src/coordinator/dataset/show_functions_result.rs b/src/coordinator/src/dataset/show_functions_result.rs similarity index 98% rename from src/coordinator/dataset/show_functions_result.rs rename to src/coordinator/src/dataset/show_functions_result.rs index c16edf6d..18673f00 100644 --- a/src/coordinator/dataset/show_functions_result.rs +++ b/src/coordinator/src/dataset/show_functions_result.rs @@ -16,7 +16,7 @@ use arrow_array::{RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema}; use super::DataSet; -use crate::storage::task::FunctionInfo; +use crate::task::FunctionInfo; #[derive(Clone, Debug)] pub struct ShowFunctionsResult { diff --git a/src/coordinator/dataset/show_streaming_tables_result.rs b/src/coordinator/src/dataset/show_streaming_tables_result.rs similarity index 97% rename from src/coordinator/dataset/show_streaming_tables_result.rs rename to src/coordinator/src/dataset/show_streaming_tables_result.rs index cae597ac..5b5c8e84 100644 --- a/src/coordinator/dataset/show_streaming_tables_result.rs +++ b/src/coordinator/src/dataset/show_streaming_tables_result.rs @@ -16,7 +16,7 @@ use arrow_array::{Int32Array, StringArray}; use arrow_schema::{DataType, Field, Schema}; use super::DataSet; -use crate::runtime::streaming::job::StreamingJobSummary; +use crate::streaming::job::StreamingJobSummary; #[derive(Clone, Debug)] pub struct ShowStreamingTablesResult { diff --git a/src/coordinator/execution/executor.rs b/src/coordinator/src/execution/executor.rs similarity index 98% rename from src/coordinator/execution/executor.rs rename to src/coordinator/src/execution/executor.rs index 22c5c1b7..a58dc6c0 100644 --- a/src/coordinator/execution/executor.rs +++ b/src/coordinator/src/execution/executor.rs @@ -31,13 +31,13 @@ use crate::coordinator::statement::{ConfigSource, FunctionSource}; use crate::coordinator::streaming_table_options::{ parse_checkpoint_interval_ms, parse_pipeline_parallelism, }; -use crate::runtime::streaming::job::JobManager; -use crate::runtime::streaming::protocol::control::StopMode; -use crate::runtime::wasm::taskexecutor::TaskManager; use crate::sql::schema::catalog::ExternalTable; use crate::sql::schema::show_create_catalog_table; use crate::sql::schema::table::CatalogEntity; -use crate::storage::stream_catalog::CatalogManager; +use crate::stream_catalog::CatalogManager; +use crate::streaming::job::JobManager; +use crate::streaming::protocol::control::StopMode; +use crate::wasm::taskexecutor::TaskManager; #[derive(Error, Debug)] pub enum ExecuteError { diff --git a/src/coordinator/execution/mod.rs b/src/coordinator/src/execution/mod.rs similarity index 100% rename from src/coordinator/execution/mod.rs rename to src/coordinator/src/execution/mod.rs diff --git a/src/coordinator/execution_context.rs b/src/coordinator/src/execution_context.rs similarity index 100% rename from src/coordinator/execution_context.rs rename to src/coordinator/src/execution_context.rs diff --git a/src/coordinator/src/lib.rs b/src/coordinator/src/lib.rs new file mode 100644 index 00000000..6e5ad596 --- /dev/null +++ b/src/coordinator/src/lib.rs @@ -0,0 +1,3 @@ +//! Query planning and job coordination. + +pub const CRATE_NAME: &str = "function-stream-coordinator"; diff --git a/src/coordinator/plan/compile_error_plan.rs b/src/coordinator/src/plan/compile_error_plan.rs similarity index 100% rename from src/coordinator/plan/compile_error_plan.rs rename to src/coordinator/src/plan/compile_error_plan.rs diff --git a/src/coordinator/plan/create_function_plan.rs b/src/coordinator/src/plan/create_function_plan.rs similarity index 100% rename from src/coordinator/plan/create_function_plan.rs rename to src/coordinator/src/plan/create_function_plan.rs diff --git a/src/coordinator/plan/create_python_function_plan.rs b/src/coordinator/src/plan/create_python_function_plan.rs similarity index 100% rename from src/coordinator/plan/create_python_function_plan.rs rename to src/coordinator/src/plan/create_python_function_plan.rs diff --git a/src/coordinator/plan/create_table_plan.rs b/src/coordinator/src/plan/create_table_plan.rs similarity index 100% rename from src/coordinator/plan/create_table_plan.rs rename to src/coordinator/src/plan/create_table_plan.rs diff --git a/src/coordinator/plan/drop_function_plan.rs b/src/coordinator/src/plan/drop_function_plan.rs similarity index 100% rename from src/coordinator/plan/drop_function_plan.rs rename to src/coordinator/src/plan/drop_function_plan.rs diff --git a/src/coordinator/plan/drop_streaming_table_plan.rs b/src/coordinator/src/plan/drop_streaming_table_plan.rs similarity index 100% rename from src/coordinator/plan/drop_streaming_table_plan.rs rename to src/coordinator/src/plan/drop_streaming_table_plan.rs diff --git a/src/coordinator/plan/drop_table_plan.rs b/src/coordinator/src/plan/drop_table_plan.rs similarity index 100% rename from src/coordinator/plan/drop_table_plan.rs rename to src/coordinator/src/plan/drop_table_plan.rs diff --git a/src/coordinator/plan/logical_plan_visitor.rs b/src/coordinator/src/plan/logical_plan_visitor.rs similarity index 92% rename from src/coordinator/plan/logical_plan_visitor.rs rename to src/coordinator/src/plan/logical_plan_visitor.rs index cb22d24e..66797ced 100644 --- a/src/coordinator/plan/logical_plan_visitor.rs +++ b/src/coordinator/src/plan/logical_plan_visitor.rs @@ -11,7 +11,6 @@ // limitations under the License. use datafusion::sql::sqlparser::ast::{ObjectType, Statement as DFStatement}; -use sqlparser::ast::Statement; use tracing::debug; use crate::coordinator::analyze::analysis::Analysis; @@ -28,9 +27,8 @@ use crate::coordinator::statement::{ StatementVisitorResult, StopFunction, StreamingTableStatement, }; use crate::sql::analysis::StreamSchemaProvider; +use crate::sql::schema::try_compile_connector_create_table; -use super::ast_utils::AstUtils; -use super::ddl_compiler::DdlCompiler; use super::streaming_compiler::StreamingCompiler; #[derive(Clone)] @@ -135,15 +133,12 @@ impl StatementVisitor for LogicalPlanVisitor { stmt: &CreateTable, _ctx: &StatementVisitorContext, ) -> StatementVisitorResult { - if let Statement::CreateTable(ast_node) = &stmt.statement - && ast_node.query.is_none() - && AstUtils::contains_connector_property(&ast_node.with_options) + if let Some(result) = + try_compile_connector_create_table(&self.schema_provider, &stmt.statement) { - let declared_role = AstUtils::peek_table_role(&ast_node.with_options); - let compiler = DdlCompiler::new(&self.schema_provider); - return match compiler.compile(ast_node, declared_role.as_deref()) { - Ok(external_table) => StatementVisitorResult::Plan(Box::new( - CreateTablePlan::external_table(external_table, ast_node.if_not_exists), + return match result { + Ok((external_table, if_not_exists)) => StatementVisitorResult::Plan(Box::new( + CreateTablePlan::external_table(external_table, if_not_exists), )), Err(err) => StatementVisitorResult::Plan(Box::new(CompileErrorPlan::new(format!( "Ingest table resolution failed - {err:#}" diff --git a/src/coordinator/plan/lookup_table_plan.rs b/src/coordinator/src/plan/lookup_table_plan.rs similarity index 100% rename from src/coordinator/plan/lookup_table_plan.rs rename to src/coordinator/src/plan/lookup_table_plan.rs diff --git a/src/coordinator/plan/mod.rs b/src/coordinator/src/plan/mod.rs similarity index 98% rename from src/coordinator/plan/mod.rs rename to src/coordinator/src/plan/mod.rs index 95978467..7f4b279a 100644 --- a/src/coordinator/plan/mod.rs +++ b/src/coordinator/src/plan/mod.rs @@ -10,12 +10,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -mod ast_utils; mod compile_error_plan; mod create_function_plan; mod create_python_function_plan; mod create_table_plan; -mod ddl_compiler; mod drop_function_plan; mod drop_streaming_table_plan; mod drop_table_plan; diff --git a/src/coordinator/plan/optimizer.rs b/src/coordinator/src/plan/optimizer.rs similarity index 100% rename from src/coordinator/plan/optimizer.rs rename to src/coordinator/src/plan/optimizer.rs diff --git a/src/coordinator/plan/show_catalog_tables_plan.rs b/src/coordinator/src/plan/show_catalog_tables_plan.rs similarity index 100% rename from src/coordinator/plan/show_catalog_tables_plan.rs rename to src/coordinator/src/plan/show_catalog_tables_plan.rs diff --git a/src/coordinator/plan/show_create_streaming_table_plan.rs b/src/coordinator/src/plan/show_create_streaming_table_plan.rs similarity index 100% rename from src/coordinator/plan/show_create_streaming_table_plan.rs rename to src/coordinator/src/plan/show_create_streaming_table_plan.rs diff --git a/src/coordinator/plan/show_create_table_plan.rs b/src/coordinator/src/plan/show_create_table_plan.rs similarity index 100% rename from src/coordinator/plan/show_create_table_plan.rs rename to src/coordinator/src/plan/show_create_table_plan.rs diff --git a/src/coordinator/plan/show_functions_plan.rs b/src/coordinator/src/plan/show_functions_plan.rs similarity index 100% rename from src/coordinator/plan/show_functions_plan.rs rename to src/coordinator/src/plan/show_functions_plan.rs diff --git a/src/coordinator/plan/show_streaming_tables_plan.rs b/src/coordinator/src/plan/show_streaming_tables_plan.rs similarity index 100% rename from src/coordinator/plan/show_streaming_tables_plan.rs rename to src/coordinator/src/plan/show_streaming_tables_plan.rs diff --git a/src/coordinator/plan/start_function_plan.rs b/src/coordinator/src/plan/start_function_plan.rs similarity index 100% rename from src/coordinator/plan/start_function_plan.rs rename to src/coordinator/src/plan/start_function_plan.rs diff --git a/src/coordinator/plan/stop_function_plan.rs b/src/coordinator/src/plan/stop_function_plan.rs similarity index 100% rename from src/coordinator/plan/stop_function_plan.rs rename to src/coordinator/src/plan/stop_function_plan.rs diff --git a/src/coordinator/plan/streaming_compiler.rs b/src/coordinator/src/plan/streaming_compiler.rs similarity index 100% rename from src/coordinator/plan/streaming_compiler.rs rename to src/coordinator/src/plan/streaming_compiler.rs diff --git a/src/coordinator/plan/streaming_table_connector_plan.rs b/src/coordinator/src/plan/streaming_table_connector_plan.rs similarity index 100% rename from src/coordinator/plan/streaming_table_connector_plan.rs rename to src/coordinator/src/plan/streaming_table_connector_plan.rs diff --git a/src/coordinator/plan/streaming_table_plan.rs b/src/coordinator/src/plan/streaming_table_plan.rs similarity index 100% rename from src/coordinator/plan/streaming_table_plan.rs rename to src/coordinator/src/plan/streaming_table_plan.rs diff --git a/src/coordinator/plan/visitor.rs b/src/coordinator/src/plan/visitor.rs similarity index 100% rename from src/coordinator/plan/visitor.rs rename to src/coordinator/src/plan/visitor.rs diff --git a/src/coordinator/runtime_context.rs b/src/coordinator/src/runtime_context.rs similarity index 93% rename from src/coordinator/runtime_context.rs rename to src/coordinator/src/runtime_context.rs index 21b9d876..2f8dc2a7 100644 --- a/src/coordinator/runtime_context.rs +++ b/src/coordinator/src/runtime_context.rs @@ -16,10 +16,10 @@ use std::sync::Arc; use anyhow::Result; -use crate::runtime::streaming::job::JobManager; -use crate::runtime::wasm::taskexecutor::TaskManager; use crate::sql::schema::StreamSchemaProvider; -use crate::storage::stream_catalog::CatalogManager; +use crate::stream_catalog::CatalogManager; +use crate::streaming::job::JobManager; +use crate::wasm::taskexecutor::TaskManager; /// Dependencies shared by analyze / plan / execute, analogous to installing globals in /// [`TaskManager`], [`CatalogManager`], and [`JobManager`]. diff --git a/src/sql/parse.rs b/src/coordinator/src/sql_classify.rs similarity index 51% rename from src/sql/parse.rs rename to src/coordinator/src/sql_classify.rs index 0c6b9541..0c1417d4 100644 --- a/src/sql/parse.rs +++ b/src/coordinator/src/sql_classify.rs @@ -10,22 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Coordinator-facing SQL parsing (`parse_sql`). -//! -//! **Data-definition / pipeline shape (this entry point)** -//! Only these table-related forms are supported: -//! - **`CREATE TABLE ... (cols [, WATERMARK FOR ...]) WITH ('connector' = '...', 'format' = '...', ...)`** -//! connector-backed **source** DDL (no `AS SELECT`; `connector` in `WITH` selects this path) -//! - **`CREATE TABLE ...`** other forms (including `CREATE TABLE ... AS SELECT` where DataFusion accepts it) -//! - **`CREATE STREAMING TABLE ... WITH (...) AS SELECT ...`** (streaming sink DDL) -//! - **`DROP TABLE`** / **`DROP TABLE IF EXISTS`** / **`DROP STREAMING TABLE`** (alias for `DROP TABLE` on the stream catalog) -//! - **`SHOW TABLES`** — list stream catalog tables (connector sources and streaming sinks) -//! - **`SHOW CREATE TABLE `** — best-effort DDL text (full `WITH` / `AS SELECT` may not be stored) -//! -//! **`INSERT` is not supported** here — use `CREATE TABLE ... AS SELECT` or -//! `CREATE STREAMING TABLE ... AS SELECT` to define the query shape instead. -//! -//! Other supported statements include function lifecycle (`CREATE FUNCTION WITH`, `START FUNCTION`, …). +//! Map sqlparser [`Statement`](datafusion::sql::sqlparser::ast::Statement) values into +//! coordinator [`Statement`](super::statement::Statement) trait objects. use std::collections::HashMap; @@ -34,93 +20,15 @@ use datafusion::error::DataFusionError; use datafusion::sql::sqlparser::ast::{ ObjectType, ShowCreateObject, SqlOption, Statement as DFStatement, }; -use datafusion::sql::sqlparser::dialect::FunctionStreamDialect; -use datafusion::sql::sqlparser::parser::Parser; -use crate::coordinator::{ +use super::{ CreateFunction, CreateTable, DropFunction, DropStreamingTableStatement, DropTableStatement, ShowCatalogTables, ShowCreateStreamingTable, ShowCreateTable, ShowFunctions, - ShowStreamingTables, StartFunction, Statement as CoordinatorStatement, StopFunction, - StreamingTableStatement, + ShowStreamingTables, StartFunction, Statement, StopFunction, StreamingTableStatement, }; -/// Streaming-specific SQL that the sqlparser dialect does not natively handle. -/// -/// Returns `Some(statement)` if the SQL was intercepted, `None` otherwise so -/// the caller falls through to the normal sqlparser pipeline. -fn try_parse_streaming_statement(sql: &str) -> Option> { - let tokens: Vec<&str> = sql.split_whitespace().collect(); - if tokens.is_empty() { - return None; - } - - // SHOW STREAMING TABLES - if tokens.len() == 3 - && tokens[0].eq_ignore_ascii_case("show") - && tokens[1].eq_ignore_ascii_case("streaming") - && tokens[2].eq_ignore_ascii_case("tables") - { - return Some(Box::new(ShowStreamingTables::new())); - } - - // SHOW CREATE STREAMING TABLE - if tokens.len() == 5 - && tokens[0].eq_ignore_ascii_case("show") - && tokens[1].eq_ignore_ascii_case("create") - && tokens[2].eq_ignore_ascii_case("streaming") - && tokens[3].eq_ignore_ascii_case("table") - { - let name = tokens[4].trim_end_matches(';').to_string(); - return Some(Box::new(ShowCreateStreamingTable::new(name))); - } - - // DROP STREAMING TABLE [IF EXISTS] - if tokens.len() >= 4 - && tokens[0].eq_ignore_ascii_case("drop") - && tokens[1].eq_ignore_ascii_case("streaming") - && tokens[2].eq_ignore_ascii_case("table") - { - let (if_exists, name_idx) = if tokens.len() >= 6 - && tokens[3].eq_ignore_ascii_case("if") - && tokens[4].eq_ignore_ascii_case("exists") - { - (true, 5) - } else { - (false, 3) - }; - - if name_idx >= tokens.len() { - return None; - } - let name = tokens[name_idx].trim_end_matches(';').to_string(); - return Some(Box::new(DropStreamingTableStatement::new(name, if_exists))); - } - - None -} - -pub fn parse_sql(query: &str) -> Result>> { - let trimmed = query.trim(); - if trimmed.is_empty() { - return plan_err!("Query is empty"); - } - - if let Some(stmt) = try_parse_streaming_statement(trimmed) { - return Ok(vec![stmt]); - } - - let dialect = FunctionStreamDialect {}; - let statements = Parser::parse_sql(&dialect, trimmed) - .map_err(|e| DataFusionError::Plan(format!("SQL parse error: {e}")))?; - - if statements.is_empty() { - return plan_err!("No SQL statements found"); - } - - statements.into_iter().map(classify_statement).collect() -} - -fn classify_statement(stmt: DFStatement) -> Result> { +/// Convert [`DFStatement`] from the FunctionStream SQL dialect into a coordinator statement. +pub fn classify_statement(stmt: DFStatement) -> Result> { match stmt { DFStatement::CreateFunctionWith { options } => { let properties = sql_options_to_map(&options); @@ -139,34 +47,54 @@ fn classify_statement(stmt: DFStatement) -> Result } DFStatement::ShowFunctions { .. } => Ok(Box::new(ShowFunctions::new())), DFStatement::ShowTables { .. } => Ok(Box::new(ShowCatalogTables::new())), - DFStatement::ShowCreate { obj_type, obj_name } => { - if obj_type != ShowCreateObject::Table { - return plan_err!( - "SHOW CREATE {obj_type} is not supported; use SHOW CREATE TABLE " - ); - } - Ok(Box::new(ShowCreateTable::new(obj_name.to_string()))) - } + DFStatement::ShowStreamingTable => Ok(Box::new(ShowStreamingTables::new())), + DFStatement::ShowCreate { obj_type, obj_name } => match obj_type { + ShowCreateObject::Table => Ok(Box::new(ShowCreateTable::new(obj_name.to_string()))), + ShowCreateObject::StreamingTable => Ok(Box::new(ShowCreateStreamingTable::new( + obj_name.to_string(), + ))), + _ => plan_err!( + "SHOW CREATE {obj_type} is not supported; use SHOW CREATE TABLE or SHOW CREATE STREAMING TABLE " + ), + }, s @ DFStatement::CreateTable(_) => Ok(Box::new(CreateTable::new(s))), s @ DFStatement::CreateStreamingTable { .. } => { Ok(Box::new(StreamingTableStatement::new(s))) } stmt @ DFStatement::Drop { .. } => { - { - let DFStatement::Drop { - object_type, names, .. - } = &stmt - else { - unreachable!() - }; - if *object_type != ObjectType::Table { - return plan_err!("Only DROP TABLE is supported in this SQL frontend"); + let DFStatement::Drop { + object_type, + names, + if_exists, + .. + } = &stmt + else { + unreachable!() + }; + match object_type { + ObjectType::Table => { + if names.len() != 1 { + return plan_err!( + "DROP TABLE supports exactly one table name per statement" + ); + } + Ok(Box::new(DropTableStatement::new(stmt))) } - if names.len() != 1 { - return plan_err!("DROP TABLE supports exactly one table name per statement"); + ObjectType::StreamingTable => { + if names.len() != 1 { + return plan_err!( + "DROP STREAMING TABLE supports exactly one table name per statement" + ); + } + let table_name = names[0].to_string(); + Ok(Box::new(DropStreamingTableStatement::new( + table_name, *if_exists, + ))) } + _ => plan_err!( + "Only DROP TABLE and DROP STREAMING TABLE are supported in this SQL frontend" + ), } - Ok(Box::new(DropTableStatement::new(stmt))) } DFStatement::Insert { .. } => plan_err!( "INSERT is not supported; only CREATE TABLE and CREATE STREAMING TABLE (with AS SELECT) \ @@ -176,7 +104,6 @@ fn classify_statement(stmt: DFStatement) -> Result } } -/// Convert Vec (KeyValue pairs) into HashMap. fn sql_options_to_map(options: &[SqlOption]) -> HashMap { options .iter() @@ -193,71 +120,72 @@ fn sql_options_to_map(options: &[SqlOption]) -> HashMap { #[cfg(test)] mod tests { use super::*; + use crate::sql::parse::parse_sql; - fn first_stmt(sql: &str) -> Box { + fn first_classified(sql: &str) -> Box { let mut stmts = parse_sql(sql).unwrap(); assert!(!stmts.is_empty()); - stmts.remove(0) + classify_statement(stmts.remove(0)).unwrap() } - fn is_type(stmt: &dyn CoordinatorStatement, prefix: &str) -> bool { - format!("{:?}", stmt).starts_with(prefix) + fn is_type(stmt: &dyn Statement, prefix: &str) -> bool { + format!("{stmt:?}").starts_with(prefix) } #[test] fn test_parse_create_function() { let sql = "CREATE FUNCTION WITH ('function_path'='./test.wasm', 'config_path'='./config.yml')"; - let stmt = first_stmt(sql); + let stmt = first_classified(sql); assert!(is_type(stmt.as_ref(), "CreateFunction")); } #[test] fn test_parse_create_function_minimal() { let sql = "CREATE FUNCTION WITH ('function_path'='./processor.wasm')"; - let stmt = first_stmt(sql); + let stmt = first_classified(sql); assert!(is_type(stmt.as_ref(), "CreateFunction")); } #[test] fn test_parse_drop_function() { - let stmt = first_stmt("DROP FUNCTION my_task"); + let stmt = first_classified("DROP FUNCTION my_task"); assert!(is_type(stmt.as_ref(), "DropFunction")); } #[test] fn test_parse_start_function() { - let stmt = first_stmt("START FUNCTION my_task"); + let stmt = first_classified("START FUNCTION my_task"); assert!(is_type(stmt.as_ref(), "StartFunction")); } #[test] fn test_parse_stop_function() { - let stmt = first_stmt("STOP FUNCTION my_task"); + let stmt = first_classified("STOP FUNCTION my_task"); assert!(is_type(stmt.as_ref(), "StopFunction")); } #[test] fn test_parse_show_functions() { - let stmt = first_stmt("SHOW FUNCTIONS"); + let stmt = first_classified("SHOW FUNCTIONS"); assert!(is_type(stmt.as_ref(), "ShowFunctions")); } #[test] fn test_parse_show_tables() { - let stmt = first_stmt("SHOW TABLES"); + let stmt = first_classified("SHOW TABLES"); assert!(is_type(stmt.as_ref(), "ShowCatalogTables")); } #[test] fn test_parse_show_create_table() { - let stmt = first_stmt("SHOW CREATE TABLE my_src"); + let stmt = first_classified("SHOW CREATE TABLE my_src"); assert!(is_type(stmt.as_ref(), "ShowCreateTable")); } #[test] fn test_parse_create_table() { - let stmt = first_stmt("CREATE TABLE foo (id INT, name VARCHAR)"); + let stmt = first_classified("CREATE TABLE foo (id INT, name VARCHAR)"); assert!(is_type(stmt.as_ref(), "CreateTable")); } @@ -267,47 +195,46 @@ mod tests { "CREATE TABLE kafka_src (id BIGINT, ts TIMESTAMP NOT NULL, WATERMARK FOR ts) ", "WITH ('connector' = 'kafka', 'format' = 'json', 'topic' = 'events')", ); - let stmt = first_stmt(sql); + let stmt = first_classified(sql); assert!(is_type(stmt.as_ref(), "CreateTable")); } #[test] fn test_parse_drop_table() { - let stmt = first_stmt("DROP TABLE foo"); + let stmt = first_classified("DROP TABLE foo"); assert!(is_type(stmt.as_ref(), "DropTableStatement")); } #[test] fn test_parse_drop_table_if_exists() { - let stmt = first_stmt("DROP TABLE IF EXISTS foo"); + let stmt = first_classified("DROP TABLE IF EXISTS foo"); assert!(is_type(stmt.as_ref(), "DropTableStatement")); } #[test] fn test_parse_drop_streaming_table() { - let stmt = first_stmt("DROP STREAMING TABLE my_sink"); + let stmt = first_classified("DROP STREAMING TABLE my_sink"); assert!(is_type(stmt.as_ref(), "DropStreamingTableStatement")); } #[test] fn test_parse_drop_streaming_table_if_exists() { - let stmt = first_stmt("DROP STREAMING TABLE IF EXISTS my_sink"); + let stmt = first_classified("DROP STREAMING TABLE IF EXISTS my_sink"); assert!(is_type(stmt.as_ref(), "DropStreamingTableStatement")); } #[test] fn test_parse_show_streaming_tables() { - let stmt = first_stmt("SHOW STREAMING TABLES"); + let stmt = first_classified("SHOW STREAMING TABLES"); assert!(is_type(stmt.as_ref(), "ShowStreamingTables")); } #[test] fn test_parse_show_create_streaming_table() { - let stmt = first_stmt("SHOW CREATE STREAMING TABLE my_sink"); + let stmt = first_classified("SHOW CREATE STREAMING TABLE my_sink"); assert!(is_type(stmt.as_ref(), "ShowCreateStreamingTable")); } - /// `CREATE STREAMING TABLE` is the sink DDL supported by FunctionStream (not `CREATE STREAM TABLE`). #[test] fn test_parse_create_streaming_table() { let sql = concat!( @@ -315,7 +242,7 @@ mod tests { "WITH ('connector' = 'kafka') ", "AS SELECT id FROM src", ); - let stmt = first_stmt(sql); + let stmt = first_classified(sql); assert!( is_type(stmt.as_ref(), "StreamingTableStatement"), "expected StreamingTableStatement, got {:?}", @@ -330,22 +257,22 @@ mod tests { "with ('connector' = 'memory') ", "as select 1 as x", ); - let stmt = first_stmt(sql); + let stmt = first_classified(sql); assert!(is_type(stmt.as_ref(), "StreamingTableStatement")); } #[test] fn test_parse_case_insensitive() { assert!(is_type( - first_stmt("create function with ('function_path'='./test.wasm')").as_ref(), + first_classified("create function with ('function_path'='./test.wasm')").as_ref(), "CreateFunction" )); assert!(is_type( - first_stmt("show functions").as_ref(), + first_classified("show functions").as_ref(), "ShowFunctions" )); assert!(is_type( - first_stmt("start function my_task").as_ref(), + first_classified("start function my_task").as_ref(), "StartFunction" )); } @@ -356,27 +283,24 @@ mod tests { "CREATE TABLE t1 (id INT); ", "CREATE STREAMING TABLE sk WITH ('connector' = 'kafka') AS SELECT id FROM t1", ); - let stmts = parse_sql(sql).unwrap(); - assert_eq!(stmts.len(), 2); - assert!(is_type(stmts[0].as_ref(), "CreateTable")); - assert!(is_type(stmts[1].as_ref(), "StreamingTableStatement")); - } - - #[test] - fn test_parse_empty() { - assert!(parse_sql("").is_err()); - assert!(parse_sql(" ").is_err()); + let mut ast = parse_sql(sql).unwrap(); + assert_eq!(ast.len(), 2); + let s0 = classify_statement(ast.remove(0)).unwrap(); + let s1 = classify_statement(ast.remove(0)).unwrap(); + assert!(is_type(s0.as_ref(), "CreateTable")); + assert!(is_type(s1.as_ref(), "StreamingTableStatement")); } #[test] - fn test_parse_unsupported_statement() { - let result = parse_sql("SELECT 1"); - assert!(result.is_err()); + fn test_classify_unsupported_statement() { + let mut stmts = parse_sql("SELECT 1").unwrap(); + assert!(classify_statement(stmts.remove(0)).is_err()); } #[test] fn test_insert_not_supported() { - let err = parse_sql("INSERT INTO sink SELECT * FROM src").unwrap_err(); + let mut stmts = parse_sql("INSERT INTO sink SELECT * FROM src").unwrap(); + let err = classify_statement(stmts.remove(0)).unwrap_err(); let msg = err.to_string(); assert!( msg.contains("INSERT") && msg.contains("not supported"), @@ -396,7 +320,7 @@ mod tests { 'parallelism'='4', 'memory-limit'='256mb' )"#; - let stmt = first_stmt(sql); + let stmt = first_classified(sql); assert!(is_type(stmt.as_ref(), "CreateFunction")); } } diff --git a/src/coordinator/statement/create_function.rs b/src/coordinator/src/statement/create_function.rs similarity index 100% rename from src/coordinator/statement/create_function.rs rename to src/coordinator/src/statement/create_function.rs diff --git a/src/coordinator/statement/create_python_function.rs b/src/coordinator/src/statement/create_python_function.rs similarity index 100% rename from src/coordinator/statement/create_python_function.rs rename to src/coordinator/src/statement/create_python_function.rs diff --git a/src/coordinator/statement/create_table.rs b/src/coordinator/src/statement/create_table.rs similarity index 100% rename from src/coordinator/statement/create_table.rs rename to src/coordinator/src/statement/create_table.rs diff --git a/src/coordinator/statement/drop_function.rs b/src/coordinator/src/statement/drop_function.rs similarity index 100% rename from src/coordinator/statement/drop_function.rs rename to src/coordinator/src/statement/drop_function.rs diff --git a/src/coordinator/statement/drop_streaming_table.rs b/src/coordinator/src/statement/drop_streaming_table.rs similarity index 100% rename from src/coordinator/statement/drop_streaming_table.rs rename to src/coordinator/src/statement/drop_streaming_table.rs diff --git a/src/coordinator/statement/drop_table.rs b/src/coordinator/src/statement/drop_table.rs similarity index 100% rename from src/coordinator/statement/drop_table.rs rename to src/coordinator/src/statement/drop_table.rs diff --git a/src/coordinator/statement/mod.rs b/src/coordinator/src/statement/mod.rs similarity index 100% rename from src/coordinator/statement/mod.rs rename to src/coordinator/src/statement/mod.rs diff --git a/src/coordinator/statement/show_catalog_tables.rs b/src/coordinator/src/statement/show_catalog_tables.rs similarity index 100% rename from src/coordinator/statement/show_catalog_tables.rs rename to src/coordinator/src/statement/show_catalog_tables.rs diff --git a/src/coordinator/statement/show_create_streaming_table.rs b/src/coordinator/src/statement/show_create_streaming_table.rs similarity index 100% rename from src/coordinator/statement/show_create_streaming_table.rs rename to src/coordinator/src/statement/show_create_streaming_table.rs diff --git a/src/coordinator/statement/show_create_table.rs b/src/coordinator/src/statement/show_create_table.rs similarity index 100% rename from src/coordinator/statement/show_create_table.rs rename to src/coordinator/src/statement/show_create_table.rs diff --git a/src/coordinator/statement/show_functions.rs b/src/coordinator/src/statement/show_functions.rs similarity index 100% rename from src/coordinator/statement/show_functions.rs rename to src/coordinator/src/statement/show_functions.rs diff --git a/src/coordinator/statement/show_streaming_tables.rs b/src/coordinator/src/statement/show_streaming_tables.rs similarity index 100% rename from src/coordinator/statement/show_streaming_tables.rs rename to src/coordinator/src/statement/show_streaming_tables.rs diff --git a/src/coordinator/statement/start_function.rs b/src/coordinator/src/statement/start_function.rs similarity index 100% rename from src/coordinator/statement/start_function.rs rename to src/coordinator/src/statement/start_function.rs diff --git a/src/coordinator/statement/stop_function.rs b/src/coordinator/src/statement/stop_function.rs similarity index 100% rename from src/coordinator/statement/stop_function.rs rename to src/coordinator/src/statement/stop_function.rs diff --git a/src/coordinator/statement/streaming_table.rs b/src/coordinator/src/statement/streaming_table.rs similarity index 100% rename from src/coordinator/statement/streaming_table.rs rename to src/coordinator/src/statement/streaming_table.rs diff --git a/src/coordinator/statement/visitor.rs b/src/coordinator/src/statement/visitor.rs similarity index 100% rename from src/coordinator/statement/visitor.rs rename to src/coordinator/src/statement/visitor.rs diff --git a/src/coordinator/streaming_table_options.rs b/src/coordinator/src/streaming_table_options.rs similarity index 100% rename from src/coordinator/streaming_table_options.rs rename to src/coordinator/src/streaming_table_options.rs diff --git a/src/coordinator/tool/mod.rs b/src/coordinator/src/tool/mod.rs similarity index 100% rename from src/coordinator/tool/mod.rs rename to src/coordinator/src/tool/mod.rs diff --git a/src/function-stream/Cargo.toml b/src/function-stream/Cargo.toml new file mode 100644 index 00000000..baf9d5ea --- /dev/null +++ b/src/function-stream/Cargo.toml @@ -0,0 +1,86 @@ +[package] +name = "function-stream" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream" +path = "src/lib.rs" + +[[bin]] +name = "function-stream" +path = "src/main.rs" + +[dependencies] +tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "sync", "time", "net", "signal"] } +serde = { version = "1.0", features = ["derive"] } +serde_yaml = "0.9" +serde_json = "1.0" +uuid = { version = "1.0", features = ["v4", "v7"] } +log = "0.4" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing-appender = "0.2" +anyhow = "1.0" +thiserror = "2" +tonic = { version = "0.12", features = ["default"] } +async-trait = "0.1" +num_cpus = "1.0" +protocol = { path = "../../protocol" } +function-stream-config = { path = "../config" } +function-stream-logger = { path = "../logger" } +function-stream-runtime-common = { path = "../runtime_common" } +function-stream-streaming-planner = { path = "../streaming_planner" } +prost = "0.13" +rdkafka = { version = "0.38", features = ["cmake-build", "ssl", "gssapi", "curl"] } +crossbeam-channel = "0.5" +wasmtime = { version = "41.0.3", features = ["component-model", "async"] } +base64 = "0.22" +wasmtime-wasi = "41.0.3" +rocksdb = { version = "0.21", features = ["multi-threaded-cf", "lz4"] } +bincode = { version = "2", features = ["serde"] } +chrono = "0.4" +tokio-stream = "0.1.18" +lru = "0.12" +parking_lot = "0.12" +arrow = { version = "55", default-features = false } +arrow-array = "55" +arrow-ipc = "55" +arrow-schema = { version = "55", features = ["serde"] } +parquet = "55" +object_store = { version = "0.12.5", features = ["aws"] } +bytes = "1" +futures = "0.3" +serde_json_path = "0.7" +xxhash-rust = { version = "0.8", features = ["xxh3"] } +proctitle = "0.1" +unicase = "2.7" +petgraph = "0.7" +rand = { version = "0.8", features = ["small_rng"] } +itertools = "0.14" +strum = { version = "0.26", features = ["derive"] } + +arrow-json = { version = "55.2.0" } +apache-avro = "0.21" +datafusion = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-common = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-execution = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-expr = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-physical-expr = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-proto = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } + +sqlparser = { git = "https://github.com/FunctionStream/sqlparser-rs", branch = "0.58.0/fs" } + +ahash = "0.8" +governor = "0.8.0" +lance = { version = "4.0.0", default-features = false, features = ["aws"] } +arrow-array-lance = { package = "arrow-array", version = "57.3.0" } +arrow-ipc-lance = { package = "arrow-ipc", version = "57.3.0" } + +[features] +default = ["incremental-cache", "python"] +incremental-cache = ["wasmtime/incremental-cache"] +python = [] + +[dev-dependencies] +tempfile = "3.27.0" diff --git a/src/storage/mod.rs b/src/function-stream/src/lib.rs similarity index 70% rename from src/storage/mod.rs rename to src/function-stream/src/lib.rs index ec32bdfa..a3fc9234 100644 --- a/src/storage/mod.rs +++ b/src/function-stream/src/lib.rs @@ -10,12 +10,39 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Library crate for function-stream + +#![allow(dead_code)] + use std::sync::Arc; use anyhow::Context; +pub use function_stream_config as config; +#[path = "../../coordinator/src/coordinator_body.rs"] +pub mod coordinator; +pub use function_stream_logger as logging; + +pub use function_stream_runtime_common::{common, memory}; + +#[path = "../../streaming_runtime/src/streaming/mod.rs"] +pub mod streaming; + +#[path = "../../streaming_runtime/src/util/mod.rs"] +pub mod util; + +#[path = "../../wasm_runtime/src/wasm/mod.rs"] +pub mod wasm; + +pub use wasm::{input, output, processor}; + +#[path = "../../wasm_runtime/src/state_backend/mod.rs"] pub mod state_backend; + +#[path = "../../catalog_storage/src/stream_catalog/mod.rs"] pub mod stream_catalog; + +#[path = "../../catalog_storage/src/task/mod.rs"] pub mod task; /// Install the process-global [`stream_catalog::CatalogManager`] from configuration. @@ -51,3 +78,7 @@ pub fn initialize_stream_catalog(config: &crate::config::GlobalConfig) -> anyhow CatalogManager::init_global(store).context("Stream catalog (CatalogManager) global init failed") } + +#[path = "../../servicer/src/servicer_body.rs"] +pub mod server; +pub use function_stream_streaming_planner as sql; diff --git a/src/main.rs b/src/function-stream/src/main.rs similarity index 77% rename from src/main.rs rename to src/function-stream/src/main.rs index 46da3c7a..b7e1093f 100644 --- a/src/main.rs +++ b/src/function-stream/src/main.rs @@ -12,15 +12,71 @@ #![allow(dead_code)] -mod config; +pub use function_stream_config as config; +#[path = "../../coordinator/src/coordinator_body.rs"] mod coordinator; -mod logging; -mod runtime; -mod server; -mod sql; -mod storage; +pub use function_stream_logger as logging; + +pub use function_stream_runtime_common::{common, memory}; + +use std::sync::Arc; use anyhow::{Context, Result}; + +#[path = "../../streaming_runtime/src/streaming/mod.rs"] +mod streaming; + +#[path = "../../streaming_runtime/src/util/mod.rs"] +mod util; + +#[path = "../../wasm_runtime/src/wasm/mod.rs"] +mod wasm; + +pub use wasm::{input, output, processor}; + +#[path = "../../wasm_runtime/src/state_backend/mod.rs"] +mod state_backend; + +#[path = "../../catalog_storage/src/stream_catalog/mod.rs"] +mod stream_catalog; + +#[path = "../../catalog_storage/src/task/mod.rs"] +mod task; + +pub fn initialize_stream_catalog(config: &crate::config::GlobalConfig) -> anyhow::Result<()> { + use stream_catalog::{CatalogManager, InMemoryMetaStore, MetaStore, RocksDbMetaStore}; + + let store: Arc = if !config.stream_catalog.persist { + Arc::new(InMemoryMetaStore::new()) + } else { + let path = config + .stream_catalog + .db_path + .as_ref() + .map(|p| crate::config::resolve_path(p)) + .unwrap_or_else(|| crate::config::get_data_dir().join("catalog.db")); + + std::fs::create_dir_all(&path).with_context(|| { + format!( + "Failed to create stream catalog RocksDB directory {}", + path.display() + ) + })?; + + Arc::new(RocksDbMetaStore::open(&path).with_context(|| { + format!( + "Failed to open stream catalog RocksDB at {}", + path.display() + ) + })?) + }; + + CatalogManager::init_global(store).context("Stream catalog (CatalogManager) global init failed") +} + +#[path = "../../servicer/src/servicer_body.rs"] +mod server; +pub use function_stream_streaming_planner as sql; use std::thread; use tokio::sync::oneshot; @@ -144,7 +200,8 @@ fn setup_environment() -> Result { config::GlobalConfig::default() }; - logging::init_logging(&config.logging).context("Logging initialization failed")?; + function_stream_logger::init_logging(&config.logging) + .context("Logging initialization failed")?; log::debug!( "Environment initialized. Data: {}, Conf: {}", diff --git a/src/logger/Cargo.toml b/src/logger/Cargo.toml new file mode 100644 index 00000000..273579a8 --- /dev/null +++ b/src/logger/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "function-stream-logger" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_logger" +path = "src/lib.rs" + +[dependencies] +anyhow = "1.0" +function-stream-config = { path = "../config" } +tracing = "0.1" +tracing-appender = "0.2" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/src/logger/src/lib.rs b/src/logger/src/lib.rs new file mode 100644 index 00000000..53fb0b31 --- /dev/null +++ b/src/logger/src/lib.rs @@ -0,0 +1,9 @@ +//! Logging setup and helpers. + +pub mod config { + pub use function_stream_config::*; +} + +mod logging; + +pub use logging::init_logging; diff --git a/src/logging/mod.rs b/src/logger/src/logging.rs similarity index 100% rename from src/logging/mod.rs rename to src/logger/src/logging.rs diff --git a/src/runtime_common/Cargo.toml b/src/runtime_common/Cargo.toml new file mode 100644 index 00000000..f8e98ed6 --- /dev/null +++ b/src/runtime_common/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "function-stream-runtime-common" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_runtime_common" +path = "src/lib.rs" + +[dependencies] +arrow-array = "55" +bincode = { version = "2", features = ["serde"] } +parking_lot = "0.12" +serde = { version = "1.0", features = ["derive"] } +tokio = { version = "1.0", features = ["sync"] } +tracing = "0.1" diff --git a/src/runtime/common/component_state.rs b/src/runtime_common/src/common/component_state.rs similarity index 100% rename from src/runtime/common/component_state.rs rename to src/runtime_common/src/common/component_state.rs diff --git a/src/runtime/common/mod.rs b/src/runtime_common/src/common/mod.rs similarity index 100% rename from src/runtime/common/mod.rs rename to src/runtime_common/src/common/mod.rs diff --git a/src/runtime/common/task_completion.rs b/src/runtime_common/src/common/task_completion.rs similarity index 100% rename from src/runtime/common/task_completion.rs rename to src/runtime_common/src/common/task_completion.rs diff --git a/src/runtime/mod.rs b/src/runtime_common/src/lib.rs similarity index 81% rename from src/runtime/mod.rs rename to src/runtime_common/src/lib.rs index 8c72b507..1c97062b 100644 --- a/src/runtime/mod.rs +++ b/src/runtime_common/src/lib.rs @@ -10,14 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Runtime module +//! Shared runtime building blocks (task lifecycle, memory pool) used by streaming and WASM. pub mod common; pub mod memory; -pub mod streaming; -pub mod util; -pub mod wasm; - -pub use wasm::input; -pub use wasm::output; -pub use wasm::processor; +pub mod streaming_protocol; diff --git a/src/runtime/memory/block.rs b/src/runtime_common/src/memory/block.rs similarity index 100% rename from src/runtime/memory/block.rs rename to src/runtime_common/src/memory/block.rs diff --git a/src/runtime/memory/error.rs b/src/runtime_common/src/memory/error.rs similarity index 100% rename from src/runtime/memory/error.rs rename to src/runtime_common/src/memory/error.rs diff --git a/src/runtime/memory/global.rs b/src/runtime_common/src/memory/global.rs similarity index 100% rename from src/runtime/memory/global.rs rename to src/runtime_common/src/memory/global.rs diff --git a/src/runtime/memory/mod.rs b/src/runtime_common/src/memory/mod.rs similarity index 100% rename from src/runtime/memory/mod.rs rename to src/runtime_common/src/memory/mod.rs diff --git a/src/runtime/memory/pool.rs b/src/runtime_common/src/memory/pool.rs similarity index 100% rename from src/runtime/memory/pool.rs rename to src/runtime_common/src/memory/pool.rs diff --git a/src/runtime/memory/ticket.rs b/src/runtime_common/src/memory/ticket.rs similarity index 100% rename from src/runtime/memory/ticket.rs rename to src/runtime_common/src/memory/ticket.rs diff --git a/src/runtime/streaming/protocol/event.rs b/src/runtime_common/src/streaming_protocol.rs similarity index 79% rename from src/runtime/streaming/protocol/event.rs rename to src/runtime_common/src/streaming_protocol.rs index 093d99ba..7c578f7a 100644 --- a/src/runtime/streaming/protocol/event.rs +++ b/src/runtime_common/src/streaming_protocol.rs @@ -10,15 +10,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! Streaming control-plane types shared by the SQL planner and the execution runtime. + use bincode::{Decode, Encode}; use serde::{Deserialize, Serialize}; -use std::sync::Arc; use std::time::SystemTime; -use arrow_array::RecordBatch; - -use crate::runtime::memory::MemoryTicket; - #[derive(Debug, Copy, Clone, PartialEq, Eq, Encode, Decode, Serialize, Deserialize)] pub enum Watermark { EventTime(SystemTime), @@ -33,44 +30,6 @@ pub struct CheckpointBarrier { pub then_stop: bool, } -#[derive(Debug, Clone)] -pub enum StreamEvent { - Data(RecordBatch), - Watermark(Watermark), - Barrier(CheckpointBarrier), - EndOfStream, -} - -#[derive(Debug, Clone)] -pub enum StreamOutput { - Forward(RecordBatch), - Keyed(u64, RecordBatch), - Broadcast(RecordBatch), - Watermark(Watermark), -} - -#[derive(Debug, Clone)] -pub struct TrackedEvent { - pub event: StreamEvent, - pub _ticket: Option>, -} - -impl TrackedEvent { - pub fn new(event: StreamEvent, ticket: Option) -> Self { - Self { - event, - _ticket: ticket.map(Arc::new), - } - } - - pub fn control(event: StreamEvent) -> Self { - Self { - event, - _ticket: None, - } - } -} - pub fn merge_watermarks(per_input: &[Option]) -> Option { if per_input.iter().any(|w| w.is_none()) { return None; diff --git a/src/servicer/Cargo.toml b/src/servicer/Cargo.toml new file mode 100644 index 00000000..16cd3109 --- /dev/null +++ b/src/servicer/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "function-stream-servicer" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_servicer" +path = "src/lib.rs" diff --git a/src/server/handler.rs b/src/servicer/src/handler.rs similarity index 96% rename from src/server/handler.rs rename to src/servicer/src/handler.rs index 0319e352..c884eb6f 100644 --- a/src/server/handler.rs +++ b/src/servicer/src/handler.rs @@ -26,7 +26,7 @@ use protocol::service::{ use crate::coordinator::{ Coordinator, CreateFunction, CreatePythonFunction, DataSet, DropFunction, PythonModule, - ShowFunctions, ShowFunctionsResult, StartFunction, Statement, StopFunction, + ShowFunctions, ShowFunctionsResult, StartFunction, Statement, StopFunction, classify_statement, }; use crate::sql::parse::parse_sql; @@ -135,12 +135,20 @@ impl FunctionStreamService for FunctionStreamServiceImpl { let timer = Instant::now(); let req = request.into_inner(); - let statements = parse_sql(&req.sql).map_err(|e| { + let ast = parse_sql(&req.sql).map_err(|e| { let detail = e.to_string(); warn!("SQL parse rejection: {}", detail); Status::invalid_argument(detail) })?; + let statements: Result>, _> = + ast.into_iter().map(classify_statement).collect(); + let statements = statements.map_err(|e| { + let detail = e.to_string(); + warn!("SQL classification rejection: {}", detail); + Status::invalid_argument(detail) + })?; + if statements.is_empty() { return Ok(TonicResponse::new(Self::build_success_response( StatusCode::Ok, diff --git a/src/server/initializer.rs b/src/servicer/src/initializer.rs similarity index 87% rename from src/server/initializer.rs rename to src/servicer/src/initializer.rs index 8a04608e..78b16c73 100644 --- a/src/server/initializer.rs +++ b/src/servicer/src/initializer.rs @@ -113,7 +113,7 @@ pub fn build_core_registry() -> ComponentRegistry { builder .register( "StreamCatalog", - crate::storage::stream_catalog::initialize_stream_catalog, + crate::stream_catalog::initialize_stream_catalog, ) .register("Coordinator", initialize_coordinator) .build() @@ -124,16 +124,16 @@ pub fn bootstrap_system(config: &GlobalConfig) -> Result<()> { registry.initialize_all(config)?; - crate::storage::stream_catalog::restore_global_catalog_from_store(); - crate::storage::stream_catalog::restore_streaming_jobs_from_store(); + crate::stream_catalog::restore_global_catalog_from_store(); + crate::stream_catalog::restore_streaming_jobs_from_store(); info!("System bootstrap finished. Node is ready to accept traffic."); Ok(()) } fn initialize_wasm_cache(config: &GlobalConfig) -> Result<()> { - crate::runtime::processor::wasm::wasm_cache::set_cache_config( - crate::runtime::processor::wasm::wasm_cache::WasmCacheConfig { + crate::processor::wasm::wasm_cache::set_cache_config( + crate::processor::wasm::wasm_cache::WasmCacheConfig { enabled: config.wasm.enable_cache, cache_dir: crate::config::paths::resolve_path(&config.wasm.cache_dir), max_size: config.wasm.max_cache_size, @@ -151,14 +151,14 @@ fn initialize_wasm_cache(config: &GlobalConfig) -> Result<()> { } fn initialize_task_manager(config: &GlobalConfig) -> Result<()> { - crate::runtime::wasm::taskexecutor::TaskManager::init(config) + crate::wasm::taskexecutor::TaskManager::init(config) .context("TaskManager service failed to start")?; Ok(()) } #[cfg(feature = "python")] fn initialize_python_service(config: &GlobalConfig) -> Result<()> { - crate::runtime::processor::python::PythonService::initialize(config) + crate::processor::python::PythonService::initialize(config) .context("Python Runtime initialization failed")?; Ok(()) } @@ -168,9 +168,9 @@ fn initialize_memory_service(config: &GlobalConfig) -> Result<()> { } fn initialize_job_manager(config: &GlobalConfig) -> Result<()> { - use crate::runtime::streaming::factory::OperatorFactory; - use crate::runtime::streaming::factory::Registry; - use crate::runtime::streaming::job::{JobManager, StateConfig}; + use crate::streaming::factory::OperatorFactory; + use crate::streaming::factory::Registry; + use crate::streaming::job::{JobManager, StateConfig}; use std::sync::Arc; let per_operator_memory_bytes = config @@ -199,16 +199,16 @@ fn initialize_job_manager(config: &GlobalConfig) -> Result<()> { } fn initialize_coordinator(_config: &GlobalConfig) -> Result<()> { - crate::runtime::wasm::taskexecutor::TaskManager::get() + crate::wasm::taskexecutor::TaskManager::get() .context("Dependency violation: Coordinator requires TaskManager")?; - crate::runtime::memory::try_global_memory_pool() + crate::memory::try_global_memory_pool() .context("Dependency violation: Coordinator requires MemoryService")?; - crate::storage::stream_catalog::CatalogManager::global() + crate::stream_catalog::CatalogManager::global() .context("Dependency violation: Coordinator requires StreamCatalog")?; - crate::runtime::streaming::job::JobManager::global() + crate::streaming::job::JobManager::global() .context("Dependency violation: Coordinator requires JobManager")?; Ok(()) diff --git a/src/servicer/src/lib.rs b/src/servicer/src/lib.rs new file mode 100644 index 00000000..8a442937 --- /dev/null +++ b/src/servicer/src/lib.rs @@ -0,0 +1,3 @@ +//! Service layer implementations and request handling. + +pub const CRATE_NAME: &str = "function-stream-servicer"; diff --git a/src/server/memory_service.rs b/src/servicer/src/memory_service.rs similarity index 97% rename from src/server/memory_service.rs rename to src/servicer/src/memory_service.rs index 2ba24eee..5c03350b 100644 --- a/src/server/memory_service.rs +++ b/src/servicer/src/memory_service.rs @@ -52,7 +52,7 @@ impl MemoryService { let total_pool_bytes = streaming_runtime_memory_bytes.saturating_add(operator_state_store_memory_bytes); - crate::runtime::memory::init_global_memory_pool(total_pool_bytes) + crate::memory::init_global_memory_pool(total_pool_bytes) .context("Global memory pool initialization failed")?; info!("MemoryService initialized"); diff --git a/src/server/service.rs b/src/servicer/src/service.rs similarity index 100% rename from src/server/service.rs rename to src/servicer/src/service.rs diff --git a/src/server/mod.rs b/src/servicer/src/servicer_body.rs similarity index 100% rename from src/server/mod.rs rename to src/servicer/src/servicer_body.rs diff --git a/src/sql/logical_node/mod.rs b/src/sql/logical_node/mod.rs deleted file mode 100644 index 2da1b8a4..00000000 --- a/src/sql/logical_node/mod.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub mod logical; - -mod macros; - -pub(crate) mod streaming_operator_blueprint; -pub(crate) use streaming_operator_blueprint::{CompiledTopologyNode, StreamingOperatorBlueprint}; - -pub(crate) mod aggregate; -pub(crate) mod debezium; -pub(crate) mod join; -pub(crate) mod key_calculation; -pub(crate) mod lookup; -pub(crate) mod projection; -pub(crate) mod remote_table; -pub(crate) mod sink; -pub(crate) mod table_source; -pub(crate) mod updating_aggregate; -pub(crate) mod watermark_node; -pub(crate) mod windows_function; - -pub(crate) mod timestamp_append; -pub(crate) use timestamp_append::SystemTimestampInjectorNode; - -pub(crate) mod async_udf; -pub(crate) use async_udf::AsyncFunctionExecutionNode; - -pub(crate) mod is_retract; -pub(crate) use is_retract::IsRetractExtension; - -mod extension_try_from; diff --git a/src/sqlparser/Cargo.toml b/src/sqlparser/Cargo.toml new file mode 100644 index 00000000..7d6b40a6 --- /dev/null +++ b/src/sqlparser/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "function-stream-sqlparser" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_sqlparser" +path = "src/lib.rs" diff --git a/src/sqlparser/src/lib.rs b/src/sqlparser/src/lib.rs new file mode 100644 index 00000000..b436f895 --- /dev/null +++ b/src/sqlparser/src/lib.rs @@ -0,0 +1,3 @@ +//! SQL parsing facade for FunctionStream. + +pub const CRATE_NAME: &str = "function-stream-sqlparser"; diff --git a/src/streaming_planner/Cargo.toml b/src/streaming_planner/Cargo.toml new file mode 100644 index 00000000..766aba1c --- /dev/null +++ b/src/streaming_planner/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "function-stream-streaming-planner" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_streaming_planner" +path = "src/lib.rs" + +[dependencies] +protocol = { path = "../../protocol" } +prost = "0.13" +function-stream-config = { path = "../config" } +function-stream-runtime-common = { path = "../runtime_common" } +tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "sync", "time", "net"] } +tokio-stream = "0.1.18" +anyhow = "1.0" +xxhash-rust = { version = "0.8", features = ["xxh3"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +thiserror = "2" +tracing = "0.1" +async-trait = "0.1" +futures = "0.3" +itertools = "0.14" +petgraph = "0.7" +unicase = "2.7" +rand = { version = "0.8", features = ["small_rng"] } +bincode = { version = "2", features = ["serde"] } +chrono = "0.4" +bytes = "1" +ahash = "0.8" +strum = { version = "0.26", features = ["derive"] } +serde_json_path = "0.7" + +arrow = { version = "55", default-features = false } +arrow-array = "55" +arrow-schema = { version = "55", features = ["serde"] } +arrow-json = { version = "55.2.0" } +apache-avro = "0.21" + +datafusion = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-common = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-execution = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-expr = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-physical-expr = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-proto = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } + +sqlparser = { git = "https://github.com/FunctionStream/sqlparser-rs", branch = "0.58.0/fs" } diff --git a/src/sql/analysis/aggregate_rewriter.rs b/src/streaming_planner/src/analysis/aggregate_rewriter.rs similarity index 96% rename from src/sql/analysis/aggregate_rewriter.rs rename to src/streaming_planner/src/analysis/aggregate_rewriter.rs index ddcb0294..22dcb03c 100644 --- a/src/sql/analysis/aggregate_rewriter.rs +++ b/src/streaming_planner/src/analysis/aggregate_rewriter.rs @@ -17,19 +17,19 @@ use datafusion::logical_expr::{Aggregate, Expr, Extension, LogicalPlan, Projecti use datafusion::prelude::col; use std::sync::Arc; -use crate::sql::analysis::streaming_window_analzer::StreamingWindowAnalzer; -use crate::sql::logical_node::aggregate::StreamWindowAggregateNode; -use crate::sql::logical_node::key_calculation::{KeyExtractionNode, KeyExtractionStrategy}; -use crate::sql::logical_node::updating_aggregate::ContinuousAggregateNode; -use crate::sql::schema::StreamSchemaProvider; -use crate::sql::types::{ +use crate::analysis::streaming_window_analzer::StreamingWindowAnalzer; +use crate::logical_node::aggregate::StreamWindowAggregateNode; +use crate::logical_node::key_calculation::{KeyExtractionNode, KeyExtractionStrategy}; +use crate::logical_node::updating_aggregate::ContinuousAggregateNode; +use crate::schema::StreamSchemaProvider; +use crate::types::{ QualifiedField, TIMESTAMP_FIELD, WindowBehavior, WindowType, build_df_schema_with_metadata, extract_qualified_fields, extract_window_type, }; /// AggregateRewriter transforms batch DataFusion aggregates into streaming stateful operators. /// It handles windowing (Tumble/Hop/Session), watermarks, and continuous updating aggregates. -pub(crate) struct AggregateRewriter<'a> { +pub struct AggregateRewriter<'a> { pub schema_provider: &'a StreamSchemaProvider, } diff --git a/src/sql/analysis/async_udf_rewriter.rs b/src/streaming_planner/src/analysis/async_udf_rewriter.rs similarity index 95% rename from src/sql/analysis/async_udf_rewriter.rs rename to src/streaming_planner/src/analysis/async_udf_rewriter.rs index d6d9b54b..80f3828d 100644 --- a/src/sql/analysis/async_udf_rewriter.rs +++ b/src/streaming_planner/src/analysis/async_udf_rewriter.rs @@ -10,10 +10,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sql::common::constants::sql_field; -use crate::sql::logical_node::AsyncFunctionExecutionNode; -use crate::sql::logical_node::remote_table::RemoteTableBoundaryNode; -use crate::sql::schema::StreamSchemaProvider; +use crate::common::constants::sql_field; +use crate::logical_node::AsyncFunctionExecutionNode; +use crate::logical_node::remote_table::RemoteTableBoundaryNode; +use crate::schema::StreamSchemaProvider; use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRewriter}; use datafusion::common::{Column, Result as DFResult, TableReference, plan_err}; use datafusion::logical_expr::expr::ScalarFunction; diff --git a/src/sql/analysis/join_rewriter.rs b/src/streaming_planner/src/analysis/join_rewriter.rs similarity index 94% rename from src/sql/analysis/join_rewriter.rs rename to src/streaming_planner/src/analysis/join_rewriter.rs index 8a9e5280..07bc0006 100644 --- a/src/sql/analysis/join_rewriter.rs +++ b/src/streaming_planner/src/analysis/join_rewriter.rs @@ -10,13 +10,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sql::analysis::streaming_window_analzer::StreamingWindowAnalzer; -use crate::sql::common::TIMESTAMP_FIELD; -use crate::sql::common::constants::mem_exec_join_side; -use crate::sql::logical_node::join::StreamingJoinNode; -use crate::sql::logical_node::key_calculation::KeyExtractionNode; -use crate::sql::schema::StreamSchemaProvider; -use crate::sql::types::{WindowType, build_df_schema_with_metadata, extract_qualified_fields}; +use crate::analysis::streaming_window_analzer::StreamingWindowAnalzer; +use crate::common::TIMESTAMP_FIELD; +use crate::common::constants::mem_exec_join_side; +use crate::logical_node::join::StreamingJoinNode; +use crate::logical_node::key_calculation::KeyExtractionNode; +use crate::schema::StreamSchemaProvider; +use crate::types::{WindowType, build_df_schema_with_metadata, extract_qualified_fields}; use datafusion::common::tree_node::{Transformed, TreeNodeRewriter}; use datafusion::common::{ JoinConstraint, JoinType, Result, ScalarValue, TableReference, not_impl_err, plan_err, @@ -29,7 +29,7 @@ use std::sync::Arc; /// JoinRewriter handles the transformation of standard SQL joins into streaming-capable joins. /// It manages stateful "Updating Joins" and time-aligned "Instant Joins". -pub(crate) struct JoinRewriter<'a> { +pub struct JoinRewriter<'a> { pub schema_provider: &'a StreamSchemaProvider, } diff --git a/src/sql/analysis/mod.rs b/src/streaming_planner/src/analysis/mod.rs similarity index 90% rename from src/sql/analysis/mod.rs rename to src/streaming_planner/src/analysis/mod.rs index 019d8bf1..7ac1d4e8 100644 --- a/src/sql/analysis/mod.rs +++ b/src/streaming_planner/src/analysis/mod.rs @@ -12,12 +12,12 @@ #![allow(clippy::new_without_default)] -pub(crate) mod aggregate_rewriter; -pub(crate) mod join_rewriter; -pub(crate) mod row_time_rewriter; -pub(crate) mod stream_rewriter; -pub(crate) mod streaming_window_analzer; -pub(crate) mod window_function_rewriter; +pub mod aggregate_rewriter; +pub mod join_rewriter; +pub mod row_time_rewriter; +pub mod stream_rewriter; +pub mod streaming_window_analzer; +pub mod window_function_rewriter; pub mod async_udf_rewriter; pub mod sink_input_rewriter; @@ -31,7 +31,7 @@ pub use sink_input_rewriter::SinkInputRewriter; pub use time_window::{TimeWindowNullCheckRemover, TimeWindowUdfChecker}; pub use unnest_rewriter::UNNESTED_COL; -pub use crate::sql::schema::schema_provider::StreamSchemaProvider; +pub use crate::schema::schema_provider::StreamSchemaProvider; use std::collections::HashMap; use std::sync::Arc; @@ -42,11 +42,11 @@ use datafusion::error::DataFusionError; use datafusion::logical_expr::{Extension, LogicalPlan, UserDefinedLogicalNodeCore}; use tracing::{debug, info, instrument}; -use crate::sql::logical_node::StreamingOperatorBlueprint; -use crate::sql::logical_node::key_calculation::{KeyExtractionNode, KeyExtractionStrategy}; -use crate::sql::logical_node::projection::StreamProjectionNode; -use crate::sql::logical_node::sink::StreamEgressNode; -use crate::sql::logical_planner::planner::NamedNode; +use crate::logical_node::StreamingOperatorBlueprint; +use crate::logical_node::key_calculation::{KeyExtractionNode, KeyExtractionStrategy}; +use crate::logical_node::projection::StreamProjectionNode; +use crate::logical_node::sink::StreamEgressNode; +use crate::logical_planner::planner::NamedNode; fn duration_from_sql_expr( expr: &datafusion::sql::sqlparser::ast::Expr, @@ -110,7 +110,7 @@ fn build_sink_inputs(extensions: &[LogicalPlan]) -> HashMap Result { +pub fn maybe_add_key_extension_to_sink(plan: LogicalPlan) -> Result { let LogicalPlan::Extension(ref ext) = plan else { return Ok(plan); }; diff --git a/src/sql/analysis/row_time_rewriter.rs b/src/streaming_planner/src/analysis/row_time_rewriter.rs similarity index 95% rename from src/sql/analysis/row_time_rewriter.rs rename to src/streaming_planner/src/analysis/row_time_rewriter.rs index 13e2a048..aa558696 100644 --- a/src/sql/analysis/row_time_rewriter.rs +++ b/src/streaming_planner/src/analysis/row_time_rewriter.rs @@ -14,8 +14,8 @@ use datafusion::common::tree_node::{Transformed, TreeNodeRewriter}; use datafusion::common::{Column, Result as DFResult}; use datafusion::logical_expr::Expr; -use crate::sql::common::constants::planning_placeholder_udf; -use crate::sql::types::TIMESTAMP_FIELD; +use crate::common::constants::planning_placeholder_udf; +use crate::types::TIMESTAMP_FIELD; /// Replaces the virtual `row_time()` scalar function with a physical reference to `_timestamp`. /// diff --git a/src/sql/analysis/sink_input_rewriter.rs b/src/streaming_planner/src/analysis/sink_input_rewriter.rs similarity index 89% rename from src/sql/analysis/sink_input_rewriter.rs rename to src/streaming_planner/src/analysis/sink_input_rewriter.rs index 201439cc..9f8fdcb7 100644 --- a/src/sql/analysis/sink_input_rewriter.rs +++ b/src/streaming_planner/src/analysis/sink_input_rewriter.rs @@ -10,9 +10,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sql::logical_node::StreamingOperatorBlueprint; -use crate::sql::logical_node::sink::StreamEgressNode; -use crate::sql::logical_planner::planner::NamedNode; +use crate::logical_node::StreamingOperatorBlueprint; +use crate::logical_node::sink::StreamEgressNode; +use crate::logical_planner::planner::NamedNode; use datafusion::common::Result as DFResult; use datafusion::common::tree_node::{Transformed, TreeNodeRecursion, TreeNodeRewriter}; use datafusion::logical_expr::{Extension, LogicalPlan, UserDefinedLogicalNodeCore}; @@ -28,7 +28,7 @@ pub struct SinkInputRewriter<'a> { } impl<'a> SinkInputRewriter<'a> { - pub(crate) fn new(sink_inputs: &'a mut SinkInputs) -> Self { + pub fn new(sink_inputs: &'a mut SinkInputs) -> Self { Self { sink_inputs, was_removed: false, diff --git a/src/sql/analysis/source_metadata_visitor.rs b/src/streaming_planner/src/analysis/source_metadata_visitor.rs similarity index 82% rename from src/sql/analysis/source_metadata_visitor.rs rename to src/streaming_planner/src/analysis/source_metadata_visitor.rs index b37d3a1b..ea2821b9 100644 --- a/src/sql/analysis/source_metadata_visitor.rs +++ b/src/streaming_planner/src/analysis/source_metadata_visitor.rs @@ -10,9 +10,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sql::logical_node::sink::{STREAM_EGRESS_NODE_NAME, StreamEgressNode}; -use crate::sql::logical_node::table_source::{STREAM_INGESTION_NODE_NAME, StreamIngestionNode}; -use crate::sql::schema::StreamSchemaProvider; +use crate::logical_node::sink::{STREAM_EGRESS_NODE_NAME, StreamEgressNode}; +use crate::logical_node::table_source::{STREAM_INGESTION_NODE_NAME, StreamIngestionNode}; +use crate::schema::StreamSchemaProvider; use datafusion::common::Result as DFResult; use datafusion::common::tree_node::{TreeNodeRecursion, TreeNodeVisitor}; use datafusion::logical_expr::{Extension, LogicalPlan}; @@ -51,9 +51,9 @@ impl<'a> SourceMetadataVisitor<'a> { let table = self.schema_provider.get_catalog_table(&table_name)?; match table { - crate::sql::schema::table::CatalogEntity::ExternalConnector(b) => match b.as_ref() { - crate::sql::schema::catalog::ExternalTable::Source(t) => t.registry_id, - crate::sql::schema::catalog::ExternalTable::Lookup(t) => t.registry_id, + crate::schema::table::CatalogEntity::ExternalConnector(b) => match b.as_ref() { + crate::schema::catalog::ExternalTable::Source(t) => t.registry_id, + crate::schema::catalog::ExternalTable::Lookup(t) => t.registry_id, _ => None, }, _ => None, diff --git a/src/sql/analysis/source_rewriter.rs b/src/streaming_planner/src/analysis/source_rewriter.rs similarity index 95% rename from src/sql/analysis/source_rewriter.rs rename to src/streaming_planner/src/analysis/source_rewriter.rs index 620ea336..e1a7c75a 100644 --- a/src/sql/analysis/source_rewriter.rs +++ b/src/streaming_planner/src/analysis/source_rewriter.rs @@ -20,16 +20,16 @@ use datafusion::logical_expr::{ self, BinaryExpr, Expr, Extension, LogicalPlan, Projection, TableScan, }; -use crate::sql::common::UPDATING_META_FIELD; -use crate::sql::logical_node::debezium::UnrollDebeziumPayloadNode; -use crate::sql::logical_node::remote_table::RemoteTableBoundaryNode; -use crate::sql::logical_node::table_source::StreamIngestionNode; -use crate::sql::logical_node::watermark_node::EventTimeWatermarkNode; -use crate::sql::schema::ColumnDescriptor; -use crate::sql::schema::StreamSchemaProvider; -use crate::sql::schema::catalog::{ExternalTable, SourceTable}; -use crate::sql::schema::table::CatalogEntity; -use crate::sql::types::TIMESTAMP_FIELD; +use crate::common::UPDATING_META_FIELD; +use crate::logical_node::debezium::UnrollDebeziumPayloadNode; +use crate::logical_node::remote_table::RemoteTableBoundaryNode; +use crate::logical_node::table_source::StreamIngestionNode; +use crate::logical_node::watermark_node::EventTimeWatermarkNode; +use crate::schema::ColumnDescriptor; +use crate::schema::StreamSchemaProvider; +use crate::schema::catalog::{ExternalTable, SourceTable}; +use crate::schema::table::CatalogEntity; +use crate::types::TIMESTAMP_FIELD; /// Rewrites table scans: projections are lifted out of scans into a dedicated projection node /// (including virtual fields), using a connector table-source extension instead of a bare diff --git a/src/sql/analysis/stream_rewriter.rs b/src/streaming_planner/src/analysis/stream_rewriter.rs similarity index 96% rename from src/sql/analysis/stream_rewriter.rs rename to src/streaming_planner/src/analysis/stream_rewriter.rs index a4393bd1..efc4e40c 100644 --- a/src/sql/analysis/stream_rewriter.rs +++ b/src/streaming_planner/src/analysis/stream_rewriter.rs @@ -13,16 +13,16 @@ use std::sync::Arc; use super::StreamSchemaProvider; -use crate::sql::analysis::TimeWindowNullCheckRemover; -use crate::sql::analysis::row_time_rewriter::RowTimeRewriter; -use crate::sql::analysis::{ +use crate::analysis::TimeWindowNullCheckRemover; +use crate::analysis::row_time_rewriter::RowTimeRewriter; +use crate::analysis::{ aggregate_rewriter::AggregateRewriter, join_rewriter::JoinRewriter, window_function_rewriter::WindowFunctionRewriter, }; -use crate::sql::logical_node::StreamingOperatorBlueprint; -use crate::sql::logical_node::remote_table::RemoteTableBoundaryNode; -use crate::sql::schema::utils::{add_timestamp_field, has_timestamp_field}; -use crate::sql::types::{QualifiedField, TIMESTAMP_FIELD}; +use crate::logical_node::StreamingOperatorBlueprint; +use crate::logical_node::remote_table::RemoteTableBoundaryNode; +use crate::schema::utils::{add_timestamp_field, has_timestamp_field}; +use crate::types::{QualifiedField, TIMESTAMP_FIELD}; use datafusion::common::tree_node::{Transformed, TreeNodeRewriter}; use datafusion::common::{Column, DataFusionError, Result, Spans, TableReference, plan_err}; use datafusion::logical_expr::{ diff --git a/src/sql/analysis/streaming_window_analzer.rs b/src/streaming_planner/src/analysis/streaming_window_analzer.rs similarity index 95% rename from src/sql/analysis/streaming_window_analzer.rs rename to src/streaming_planner/src/analysis/streaming_window_analzer.rs index b8a7f78f..5e6d3e91 100644 --- a/src/sql/analysis/streaming_window_analzer.rs +++ b/src/streaming_planner/src/analysis/streaming_window_analzer.rs @@ -17,16 +17,16 @@ use datafusion::common::tree_node::{TreeNodeRecursion, TreeNodeVisitor}; use datafusion::common::{Column, DFSchema, DataFusionError, Result}; use datafusion::logical_expr::{Expr, Extension, LogicalPlan, expr::Alias}; -use crate::sql::logical_node::aggregate::{STREAM_AGG_EXTENSION_NAME, StreamWindowAggregateNode}; -use crate::sql::logical_node::join::STREAM_JOIN_NODE_TYPE; -use crate::sql::types::{ +use crate::logical_node::aggregate::{STREAM_AGG_EXTENSION_NAME, StreamWindowAggregateNode}; +use crate::logical_node::join::STREAM_JOIN_NODE_TYPE; +use crate::types::{ QualifiedField, WindowBehavior, WindowType, extract_qualified_fields, extract_window_type, }; /// WindowDetectingVisitor identifies windowing strategies and tracks window-carrying fields /// as they propagate upward through the logical plan tree. #[derive(Debug, Default)] -pub(crate) struct StreamingWindowAnalzer { +pub struct StreamingWindowAnalzer { /// The specific window type discovered (Tumble, Hop, etc.) pub(crate) window: Option, /// Set of fields in the current plan node that carry window semantics. @@ -35,7 +35,7 @@ pub(crate) struct StreamingWindowAnalzer { impl StreamingWindowAnalzer { /// Entry point to resolve the WindowType of a given plan branch. - pub(crate) fn get_window(logical_plan: &LogicalPlan) -> Result> { + pub fn get_window(logical_plan: &LogicalPlan) -> Result> { let mut visitor = Self::default(); logical_plan.visit_with_subqueries(&mut visitor)?; Ok(visitor.window) @@ -89,7 +89,7 @@ impl StreamingWindowAnalzer { } } -pub(crate) fn extract_column(expr: &Expr) -> Option<&Column> { +pub fn extract_column(expr: &Expr) -> Option<&Column> { match expr { Expr::Column(column) => Some(column), Expr::Alias(Alias { expr, .. }) => extract_column(expr), diff --git a/src/sql/analysis/time_window.rs b/src/streaming_planner/src/analysis/time_window.rs similarity index 100% rename from src/sql/analysis/time_window.rs rename to src/streaming_planner/src/analysis/time_window.rs diff --git a/src/sql/analysis/udafs.rs b/src/streaming_planner/src/analysis/udafs.rs similarity index 100% rename from src/sql/analysis/udafs.rs rename to src/streaming_planner/src/analysis/udafs.rs diff --git a/src/sql/analysis/unnest_rewriter.rs b/src/streaming_planner/src/analysis/unnest_rewriter.rs similarity index 97% rename from src/sql/analysis/unnest_rewriter.rs rename to src/streaming_planner/src/analysis/unnest_rewriter.rs index 147b1f49..e63878f7 100644 --- a/src/sql/analysis/unnest_rewriter.rs +++ b/src/streaming_planner/src/analysis/unnest_rewriter.rs @@ -18,8 +18,8 @@ use datafusion::common::{Column, Result as DFResult, plan_err}; use datafusion::logical_expr::expr::ScalarFunction; use datafusion::logical_expr::{ColumnUnnestList, Expr, LogicalPlan, Projection, Unnest}; -use crate::sql::common::constants::planning_placeholder_udf; -use crate::sql::types::{QualifiedField, build_df_schema, extract_qualified_fields}; +use crate::common::constants::planning_placeholder_udf; +use crate::types::{QualifiedField, build_df_schema, extract_qualified_fields}; pub const UNNESTED_COL: &str = "__unnested"; diff --git a/src/sql/analysis/window_function_rewriter.rs b/src/streaming_planner/src/analysis/window_function_rewriter.rs similarity index 93% rename from src/sql/analysis/window_function_rewriter.rs rename to src/streaming_planner/src/analysis/window_function_rewriter.rs index c1e3396d..569a3c00 100644 --- a/src/sql/analysis/window_function_rewriter.rs +++ b/src/streaming_planner/src/analysis/window_function_rewriter.rs @@ -20,14 +20,14 @@ use datafusion_common::DataFusionError; use std::sync::Arc; use tracing::debug; -use crate::sql::analysis::streaming_window_analzer::{StreamingWindowAnalzer, extract_column}; -use crate::sql::logical_node::key_calculation::{KeyExtractionNode, KeyExtractionStrategy}; -use crate::sql::logical_node::windows_function::StreamingWindowFunctionNode; -use crate::sql::types::{WindowType, build_df_schema, extract_qualified_fields}; +use crate::analysis::streaming_window_analzer::{StreamingWindowAnalzer, extract_column}; +use crate::logical_node::key_calculation::{KeyExtractionNode, KeyExtractionStrategy}; +use crate::logical_node::windows_function::StreamingWindowFunctionNode; +use crate::types::{WindowType, build_df_schema, extract_qualified_fields}; /// WindowFunctionRewriter transforms standard SQL Window functions into streaming-compatible /// stateful operators, ensuring proper data partitioning and sorting for distributed execution. -pub(crate) struct WindowFunctionRewriter; +pub struct WindowFunctionRewriter; impl WindowFunctionRewriter { /// Recursively unwraps Aliases to find the underlying WindowFunction. @@ -48,7 +48,7 @@ impl WindowFunctionRewriter { &self, params: &WindowFunctionParams, input: &LogicalPlan, - input_window_fields: &std::collections::HashSet, + input_window_fields: &std::collections::HashSet, ) -> DFResult { let matched: Vec<_> = params .partition_by diff --git a/src/sql/api/checkpoints.rs b/src/streaming_planner/src/api/checkpoints.rs similarity index 98% rename from src/sql/api/checkpoints.rs rename to src/streaming_planner/src/api/checkpoints.rs index d9bdc139..bd326b10 100644 --- a/src/sql/api/checkpoints.rs +++ b/src/streaming_planner/src/api/checkpoints.rs @@ -10,7 +10,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sql::common::to_micros; +use crate::common::to_micros; use serde::{Deserialize, Serialize}; use std::time::SystemTime; diff --git a/src/sql/api/connections.rs b/src/streaming_planner/src/api/connections.rs similarity index 99% rename from src/sql/api/connections.rs rename to src/streaming_planner/src/api/connections.rs index 3c5caf76..d2d92bcb 100644 --- a/src/sql/api/connections.rs +++ b/src/streaming_planner/src/api/connections.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sql::common::formats::{BadData, Format, Framing}; -use crate::sql::common::{FsExtensionType, FsSchema}; +use crate::common::formats::{BadData, Format, Framing}; +use crate::common::{FsExtensionType, FsSchema}; use datafusion::arrow::datatypes::{DataType, Field, Fields, TimeUnit}; use serde::ser::SerializeMap; use serde::{Deserialize, Serialize, Serializer}; diff --git a/src/sql/api/metrics.rs b/src/streaming_planner/src/api/metrics.rs similarity index 100% rename from src/sql/api/metrics.rs rename to src/streaming_planner/src/api/metrics.rs diff --git a/src/sql/api/mod.rs b/src/streaming_planner/src/api/mod.rs similarity index 100% rename from src/sql/api/mod.rs rename to src/streaming_planner/src/api/mod.rs diff --git a/src/sql/api/pipelines.rs b/src/streaming_planner/src/api/pipelines.rs similarity index 98% rename from src/sql/api/pipelines.rs rename to src/streaming_planner/src/api/pipelines.rs index d6cc5253..990c1ba9 100644 --- a/src/sql/api/pipelines.rs +++ b/src/streaming_planner/src/api/pipelines.rs @@ -11,7 +11,7 @@ // limitations under the License. use super::udfs::Udf; -use crate::sql::common::control::ErrorDomain; +use crate::common::control::ErrorDomain; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Clone, Debug)] diff --git a/src/sql/api/public_ids.rs b/src/streaming_planner/src/api/public_ids.rs similarity index 100% rename from src/sql/api/public_ids.rs rename to src/streaming_planner/src/api/public_ids.rs diff --git a/src/sql/api/schema_resolver.rs b/src/streaming_planner/src/api/schema_resolver.rs similarity index 100% rename from src/sql/api/schema_resolver.rs rename to src/streaming_planner/src/api/schema_resolver.rs diff --git a/src/sql/api/udfs.rs b/src/streaming_planner/src/api/udfs.rs similarity index 100% rename from src/sql/api/udfs.rs rename to src/streaming_planner/src/api/udfs.rs diff --git a/src/sql/api/var_str.rs b/src/streaming_planner/src/api/var_str.rs similarity index 100% rename from src/sql/api/var_str.rs rename to src/streaming_planner/src/api/var_str.rs diff --git a/src/sql/common/arrow_ext.rs b/src/streaming_planner/src/common/arrow_ext.rs similarity index 100% rename from src/sql/common/arrow_ext.rs rename to src/streaming_planner/src/common/arrow_ext.rs diff --git a/src/coordinator/plan/ast_utils.rs b/src/streaming_planner/src/common/ast_utils.rs similarity index 98% rename from src/coordinator/plan/ast_utils.rs rename to src/streaming_planner/src/common/ast_utils.rs index d2432888..cc0eb95f 100644 --- a/src/coordinator/plan/ast_utils.rs +++ b/src/streaming_planner/src/common/ast_utils.rs @@ -20,7 +20,7 @@ use datafusion::common::{Result, plan_err}; use datafusion::sql::sqlparser::ast::{Expr as SqlExpr, SqlOption, TableConstraint}; -use crate::sql::common::with_option_keys as opt; +use crate::common::with_option_keys as opt; /// Namespace for AST extraction helpers. pub struct AstUtils; diff --git a/src/sql/common/connector_options.rs b/src/streaming_planner/src/common/connector_options.rs similarity index 100% rename from src/sql/common/connector_options.rs rename to src/streaming_planner/src/common/connector_options.rs diff --git a/src/sql/common/constants.rs b/src/streaming_planner/src/common/constants.rs similarity index 100% rename from src/sql/common/constants.rs rename to src/streaming_planner/src/common/constants.rs diff --git a/src/sql/common/control.rs b/src/streaming_planner/src/common/control.rs similarity index 98% rename from src/sql/common/control.rs rename to src/streaming_planner/src/common/control.rs index eba88596..59ef409e 100644 --- a/src/sql/common/control.rs +++ b/src/streaming_planner/src/common/control.rs @@ -13,7 +13,7 @@ use std::collections::HashMap; use std::time::SystemTime; -use crate::runtime::streaming::protocol::CheckpointBarrier; +use function_stream_runtime_common::streaming_protocol::CheckpointBarrier; /// Control messages sent from the controller to worker tasks. #[derive(Debug, Clone)] diff --git a/src/sql/common/converter.rs b/src/streaming_planner/src/common/converter.rs similarity index 100% rename from src/sql/common/converter.rs rename to src/streaming_planner/src/common/converter.rs diff --git a/src/sql/common/date.rs b/src/streaming_planner/src/common/date.rs similarity index 100% rename from src/sql/common/date.rs rename to src/streaming_planner/src/common/date.rs diff --git a/src/sql/common/debezium.rs b/src/streaming_planner/src/common/debezium.rs similarity index 100% rename from src/sql/common/debezium.rs rename to src/streaming_planner/src/common/debezium.rs diff --git a/src/sql/common/errors.rs b/src/streaming_planner/src/common/errors.rs similarity index 97% rename from src/sql/common/errors.rs rename to src/streaming_planner/src/common/errors.rs index fa4a722e..b7e97400 100644 --- a/src/sql/common/errors.rs +++ b/src/streaming_planner/src/common/errors.rs @@ -67,7 +67,7 @@ impl From for DataflowError { #[macro_export] macro_rules! connector_err { ($($arg:tt)*) => { - $crate::sql::common::errors::DataflowError::Connector(format!($($arg)*)) + $crate::common::errors::DataflowError::Connector(format!($($arg)*)) }; } diff --git a/src/sql/common/format_from_opts.rs b/src/streaming_planner/src/common/format_from_opts.rs similarity index 100% rename from src/sql/common/format_from_opts.rs rename to src/streaming_planner/src/common/format_from_opts.rs diff --git a/src/sql/common/formats.rs b/src/streaming_planner/src/common/formats.rs similarity index 100% rename from src/sql/common/formats.rs rename to src/streaming_planner/src/common/formats.rs diff --git a/src/sql/common/fs_schema.rs b/src/streaming_planner/src/common/fs_schema.rs similarity index 99% rename from src/sql/common/fs_schema.rs rename to src/streaming_planner/src/common/fs_schema.rs index 76a08537..9c548f69 100644 --- a/src/sql/common/fs_schema.rs +++ b/src/streaming_planner/src/common/fs_schema.rs @@ -11,7 +11,7 @@ // limitations under the License. use super::{TIMESTAMP_FIELD, to_nanos}; -use crate::sql::common::converter::Converter; +use crate::common::converter::Converter; use arrow::compute::kernels::cmp::gt_eq; use arrow::compute::kernels::numeric::div; use arrow::compute::{SortColumn, filter_record_batch, lexsort_to_indices, partition, take}; diff --git a/src/sql/common/kafka_catalog.rs b/src/streaming_planner/src/common/kafka_catalog.rs similarity index 100% rename from src/sql/common/kafka_catalog.rs rename to src/streaming_planner/src/common/kafka_catalog.rs diff --git a/src/sql/common/mod.rs b/src/streaming_planner/src/common/mod.rs similarity index 90% rename from src/sql/common/mod.rs rename to src/streaming_planner/src/common/mod.rs index af44cb0f..0bf6bdc0 100644 --- a/src/sql/common/mod.rs +++ b/src/streaming_planner/src/common/mod.rs @@ -10,12 +10,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Shared core types and constants for FunctionStream (`crate::sql::common`). +//! Shared core types and constants for FunctionStream streaming planner (`crate::common`). //! //! Used by the runtime, SQL planner, coordinator, and other subsystems — //! analogous to `arroyo-types` + `arroyo-rpc` in Arroyo. pub mod arrow_ext; +pub mod ast_utils; pub mod connector_options; pub mod constants; pub mod control; @@ -33,11 +34,12 @@ pub mod topology; pub mod with_option_keys; // ── Re-exports from existing modules ── -pub use crate::runtime::streaming::protocol::{CheckpointBarrier, Watermark}; pub use arrow_ext::FsExtensionType; +pub use function_stream_runtime_common::streaming_protocol::{CheckpointBarrier, Watermark}; pub use time_utils::{from_nanos, to_micros, to_millis, to_nanos}; // ── Re-exports from new modules ── +pub use ast_utils::AstUtils; pub use connector_options::ConnectorOptions; pub use formats::{BadData, Format, Framing, JsonCompression, JsonFormat}; pub use fs_schema::{FsSchema, FsSchemaRef}; diff --git a/src/sql/common/operator_config.rs b/src/streaming_planner/src/common/operator_config.rs similarity index 100% rename from src/sql/common/operator_config.rs rename to src/streaming_planner/src/common/operator_config.rs diff --git a/src/sql/common/time_utils.rs b/src/streaming_planner/src/common/time_utils.rs similarity index 100% rename from src/sql/common/time_utils.rs rename to src/streaming_planner/src/common/time_utils.rs diff --git a/src/sql/common/topology.rs b/src/streaming_planner/src/common/topology.rs similarity index 100% rename from src/sql/common/topology.rs rename to src/streaming_planner/src/common/topology.rs diff --git a/src/sql/common/with_option_keys.rs b/src/streaming_planner/src/common/with_option_keys.rs similarity index 100% rename from src/sql/common/with_option_keys.rs rename to src/streaming_planner/src/common/with_option_keys.rs diff --git a/src/sql/connector/config.rs b/src/streaming_planner/src/connector/config.rs similarity index 100% rename from src/sql/connector/config.rs rename to src/streaming_planner/src/connector/config.rs diff --git a/src/sql/connector/factory.rs b/src/streaming_planner/src/connector/factory.rs similarity index 93% rename from src/sql/connector/factory.rs rename to src/streaming_planner/src/connector/factory.rs index 8c37a15a..e89bafa1 100644 --- a/src/sql/connector/factory.rs +++ b/src/streaming_planner/src/connector/factory.rs @@ -18,9 +18,9 @@ use datafusion::common::Result; use super::config::ConnectorConfig; use super::registry::REGISTRY; use super::sink::runtime_config::SinkRuntimeConfig; -use crate::sql::common::connector_options::ConnectorOptions; -use crate::sql::common::formats::{BadData, Format}; -use crate::sql::schema::table_role::TableRole; +use crate::common::connector_options::ConnectorOptions; +use crate::common::formats::{BadData, Format}; +use crate::schema::table_role::TableRole; pub fn build_connector_config( connector_name: &str, diff --git a/src/sql/connector/mod.rs b/src/streaming_planner/src/connector/mod.rs similarity index 100% rename from src/sql/connector/mod.rs rename to src/streaming_planner/src/connector/mod.rs diff --git a/src/sql/connector/provider.rs b/src/streaming_planner/src/connector/provider.rs similarity index 93% rename from src/sql/connector/provider.rs rename to src/streaming_planner/src/connector/provider.rs index 8875ee0c..83e46aa7 100644 --- a/src/sql/connector/provider.rs +++ b/src/streaming_planner/src/connector/provider.rs @@ -14,8 +14,8 @@ use datafusion::common::{DataFusionError, Result}; use super::config::ConnectorConfig; use super::sink::runtime_config::SinkRuntimeProperties; -use crate::sql::common::connector_options::ConnectorOptions; -use crate::sql::common::formats::{BadData, Format}; +use crate::common::connector_options::ConnectorOptions; +use crate::common::formats::{BadData, Format}; pub trait SourceProvider: Send + Sync { fn name(&self) -> &'static str; diff --git a/src/sql/connector/registry.rs b/src/streaming_planner/src/connector/registry.rs similarity index 100% rename from src/sql/connector/registry.rs rename to src/streaming_planner/src/connector/registry.rs diff --git a/src/sql/connector/sink/delta.rs b/src/streaming_planner/src/connector/sink/delta.rs similarity index 83% rename from src/sql/connector/sink/delta.rs rename to src/streaming_planner/src/connector/sink/delta.rs index cd86660d..08d86d96 100644 --- a/src/sql/connector/sink/delta.rs +++ b/src/streaming_planner/src/connector/sink/delta.rs @@ -13,13 +13,13 @@ use datafusion::common::Result; use protocol::function_stream_graph::{DeltaSinkConfig, SinkFormatProto}; -use crate::sql::common::Format; -use crate::sql::common::connector_options::ConnectorOptions; -use crate::sql::common::constants::connector_type; -use crate::sql::connector::config::ConnectorConfig; -use crate::sql::connector::provider::SinkProvider; -use crate::sql::connector::sink::runtime_config::SinkRuntimeProperties; -use crate::sql::connector::sink::utils::SinkUtils; +use crate::common::Format; +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::connector_type; +use crate::connector::config::ConnectorConfig; +use crate::connector::provider::SinkProvider; +use crate::connector::sink::runtime_config::SinkRuntimeProperties; +use crate::connector::sink::utils::SinkUtils; pub struct DeltaSinkConnector; diff --git a/src/sql/connector/sink/filesystem.rs b/src/streaming_planner/src/connector/sink/filesystem.rs similarity index 83% rename from src/sql/connector/sink/filesystem.rs rename to src/streaming_planner/src/connector/sink/filesystem.rs index 224b1805..e529ed79 100644 --- a/src/sql/connector/sink/filesystem.rs +++ b/src/streaming_planner/src/connector/sink/filesystem.rs @@ -13,13 +13,13 @@ use datafusion::common::Result; use protocol::function_stream_graph::{FilesystemSinkConfig, SinkFormatProto}; -use crate::sql::common::Format; -use crate::sql::common::connector_options::ConnectorOptions; -use crate::sql::common::constants::connector_type; -use crate::sql::connector::config::ConnectorConfig; -use crate::sql::connector::provider::SinkProvider; -use crate::sql::connector::sink::runtime_config::SinkRuntimeProperties; -use crate::sql::connector::sink::utils::SinkUtils; +use crate::common::Format; +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::connector_type; +use crate::connector::config::ConnectorConfig; +use crate::connector::provider::SinkProvider; +use crate::connector::sink::runtime_config::SinkRuntimeProperties; +use crate::connector::sink::utils::SinkUtils; pub struct FilesystemSinkConnector; diff --git a/src/sql/connector/sink/iceberg.rs b/src/streaming_planner/src/connector/sink/iceberg.rs similarity index 82% rename from src/sql/connector/sink/iceberg.rs rename to src/streaming_planner/src/connector/sink/iceberg.rs index 12f0d378..4f4854a1 100644 --- a/src/sql/connector/sink/iceberg.rs +++ b/src/streaming_planner/src/connector/sink/iceberg.rs @@ -13,13 +13,13 @@ use datafusion::common::Result; use protocol::function_stream_graph::{IcebergSinkConfig, SinkFormatProto}; -use crate::sql::common::Format; -use crate::sql::common::connector_options::ConnectorOptions; -use crate::sql::common::constants::connector_type; -use crate::sql::connector::config::ConnectorConfig; -use crate::sql::connector::provider::SinkProvider; -use crate::sql::connector::sink::runtime_config::SinkRuntimeProperties; -use crate::sql::connector::sink::utils::SinkUtils; +use crate::common::Format; +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::connector_type; +use crate::connector::config::ConnectorConfig; +use crate::connector::provider::SinkProvider; +use crate::connector::sink::runtime_config::SinkRuntimeProperties; +use crate::connector::sink::utils::SinkUtils; pub struct IcebergSinkConnector; diff --git a/src/sql/connector/sink/kafka.rs b/src/streaming_planner/src/connector/sink/kafka.rs similarity index 94% rename from src/sql/connector/sink/kafka.rs rename to src/streaming_planner/src/connector/sink/kafka.rs index a6fd115c..79292929 100644 --- a/src/sql/connector/sink/kafka.rs +++ b/src/streaming_planner/src/connector/sink/kafka.rs @@ -17,16 +17,16 @@ use protocol::function_stream_graph::{ TimestampFormatProto, format_config, kafka_auth_config, }; -use crate::sql::common::connector_options::ConnectorOptions; -use crate::sql::common::constants::{connector_type, kafka_with_value}; -use crate::sql::common::formats::{ +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::{connector_type, kafka_with_value}; +use crate::common::formats::{ DecimalEncoding as SqlDecimalEncoding, Format as SqlFormat, TimestampFormat as SqlTimestampFormat, }; -use crate::sql::common::with_option_keys as opt; -use crate::sql::connector::config::ConnectorConfig; -use crate::sql::connector::provider::SinkProvider; -use crate::sql::connector::sink::runtime_config::SinkRuntimeProperties; +use crate::common::with_option_keys as opt; +use crate::connector::config::ConnectorConfig; +use crate::connector::provider::SinkProvider; +use crate::connector::sink::runtime_config::SinkRuntimeProperties; pub struct KafkaSinkConnector; diff --git a/src/sql/connector/sink/lancedb.rs b/src/streaming_planner/src/connector/sink/lancedb.rs similarity index 84% rename from src/sql/connector/sink/lancedb.rs rename to src/streaming_planner/src/connector/sink/lancedb.rs index aee79735..87ce99be 100644 --- a/src/sql/connector/sink/lancedb.rs +++ b/src/streaming_planner/src/connector/sink/lancedb.rs @@ -13,13 +13,13 @@ use datafusion::common::Result; use protocol::function_stream_graph::{LanceDbSinkConfig, SinkFormatProto}; -use crate::sql::common::Format; -use crate::sql::common::connector_options::ConnectorOptions; -use crate::sql::common::with_option_keys as opt; -use crate::sql::connector::config::ConnectorConfig; -use crate::sql::connector::provider::SinkProvider; -use crate::sql::connector::sink::runtime_config::SinkRuntimeProperties; -use crate::sql::connector::sink::utils::SinkUtils; +use crate::common::Format; +use crate::common::connector_options::ConnectorOptions; +use crate::common::with_option_keys as opt; +use crate::connector::config::ConnectorConfig; +use crate::connector::provider::SinkProvider; +use crate::connector::sink::runtime_config::SinkRuntimeProperties; +use crate::connector::sink::utils::SinkUtils; pub struct LanceDbSinkConnector; diff --git a/src/sql/connector/sink/mod.rs b/src/streaming_planner/src/connector/sink/mod.rs similarity index 100% rename from src/sql/connector/sink/mod.rs rename to src/streaming_planner/src/connector/sink/mod.rs diff --git a/src/sql/connector/sink/runtime_config.rs b/src/streaming_planner/src/connector/sink/runtime_config.rs similarity index 95% rename from src/sql/connector/sink/runtime_config.rs rename to src/streaming_planner/src/connector/sink/runtime_config.rs index e0ffaeee..b68f7aa9 100644 --- a/src/sql/connector/sink/runtime_config.rs +++ b/src/streaming_planner/src/connector/sink/runtime_config.rs @@ -14,12 +14,12 @@ use std::collections::HashMap; use datafusion::common::{DataFusionError, Result, plan_err}; -use crate::config::global_config::{ +use crate::common::connector_options::ConnectorOptions; +use crate::common::with_option_keys as opt; +use function_stream_config::global_config::{ DEFAULT_OPERATOR_STATE_STORE_MEMORY_BYTES, DEFAULT_SINK_BUFFER_MEMORY_BYTES, }; -use crate::config::streaming_job::DEFAULT_CHECKPOINT_INTERVAL_MS; -use crate::sql::common::connector_options::ConnectorOptions; -use crate::sql::common::with_option_keys as opt; +use function_stream_config::streaming_job::DEFAULT_CHECKPOINT_INTERVAL_MS; #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct SinkRuntimeConfig { diff --git a/src/sql/connector/sink/s3.rs b/src/streaming_planner/src/connector/sink/s3.rs similarity index 84% rename from src/sql/connector/sink/s3.rs rename to src/streaming_planner/src/connector/sink/s3.rs index 5d04ce46..51cf85bf 100644 --- a/src/sql/connector/sink/s3.rs +++ b/src/streaming_planner/src/connector/sink/s3.rs @@ -13,14 +13,14 @@ use datafusion::common::Result; use protocol::function_stream_graph::{S3SinkConfig, SinkFormatProto}; -use crate::sql::common::Format; -use crate::sql::common::connector_options::ConnectorOptions; -use crate::sql::common::constants::connector_type; -use crate::sql::common::with_option_keys as opt; -use crate::sql::connector::config::ConnectorConfig; -use crate::sql::connector::provider::SinkProvider; -use crate::sql::connector::sink::runtime_config::SinkRuntimeProperties; -use crate::sql::connector::sink::utils::SinkUtils; +use crate::common::Format; +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::connector_type; +use crate::common::with_option_keys as opt; +use crate::connector::config::ConnectorConfig; +use crate::connector::provider::SinkProvider; +use crate::connector::sink::runtime_config::SinkRuntimeProperties; +use crate::connector::sink::utils::SinkUtils; pub struct S3SinkConnector; diff --git a/src/sql/connector/sink/utils.rs b/src/streaming_planner/src/connector/sink/utils.rs similarity index 94% rename from src/sql/connector/sink/utils.rs rename to src/streaming_planner/src/connector/sink/utils.rs index e61cd870..481ad3b2 100644 --- a/src/sql/connector/sink/utils.rs +++ b/src/streaming_planner/src/connector/sink/utils.rs @@ -13,10 +13,10 @@ use datafusion::common::{DataFusionError, Result, plan_err}; use protocol::function_stream_graph::{ParquetCompressionProto, SinkFormatProto}; -use crate::sql::common::Format; -use crate::sql::common::connector_options::ConnectorOptions; -use crate::sql::common::constants::parquet_compression_value; -use crate::sql::common::with_option_keys as opt; +use crate::common::Format; +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::parquet_compression_value; +use crate::common::with_option_keys as opt; pub struct SinkUtils; diff --git a/src/sql/connector/source/kafka.rs b/src/streaming_planner/src/connector/source/kafka.rs similarity index 95% rename from src/sql/connector/source/kafka.rs rename to src/streaming_planner/src/connector/source/kafka.rs index 0bc220d9..71672279 100644 --- a/src/sql/connector/source/kafka.rs +++ b/src/streaming_planner/src/connector/source/kafka.rs @@ -17,15 +17,15 @@ use protocol::function_stream_graph::{ RawStringFormatConfig, TimestampFormatProto, format_config, kafka_auth_config, }; -use crate::sql::common::connector_options::ConnectorOptions; -use crate::sql::common::constants::{connector_type, kafka_with_value}; -use crate::sql::common::formats::{ +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::{connector_type, kafka_with_value}; +use crate::common::formats::{ BadData, DecimalEncoding as SqlDecimalEncoding, Format as SqlFormat, TimestampFormat as SqlTimestampFormat, }; -use crate::sql::common::with_option_keys as opt; -use crate::sql::connector::config::ConnectorConfig; -use crate::sql::connector::provider::SourceProvider; +use crate::common::with_option_keys as opt; +use crate::connector::config::ConnectorConfig; +use crate::connector::provider::SourceProvider; pub struct KafkaSourceConnector; diff --git a/src/runtime/streaming/operators/source/mod.rs b/src/streaming_planner/src/connector/source/mod.rs similarity index 100% rename from src/runtime/streaming/operators/source/mod.rs rename to src/streaming_planner/src/connector/source/mod.rs diff --git a/src/sql/functions/mod.rs b/src/streaming_planner/src/functions/mod.rs similarity index 98% rename from src/sql/functions/mod.rs rename to src/streaming_planner/src/functions/mod.rs index b78f5d2a..9ec1007f 100644 --- a/src/sql/functions/mod.rs +++ b/src/streaming_planner/src/functions/mod.rs @@ -10,7 +10,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sql::schema::StreamSchemaProvider; +use crate::schema::StreamSchemaProvider; use datafusion::arrow::array::{ Array, ArrayRef, StringArray, UnionArray, builder::{FixedSizeBinaryBuilder, ListBuilder, StringBuilder}, @@ -34,7 +34,7 @@ use std::collections::HashMap; use std::fmt::{Debug, Write}; use std::sync::{Arc, OnceLock}; -use crate::sql::common::constants::scalar_fn; +use crate::common::constants::scalar_fn; /// Borrowed from DataFusion /// @@ -336,14 +336,14 @@ pub fn extract_json_string(args: &[ColumnarValue]) -> Result { // https://github.com/datafusion-contrib/datafusion-functions-json/blob/main/src/common_union.rs // as the `is_json_union` function is not public. It should be kept in sync with that code so // that we are able to detect JSON unions and rewrite them to serialized JSON for sinks. -pub(crate) fn is_json_union(data_type: &DataType) -> bool { +pub fn is_json_union(data_type: &DataType) -> bool { match data_type { DataType::Union(fields, UnionMode::Sparse) => fields == &union_fields(), _ => false, } } -pub(crate) const TYPE_ID_NULL: i8 = 0; +pub const TYPE_ID_NULL: i8 = 0; const TYPE_ID_BOOL: i8 = 1; const TYPE_ID_INT: i8 = 2; const TYPE_ID_FLOAT: i8 = 3; @@ -456,7 +456,7 @@ fn write_value(b: &mut StringBuilder, id: i8, a: &ArrayRef) -> Result<(), std::f Ok(()) } -pub(crate) fn serialize_outgoing_json( +pub fn serialize_outgoing_json( registry: &StreamSchemaProvider, node: Arc, ) -> LogicalPlan { diff --git a/src/sql/mod.rs b/src/streaming_planner/src/lib.rs similarity index 68% rename from src/sql/mod.rs rename to src/streaming_planner/src/lib.rs index f4a0eef6..2dbc2b47 100644 --- a/src/sql/mod.rs +++ b/src/streaming_planner/src/lib.rs @@ -10,6 +10,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(dead_code)] // Planner keeps helpers/types for upcoming features; strict -D dead_code is noisy. + +//! Streaming SQL planning (logical graph, connectors, schema, physical codec). +//! +//! The `function-stream` binary/library re-exports this crate as `function_stream::sql` for +//! stable `crate::sql::…` paths in in-tree modules. + pub mod api; pub mod common; @@ -20,7 +27,7 @@ pub mod logical_node; pub mod logical_planner; pub mod parse; pub mod physical; -pub(crate) mod planning_runtime; +pub mod planning_runtime; pub mod schema; pub mod types; diff --git a/src/sql/logical_node/aggregate.rs b/src/streaming_planner/src/logical_node/aggregate.rs similarity index 97% rename from src/sql/logical_node/aggregate.rs rename to src/streaming_planner/src/logical_node/aggregate.rs index 1e288ab5..cef337c0 100644 --- a/src/sql/logical_node/aggregate.rs +++ b/src/streaming_planner/src/logical_node/aggregate.rs @@ -30,25 +30,25 @@ use protocol::function_stream_graph::{ SessionWindowAggregateOperator, SlidingWindowAggregateOperator, TumblingWindowAggregateOperator, }; -use crate::multifield_partial_ord; -use crate::sql::common::constants::{extension_node, proto_operator_name}; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; -use crate::sql::logical_node::{ +use crate::common::constants::{extension_node, proto_operator_name}; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; +use crate::logical_node::{ CompiledTopologyNode, StreamingOperatorBlueprint, SystemTimestampInjectorNode, }; -use crate::sql::logical_planner::planner::{NamedNode, Planner, SplitPlanOutput}; -use crate::sql::physical::{StreamingExtensionCodec, window}; -use crate::sql::types::{ +use crate::logical_planner::planner::{NamedNode, Planner, SplitPlanOutput}; +use crate::multifield_partial_ord; +use crate::physical::{StreamingExtensionCodec, window}; +use crate::types::{ QualifiedField, TIMESTAMP_FIELD, WindowBehavior, WindowType, build_df_schema, build_df_schema_with_metadata, extract_qualified_fields, }; -pub(crate) const STREAM_AGG_EXTENSION_NAME: &str = extension_node::STREAM_WINDOW_AGGREGATE; +pub const STREAM_AGG_EXTENSION_NAME: &str = extension_node::STREAM_WINDOW_AGGREGATE; /// Represents a streaming windowed aggregation node in the logical plan. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct StreamWindowAggregateNode { +pub struct StreamWindowAggregateNode { pub(crate) window_spec: WindowBehavior, pub(crate) base_agg_plan: LogicalPlan, pub(crate) output_schema: DFSchemaRef, diff --git a/src/sql/logical_node/async_udf.rs b/src/streaming_planner/src/logical_node/async_udf.rs similarity index 93% rename from src/sql/logical_node/async_udf.rs rename to src/streaming_planner/src/logical_node/async_udf.rs index 1c35398e..a3d1f68c 100644 --- a/src/sql/logical_node/async_udf.rs +++ b/src/streaming_planner/src/logical_node/async_udf.rs @@ -24,25 +24,25 @@ use datafusion_proto::physical_plan::to_proto::serialize_physical_expr; use prost::Message; use protocol::function_stream_graph::{AsyncUdfOperator, AsyncUdfOrdering}; -use crate::multifield_partial_ord; -use crate::sql::common::constants::extension_node; -use crate::sql::common::constants::sql_field; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::logical::{ +use crate::common::constants::extension_node; +use crate::common::constants::sql_field; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::logical::{ DylibUdfConfig, LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName, }; -use crate::sql::logical_node::streaming_operator_blueprint::{ +use crate::logical_node::streaming_operator_blueprint::{ CompiledTopologyNode, StreamingOperatorBlueprint, }; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::types::{QualifiedField, build_df_schema, extract_qualified_fields}; +use crate::logical_planner::planner::{NamedNode, Planner}; +use crate::multifield_partial_ord; +use crate::types::{QualifiedField, build_df_schema, extract_qualified_fields}; -pub(crate) const NODE_TYPE_NAME: &str = extension_node::ASYNC_FUNCTION_EXECUTION; +pub const NODE_TYPE_NAME: &str = extension_node::ASYNC_FUNCTION_EXECUTION; /// Represents a logical node that executes an external asynchronous function (UDF) /// and projects the final results into the streaming pipeline. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct AsyncFunctionExecutionNode { +pub struct AsyncFunctionExecutionNode { pub(crate) upstream_plan: Arc, pub(crate) operator_name: String, pub(crate) function_config: DylibUdfConfig, diff --git a/src/sql/logical_node/debezium.rs b/src/streaming_planner/src/logical_node/debezium.rs similarity index 95% rename from src/sql/logical_node/debezium.rs rename to src/streaming_planner/src/logical_node/debezium.rs index 8d69c6ec..2b13f090 100644 --- a/src/sql/logical_node/debezium.rs +++ b/src/streaming_planner/src/logical_node/debezium.rs @@ -19,12 +19,12 @@ use datafusion::common::{ use datafusion::logical_expr::{Expr, LogicalPlan, UserDefinedLogicalNodeCore}; use datafusion::physical_plan::DisplayAs; +use crate::common::constants::{cdc, extension_node}; +use crate::common::{FsSchema, FsSchemaRef, UPDATING_META_FIELD}; +use crate::logical_planner::planner::{NamedNode, Planner}; use crate::multifield_partial_ord; -use crate::sql::common::constants::{cdc, extension_node}; -use crate::sql::common::{FsSchema, FsSchemaRef, UPDATING_META_FIELD}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::physical::updating_meta_field; -use crate::sql::types::TIMESTAMP_FIELD; +use crate::physical::updating_meta_field; +use crate::types::TIMESTAMP_FIELD; use super::{CompiledTopologyNode, StreamingOperatorBlueprint}; @@ -32,19 +32,19 @@ use super::{CompiledTopologyNode, StreamingOperatorBlueprint}; // Constants & Identifiers // ----------------------------------------------------------------------------- -pub(crate) const UNROLL_NODE_NAME: &str = extension_node::UNROLL_DEBEZIUM_PAYLOAD; -pub(crate) const PACK_NODE_NAME: &str = extension_node::PACK_DEBEZIUM_ENVELOPE; +pub const UNROLL_NODE_NAME: &str = extension_node::UNROLL_DEBEZIUM_PAYLOAD; +pub const PACK_NODE_NAME: &str = extension_node::PACK_DEBEZIUM_ENVELOPE; // ----------------------------------------------------------------------------- // Core Schema Codec // ----------------------------------------------------------------------------- /// Transforms between flat schemas and Debezium CDC envelopes. -pub(crate) struct DebeziumSchemaCodec; +pub struct DebeziumSchemaCodec; impl DebeziumSchemaCodec { /// Wraps a flat physical schema into a Debezium CDC envelope structure. - pub(crate) fn wrap_into_envelope( + pub fn wrap_into_envelope( flat_schema: &DFSchemaRef, qualifier_override: Option, ) -> Result { @@ -299,7 +299,7 @@ impl StreamingOperatorBlueprint for UnrollDebeziumPayloadNode { /// Encodes a flat updating stream back into a Debezium CDC envelope representation. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct PackDebeziumEnvelopeNode { +pub struct PackDebeziumEnvelopeNode { upstream_plan: Arc, envelope_schema: DFSchemaRef, } @@ -307,7 +307,7 @@ pub(crate) struct PackDebeziumEnvelopeNode { multifield_partial_ord!(PackDebeziumEnvelopeNode, upstream_plan); impl PackDebeziumEnvelopeNode { - pub(crate) fn try_new(upstream_plan: LogicalPlan) -> Result { + pub fn try_new(upstream_plan: LogicalPlan) -> Result { let envelope_schema = DebeziumSchemaCodec::wrap_into_envelope(upstream_plan.schema(), None) .map_err(|e| { DataFusionError::Plan(format!("Failed to compile Debezium envelope schema: {e}")) diff --git a/src/sql/logical_node/extension_try_from.rs b/src/streaming_planner/src/logical_node/extension_try_from.rs similarity index 72% rename from src/sql/logical_node/extension_try_from.rs rename to src/streaming_planner/src/logical_node/extension_try_from.rs index 32b12d6c..c13532dc 100644 --- a/src/sql/logical_node/extension_try_from.rs +++ b/src/streaming_planner/src/logical_node/extension_try_from.rs @@ -15,20 +15,20 @@ use std::sync::Arc; use datafusion::common::{DataFusionError, Result}; use datafusion::logical_expr::UserDefinedLogicalNode; -use crate::sql::logical_node::aggregate::StreamWindowAggregateNode; -use crate::sql::logical_node::async_udf::AsyncFunctionExecutionNode; -use crate::sql::logical_node::debezium::{PackDebeziumEnvelopeNode, UnrollDebeziumPayloadNode}; -use crate::sql::logical_node::join::StreamingJoinNode; -use crate::sql::logical_node::key_calculation::KeyExtractionNode; -use crate::sql::logical_node::lookup::StreamReferenceJoinNode; -use crate::sql::logical_node::projection::StreamProjectionNode; -use crate::sql::logical_node::remote_table::RemoteTableBoundaryNode; -use crate::sql::logical_node::sink::StreamEgressNode; -use crate::sql::logical_node::streaming_operator_blueprint::StreamingOperatorBlueprint; -use crate::sql::logical_node::table_source::StreamIngestionNode; -use crate::sql::logical_node::updating_aggregate::ContinuousAggregateNode; -use crate::sql::logical_node::watermark_node::EventTimeWatermarkNode; -use crate::sql::logical_node::windows_function::StreamingWindowFunctionNode; +use crate::logical_node::aggregate::StreamWindowAggregateNode; +use crate::logical_node::async_udf::AsyncFunctionExecutionNode; +use crate::logical_node::debezium::{PackDebeziumEnvelopeNode, UnrollDebeziumPayloadNode}; +use crate::logical_node::join::StreamingJoinNode; +use crate::logical_node::key_calculation::KeyExtractionNode; +use crate::logical_node::lookup::StreamReferenceJoinNode; +use crate::logical_node::projection::StreamProjectionNode; +use crate::logical_node::remote_table::RemoteTableBoundaryNode; +use crate::logical_node::sink::StreamEgressNode; +use crate::logical_node::streaming_operator_blueprint::StreamingOperatorBlueprint; +use crate::logical_node::table_source::StreamIngestionNode; +use crate::logical_node::updating_aggregate::ContinuousAggregateNode; +use crate::logical_node::watermark_node::EventTimeWatermarkNode; +use crate::logical_node::windows_function::StreamingWindowFunctionNode; fn try_from_t( node: &dyn UserDefinedLogicalNode, diff --git a/src/sql/logical_node/is_retract.rs b/src/streaming_planner/src/logical_node/is_retract.rs similarity index 89% rename from src/sql/logical_node/is_retract.rs rename to src/streaming_planner/src/logical_node/is_retract.rs index 4370f6ae..17fad0db 100644 --- a/src/sql/logical_node/is_retract.rs +++ b/src/streaming_planner/src/logical_node/is_retract.rs @@ -17,13 +17,11 @@ use datafusion::common::{DFSchemaRef, Result, TableReference}; use datafusion::logical_expr::{Expr, LogicalPlan, UserDefinedLogicalNodeCore}; use crate::multifield_partial_ord; -use crate::sql::physical::updating_meta_field; -use crate::sql::types::{ - QualifiedField, TIMESTAMP_FIELD, build_df_schema, extract_qualified_fields, -}; +use crate::physical::updating_meta_field; +use crate::types::{QualifiedField, TIMESTAMP_FIELD, build_df_schema, extract_qualified_fields}; #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct IsRetractExtension { +pub struct IsRetractExtension { pub(crate) input: LogicalPlan, pub(crate) schema: DFSchemaRef, pub(crate) timestamp_qualifier: Option, @@ -32,7 +30,7 @@ pub(crate) struct IsRetractExtension { multifield_partial_ord!(IsRetractExtension, input, timestamp_qualifier); impl IsRetractExtension { - pub(crate) fn new(input: LogicalPlan, timestamp_qualifier: Option) -> Self { + pub fn new(input: LogicalPlan, timestamp_qualifier: Option) -> Self { let mut output_fields = extract_qualified_fields(input.schema()); let timestamp_index = output_fields.len() - 1; diff --git a/src/sql/logical_node/join.rs b/src/streaming_planner/src/logical_node/join.rs similarity index 93% rename from src/sql/logical_node/join.rs rename to src/streaming_planner/src/logical_node/join.rs index 15631f1f..35d645f1 100644 --- a/src/sql/logical_node/join.rs +++ b/src/streaming_planner/src/logical_node/join.rs @@ -22,18 +22,18 @@ use datafusion_proto::protobuf::PhysicalPlanNode; use prost::Message; use protocol::function_stream_graph::JoinOperator; -use crate::sql::common::constants::{extension_node, runtime_operator_kind}; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; -use crate::sql::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::physical::StreamingExtensionCodec; +use crate::common::constants::{extension_node, runtime_operator_kind}; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; +use crate::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; +use crate::logical_planner::planner::{NamedNode, Planner}; +use crate::physical::StreamingExtensionCodec; // ----------------------------------------------------------------------------- // Constants // ----------------------------------------------------------------------------- -pub(crate) const STREAM_JOIN_NODE_TYPE: &str = extension_node::STREAMING_JOIN; +pub const STREAM_JOIN_NODE_TYPE: &str = extension_node::STREAMING_JOIN; // ----------------------------------------------------------------------------- // Logical Node Definition diff --git a/src/sql/logical_node/key_calculation.rs b/src/streaming_planner/src/logical_node/key_calculation.rs similarity index 94% rename from src/sql/logical_node/key_calculation.rs rename to src/streaming_planner/src/logical_node/key_calculation.rs index ec83e108..8bd03b6e 100644 --- a/src/sql/logical_node/key_calculation.rs +++ b/src/streaming_planner/src/logical_node/key_calculation.rs @@ -26,16 +26,16 @@ use prost::Message; use protocol::function_stream_graph::{KeyPlanOperator, ProjectionOperator}; +use crate::common::constants::{extension_node, sql_field}; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; +use crate::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; +use crate::logical_planner::planner::{NamedNode, Planner}; use crate::multifield_partial_ord; -use crate::sql::common::constants::{extension_node, sql_field}; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; -use crate::sql::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::physical::StreamingExtensionCodec; -use crate::sql::types::{build_df_schema_with_metadata, extract_qualified_fields}; +use crate::physical::StreamingExtensionCodec; +use crate::types::{build_df_schema_with_metadata, extract_qualified_fields}; -pub(crate) const EXTENSION_NODE_IDENTIFIER: &str = extension_node::KEY_EXTRACTION; +pub const EXTENSION_NODE_IDENTIFIER: &str = extension_node::KEY_EXTRACTION; /// Routing strategy for shuffling data across the stream topology. #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd)] @@ -46,7 +46,7 @@ pub enum KeyExtractionStrategy { /// Logical node that computes or extracts routing keys. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct KeyExtractionNode { +pub struct KeyExtractionNode { pub(crate) operator_label: Option, pub(crate) upstream_plan: LogicalPlan, pub(crate) extraction_strategy: KeyExtractionStrategy, diff --git a/src/sql/logical_node/logical/dylib_udf_config.rs b/src/streaming_planner/src/logical_node/logical/dylib_udf_config.rs similarity index 100% rename from src/sql/logical_node/logical/dylib_udf_config.rs rename to src/streaming_planner/src/logical_node/logical/dylib_udf_config.rs diff --git a/src/sql/logical_node/logical/fs_program_convert.rs b/src/streaming_planner/src/logical_node/logical/fs_program_convert.rs similarity index 98% rename from src/sql/logical_node/logical/fs_program_convert.rs rename to src/streaming_planner/src/logical_node/logical/fs_program_convert.rs index b05d68f5..53316bae 100644 --- a/src/sql/logical_node/logical/fs_program_convert.rs +++ b/src/streaming_planner/src/logical_node/logical/fs_program_convert.rs @@ -23,8 +23,8 @@ use protocol::function_stream_graph::{ FsSchema as ProtoFsSchema, }; -use crate::sql::api::pipelines::{PipelineEdge, PipelineGraph, PipelineNode}; -use crate::sql::common::FsSchema; +use crate::api::pipelines::{PipelineEdge, PipelineGraph, PipelineNode}; +use crate::common::FsSchema; use super::logical_edge::logical_edge_type_from_proto_i32; use super::operator_chain::{ChainedLogicalOperator, OperatorChain}; diff --git a/src/sql/logical_node/logical/logical_edge.rs b/src/streaming_planner/src/logical_node/logical/logical_edge.rs similarity index 96% rename from src/sql/logical_node/logical/logical_edge.rs rename to src/streaming_planner/src/logical_node/logical/logical_edge.rs index 87950e70..d599f7f1 100644 --- a/src/sql/logical_node/logical/logical_edge.rs +++ b/src/streaming_planner/src/logical_node/logical/logical_edge.rs @@ -17,7 +17,7 @@ use datafusion::common::{DataFusionError, Result}; use protocol::function_stream_graph::EdgeType as ProtoEdgeType; use serde::{Deserialize, Serialize}; -use crate::sql::common::FsSchema; +use crate::common::FsSchema; #[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, Serialize, Deserialize)] pub enum LogicalEdgeType { @@ -64,7 +64,7 @@ impl From for ProtoEdgeType { } } -pub(crate) fn logical_edge_type_from_proto_i32(i: i32) -> Result { +pub fn logical_edge_type_from_proto_i32(i: i32) -> Result { let e = ProtoEdgeType::try_from(i).map_err(|_| { DataFusionError::Plan(format!("invalid protobuf EdgeType discriminant {i}")) })?; diff --git a/src/sql/logical_node/logical/logical_graph.rs b/src/streaming_planner/src/logical_node/logical/logical_graph.rs similarity index 100% rename from src/sql/logical_node/logical/logical_graph.rs rename to src/streaming_planner/src/logical_node/logical/logical_graph.rs diff --git a/src/sql/logical_node/logical/logical_node.rs b/src/streaming_planner/src/logical_node/logical/logical_node.rs similarity index 100% rename from src/sql/logical_node/logical/logical_node.rs rename to src/streaming_planner/src/logical_node/logical/logical_node.rs diff --git a/src/sql/logical_node/logical/logical_program.rs b/src/streaming_planner/src/logical_node/logical/logical_program.rs similarity index 100% rename from src/sql/logical_node/logical/logical_program.rs rename to src/streaming_planner/src/logical_node/logical/logical_program.rs diff --git a/src/sql/logical_node/logical/mod.rs b/src/streaming_planner/src/logical_node/logical/mod.rs similarity index 100% rename from src/sql/logical_node/logical/mod.rs rename to src/streaming_planner/src/logical_node/logical/mod.rs diff --git a/src/sql/logical_node/logical/operator_chain.rs b/src/streaming_planner/src/logical_node/logical/operator_chain.rs similarity index 99% rename from src/sql/logical_node/logical/operator_chain.rs rename to src/streaming_planner/src/logical_node/logical/operator_chain.rs index 2aecddd6..34be2f57 100644 --- a/src/sql/logical_node/logical/operator_chain.rs +++ b/src/streaming_planner/src/logical_node/logical/operator_chain.rs @@ -18,7 +18,7 @@ use protocol::function_stream_graph::ConnectorOp; use serde::{Deserialize, Serialize}; use super::operator_name::OperatorName; -use crate::sql::common::FsSchema; +use crate::common::FsSchema; #[derive(Clone, Debug, Serialize, Deserialize)] pub struct ChainedLogicalOperator { diff --git a/src/sql/logical_node/logical/operator_name.rs b/src/streaming_planner/src/logical_node/logical/operator_name.rs similarity index 98% rename from src/sql/logical_node/logical/operator_name.rs rename to src/streaming_planner/src/logical_node/logical/operator_name.rs index 57f53f90..4377b0bd 100644 --- a/src/sql/logical_node/logical/operator_name.rs +++ b/src/streaming_planner/src/logical_node/logical/operator_name.rs @@ -15,7 +15,7 @@ use std::str::FromStr; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use strum::{Display, EnumString, IntoStaticStr}; -use crate::sql::common::constants::operator_feature; +use crate::common::constants::operator_feature; #[derive(Clone, Copy, Debug, Eq, PartialEq, EnumString, Display, IntoStaticStr)] pub enum OperatorName { diff --git a/src/sql/logical_node/logical/program_config.rs b/src/streaming_planner/src/logical_node/logical/program_config.rs similarity index 100% rename from src/sql/logical_node/logical/program_config.rs rename to src/streaming_planner/src/logical_node/logical/program_config.rs diff --git a/src/sql/logical_node/logical/python_udf_config.rs b/src/streaming_planner/src/logical_node/logical/python_udf_config.rs similarity index 100% rename from src/sql/logical_node/logical/python_udf_config.rs rename to src/streaming_planner/src/logical_node/logical/python_udf_config.rs diff --git a/src/sql/logical_node/lookup.rs b/src/streaming_planner/src/logical_node/lookup.rs similarity index 95% rename from src/sql/logical_node/lookup.rs rename to src/streaming_planner/src/logical_node/lookup.rs index d2817c85..ca95c4a6 100644 --- a/src/sql/logical_node/lookup.rs +++ b/src/streaming_planner/src/logical_node/lookup.rs @@ -23,14 +23,14 @@ use prost::Message; use protocol::function_stream_graph; use protocol::function_stream_graph::{ConnectorOp, LookupJoinCondition, LookupJoinOperator}; +use crate::common::constants::extension_node; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; +use crate::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; +use crate::logical_planner::planner::{NamedNode, Planner}; use crate::multifield_partial_ord; -use crate::sql::common::constants::extension_node; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; -use crate::sql::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::schema::LookupTable; -use crate::sql::schema::utils::add_timestamp_field_arrow; +use crate::schema::LookupTable; +use crate::schema::utils::add_timestamp_field_arrow; pub const DICTIONARY_SOURCE_NODE_NAME: &str = extension_node::REFERENCE_TABLE_SOURCE; pub const STREAM_DICTIONARY_JOIN_NODE_NAME: &str = extension_node::STREAM_REFERENCE_JOIN; diff --git a/src/sql/logical_node/macros.rs b/src/streaming_planner/src/logical_node/macros.rs similarity index 100% rename from src/sql/logical_node/macros.rs rename to src/streaming_planner/src/logical_node/macros.rs diff --git a/src/streaming_planner/src/logical_node/mod.rs b/src/streaming_planner/src/logical_node/mod.rs new file mode 100644 index 00000000..3ef02fc8 --- /dev/null +++ b/src/streaming_planner/src/logical_node/mod.rs @@ -0,0 +1,42 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod logical; + +mod macros; + +pub mod streaming_operator_blueprint; +pub use streaming_operator_blueprint::{CompiledTopologyNode, StreamingOperatorBlueprint}; + +pub mod aggregate; +pub mod debezium; +pub mod join; +pub mod key_calculation; +pub mod lookup; +pub mod projection; +pub mod remote_table; +pub mod sink; +pub mod table_source; +pub mod updating_aggregate; +pub mod watermark_node; +pub mod windows_function; + +pub mod timestamp_append; +pub use timestamp_append::SystemTimestampInjectorNode; + +pub mod async_udf; +pub use async_udf::AsyncFunctionExecutionNode; + +pub mod is_retract; +pub use is_retract::IsRetractExtension; + +mod extension_try_from; diff --git a/src/sql/logical_node/projection.rs b/src/streaming_planner/src/logical_node/projection.rs similarity index 92% rename from src/sql/logical_node/projection.rs rename to src/streaming_planner/src/logical_node/projection.rs index 3c5cfccb..f56b109f 100644 --- a/src/sql/logical_node/projection.rs +++ b/src/streaming_planner/src/logical_node/projection.rs @@ -21,19 +21,19 @@ use prost::Message; use protocol::function_stream_graph::ProjectionOperator; +use crate::common::constants::{extension_node, sql_field}; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; +use crate::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; +use crate::logical_planner::planner::{NamedNode, Planner}; use crate::multifield_partial_ord; -use crate::sql::common::constants::{extension_node, sql_field}; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; -use crate::sql::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::types::{QualifiedField, build_df_schema}; +use crate::types::{QualifiedField, build_df_schema}; // ----------------------------------------------------------------------------- // Constants & Identifiers // ----------------------------------------------------------------------------- -pub(crate) const STREAM_PROJECTION_NODE_NAME: &str = extension_node::STREAM_PROJECTION; +pub const STREAM_PROJECTION_NODE_NAME: &str = extension_node::STREAM_PROJECTION; const DEFAULT_PROJECTION_LABEL: &str = sql_field::DEFAULT_PROJECTION_LABEL; // ----------------------------------------------------------------------------- @@ -42,7 +42,7 @@ const DEFAULT_PROJECTION_LABEL: &str = sql_field::DEFAULT_PROJECTION_LABEL; /// Projection within a streaming execution topology. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct StreamProjectionNode { +pub struct StreamProjectionNode { pub(crate) upstream_plans: Vec, pub(crate) operator_label: Option, pub(crate) projection_exprs: Vec, @@ -53,7 +53,7 @@ pub(crate) struct StreamProjectionNode { multifield_partial_ord!(StreamProjectionNode, operator_label, projection_exprs); impl StreamProjectionNode { - pub(crate) fn try_new( + pub fn try_new( upstream_plans: Vec, operator_label: Option, projection_exprs: Vec, @@ -81,7 +81,7 @@ impl StreamProjectionNode { }) } - pub(crate) fn with_shuffle_routing(mut self) -> Self { + pub fn with_shuffle_routing(mut self) -> Self { self.requires_shuffle = true; self } diff --git a/src/sql/logical_node/remote_table.rs b/src/streaming_planner/src/logical_node/remote_table.rs similarity index 92% rename from src/sql/logical_node/remote_table.rs rename to src/streaming_planner/src/logical_node/remote_table.rs index bde1d47f..302228e7 100644 --- a/src/sql/logical_node/remote_table.rs +++ b/src/streaming_planner/src/logical_node/remote_table.rs @@ -21,19 +21,19 @@ use prost::Message; use protocol::function_stream_graph::ValuePlanOperator; +use crate::common::constants::extension_node; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; +use crate::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; +use crate::logical_planner::planner::{NamedNode, Planner}; use crate::multifield_partial_ord; -use crate::sql::common::constants::extension_node; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; -use crate::sql::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::physical::StreamingExtensionCodec; +use crate::physical::StreamingExtensionCodec; // ----------------------------------------------------------------------------- // Constants & Identifiers // ----------------------------------------------------------------------------- -pub(crate) const REMOTE_TABLE_NODE_NAME: &str = extension_node::REMOTE_TABLE_BOUNDARY; +pub const REMOTE_TABLE_NODE_NAME: &str = extension_node::REMOTE_TABLE_BOUNDARY; // ----------------------------------------------------------------------------- // Logical Node Definition @@ -41,7 +41,7 @@ pub(crate) const REMOTE_TABLE_NODE_NAME: &str = extension_node::REMOTE_TABLE_BOU /// Segments the execution graph and merges nodes sharing the same identifier; acts as a boundary. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct RemoteTableBoundaryNode { +pub struct RemoteTableBoundaryNode { pub(crate) upstream_plan: LogicalPlan, pub(crate) table_identifier: TableReference, pub(crate) resolved_schema: DFSchemaRef, diff --git a/src/sql/logical_node/sink.rs b/src/streaming_planner/src/logical_node/sink.rs similarity index 94% rename from src/sql/logical_node/sink.rs rename to src/streaming_planner/src/logical_node/sink.rs index d767afe3..f0b66187 100644 --- a/src/sql/logical_node/sink.rs +++ b/src/streaming_planner/src/logical_node/sink.rs @@ -17,14 +17,14 @@ use datafusion::common::{DFSchemaRef, Result, TableReference, plan_err}; use datafusion::logical_expr::{Expr, Extension, LogicalPlan, UserDefinedLogicalNodeCore}; use prost::Message; +use crate::common::constants::extension_node; +use crate::common::{FsSchema, FsSchemaRef, UPDATING_META_FIELD}; +use crate::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; +use crate::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; +use crate::logical_planner::planner::{NamedNode, Planner}; use crate::multifield_partial_ord; -use crate::sql::common::constants::extension_node; -use crate::sql::common::{FsSchema, FsSchemaRef, UPDATING_META_FIELD}; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; -use crate::sql::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::schema::CatalogEntity; -use crate::sql::schema::catalog::ExternalTable; +use crate::schema::CatalogEntity; +use crate::schema::catalog::ExternalTable; use super::debezium::PackDebeziumEnvelopeNode; use super::remote_table::RemoteTableBoundaryNode; @@ -33,7 +33,7 @@ use super::remote_table::RemoteTableBoundaryNode; // Constants & Identifiers // ----------------------------------------------------------------------------- -pub(crate) const STREAM_EGRESS_NODE_NAME: &str = extension_node::STREAM_EGRESS; +pub const STREAM_EGRESS_NODE_NAME: &str = extension_node::STREAM_EGRESS; // ----------------------------------------------------------------------------- // Logical Node Definition @@ -41,7 +41,7 @@ pub(crate) const STREAM_EGRESS_NODE_NAME: &str = extension_node::STREAM_EGRESS; /// Terminal node routing processed data into an external sink (e.g. Kafka, PostgreSQL). #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct StreamEgressNode { +pub struct StreamEgressNode { pub(crate) target_identifier: TableReference, pub(crate) destination_table: CatalogEntity, pub(crate) egress_schema: DFSchemaRef, diff --git a/src/sql/logical_node/streaming_operator_blueprint.rs b/src/streaming_planner/src/logical_node/streaming_operator_blueprint.rs similarity index 88% rename from src/sql/logical_node/streaming_operator_blueprint.rs rename to src/streaming_planner/src/logical_node/streaming_operator_blueprint.rs index d3f9d459..d9afaeab 100644 --- a/src/sql/logical_node/streaming_operator_blueprint.rs +++ b/src/streaming_planner/src/logical_node/streaming_operator_blueprint.rs @@ -14,16 +14,16 @@ use std::fmt::Debug; use datafusion::common::Result; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalNode}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::logical::{LogicalEdge, LogicalNode}; +use crate::logical_planner::planner::{NamedNode, Planner}; // ----------------------------------------------------------------------------- // Core Execution Blueprint // ----------------------------------------------------------------------------- /// Atomic unit within a streaming execution topology: translates streaming SQL into graph nodes. -pub(crate) trait StreamingOperatorBlueprint: Debug { +pub trait StreamingOperatorBlueprint: Debug { /// Canonical named identity for this operator, if any (sources, sinks, etc.). fn operator_identity(&self) -> Option; @@ -50,7 +50,7 @@ pub(crate) trait StreamingOperatorBlueprint: Debug { /// Compiled vertex: execution unit plus upstream routing edges. #[derive(Debug, Clone)] -pub(crate) struct CompiledTopologyNode { +pub struct CompiledTopologyNode { pub execution_unit: LogicalNode, pub routing_edges: Vec, } diff --git a/src/sql/logical_node/table_source.rs b/src/streaming_planner/src/logical_node/table_source.rs similarity index 91% rename from src/sql/logical_node/table_source.rs rename to src/streaming_planner/src/logical_node/table_source.rs index b1c6bfdd..dad8f921 100644 --- a/src/sql/logical_node/table_source.rs +++ b/src/streaming_planner/src/logical_node/table_source.rs @@ -17,15 +17,15 @@ use datafusion::common::{DFSchemaRef, Result, TableReference, plan_err}; use datafusion::logical_expr::{Expr, LogicalPlan, UserDefinedLogicalNodeCore}; use prost::Message; +use crate::common::constants::extension_node; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::debezium::DebeziumSchemaCodec; +use crate::logical_node::logical::{LogicalNode, OperatorName}; +use crate::logical_planner::planner::{NamedNode, Planner}; use crate::multifield_partial_ord; -use crate::sql::common::constants::extension_node; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::debezium::DebeziumSchemaCodec; -use crate::sql::logical_node::logical::{LogicalNode, OperatorName}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::schema::SourceTable; -use crate::sql::schema::utils::add_timestamp_field; -use crate::sql::types::build_df_schema; +use crate::schema::SourceTable; +use crate::schema::utils::add_timestamp_field; +use crate::types::build_df_schema; use super::{CompiledTopologyNode, StreamingOperatorBlueprint}; @@ -33,7 +33,7 @@ use super::{CompiledTopologyNode, StreamingOperatorBlueprint}; // Constants & Identifiers // ----------------------------------------------------------------------------- -pub(crate) const STREAM_INGESTION_NODE_NAME: &str = extension_node::STREAM_INGESTION; +pub const STREAM_INGESTION_NODE_NAME: &str = extension_node::STREAM_INGESTION; // ----------------------------------------------------------------------------- // Logical Node Definition @@ -41,7 +41,7 @@ pub(crate) const STREAM_INGESTION_NODE_NAME: &str = extension_node::STREAM_INGES /// Foundational ingestion point: connects to external systems and injects raw or CDC data. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct StreamIngestionNode { +pub struct StreamIngestionNode { pub(crate) source_identifier: TableReference, pub(crate) source_definition: SourceTable, pub(crate) resolved_schema: DFSchemaRef, diff --git a/src/sql/logical_node/timestamp_append.rs b/src/streaming_planner/src/logical_node/timestamp_append.rs similarity index 92% rename from src/sql/logical_node/timestamp_append.rs rename to src/streaming_planner/src/logical_node/timestamp_append.rs index 630e5a66..a490243a 100644 --- a/src/sql/logical_node/timestamp_append.rs +++ b/src/streaming_planner/src/logical_node/timestamp_append.rs @@ -15,15 +15,15 @@ use std::fmt::Formatter; use datafusion::common::{DFSchemaRef, Result, TableReference, internal_err}; use datafusion::logical_expr::{Expr, LogicalPlan, UserDefinedLogicalNodeCore}; +use crate::common::constants::extension_node; use crate::multifield_partial_ord; -use crate::sql::common::constants::extension_node; -use crate::sql::schema::utils::{add_timestamp_field, has_timestamp_field}; +use crate::schema::utils::{add_timestamp_field, has_timestamp_field}; // ----------------------------------------------------------------------------- // Constants & Identifiers // ----------------------------------------------------------------------------- -pub(crate) const TIMESTAMP_INJECTOR_NODE_NAME: &str = extension_node::SYSTEM_TIMESTAMP_INJECTOR; +pub const TIMESTAMP_INJECTOR_NODE_NAME: &str = extension_node::SYSTEM_TIMESTAMP_INJECTOR; // ----------------------------------------------------------------------------- // Logical Node Definition @@ -31,7 +31,7 @@ pub(crate) const TIMESTAMP_INJECTOR_NODE_NAME: &str = extension_node::SYSTEM_TIM /// Injects the mandatory system `_timestamp` field into the upstream streaming schema. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct SystemTimestampInjectorNode { +pub struct SystemTimestampInjectorNode { pub(crate) upstream_plan: LogicalPlan, pub(crate) target_qualifier: Option, pub(crate) resolved_schema: DFSchemaRef, @@ -40,7 +40,7 @@ pub(crate) struct SystemTimestampInjectorNode { multifield_partial_ord!(SystemTimestampInjectorNode, upstream_plan, target_qualifier); impl SystemTimestampInjectorNode { - pub(crate) fn try_new( + pub fn try_new( upstream_plan: LogicalPlan, target_qualifier: Option, ) -> Result { diff --git a/src/sql/logical_node/updating_aggregate.rs b/src/streaming_planner/src/logical_node/updating_aggregate.rs similarity index 92% rename from src/sql/logical_node/updating_aggregate.rs rename to src/streaming_planner/src/logical_node/updating_aggregate.rs index 0ddb2b28..14f52ea7 100644 --- a/src/sql/logical_node/updating_aggregate.rs +++ b/src/streaming_planner/src/logical_node/updating_aggregate.rs @@ -25,21 +25,19 @@ use datafusion_proto::protobuf::PhysicalPlanNode; use prost::Message; use protocol::function_stream_graph::UpdatingAggregateOperator; -use crate::sql::common::constants::{extension_node, proto_operator_name, updating_state_field}; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::functions::multi_hash; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; -use crate::sql::logical_node::{ - CompiledTopologyNode, IsRetractExtension, StreamingOperatorBlueprint, -}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::physical::StreamingExtensionCodec; +use crate::common::constants::{extension_node, proto_operator_name, updating_state_field}; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::functions::multi_hash; +use crate::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; +use crate::logical_node::{CompiledTopologyNode, IsRetractExtension, StreamingOperatorBlueprint}; +use crate::logical_planner::planner::{NamedNode, Planner}; +use crate::physical::StreamingExtensionCodec; // ----------------------------------------------------------------------------- // Constants & Configuration // ----------------------------------------------------------------------------- -pub(crate) const CONTINUOUS_AGGREGATE_NODE_NAME: &str = extension_node::CONTINUOUS_AGGREGATE; +pub const CONTINUOUS_AGGREGATE_NODE_NAME: &str = extension_node::CONTINUOUS_AGGREGATE; const DEFAULT_FLUSH_INTERVAL_MICROS: u64 = 10_000_000; @@ -51,7 +49,7 @@ const STATIC_HASH_SIZE_BYTES: i32 = 16; /// Stateful continuous aggregation: running aggregates with updating / retraction semantics. #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd)] -pub(crate) struct ContinuousAggregateNode { +pub struct ContinuousAggregateNode { pub(crate) base_aggregate_plan: LogicalPlan, pub(crate) partition_key_indices: Vec, pub(crate) retract_injected_plan: LogicalPlan, diff --git a/src/sql/logical_node/watermark_node.rs b/src/streaming_planner/src/logical_node/watermark_node.rs similarity index 91% rename from src/sql/logical_node/watermark_node.rs rename to src/streaming_planner/src/logical_node/watermark_node.rs index 9a8fc9d6..8d51fc47 100644 --- a/src/sql/logical_node/watermark_node.rs +++ b/src/streaming_planner/src/logical_node/watermark_node.rs @@ -21,20 +21,20 @@ use datafusion_proto::physical_plan::to_proto::serialize_physical_expr; use prost::Message; use protocol::function_stream_graph::ExpressionWatermarkConfig; +use crate::common::constants::{extension_node, runtime_operator_kind}; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; +use crate::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; +use crate::logical_planner::planner::{NamedNode, Planner}; use crate::multifield_partial_ord; -use crate::sql::common::constants::{extension_node, runtime_operator_kind}; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; -use crate::sql::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::schema::utils::add_timestamp_field; -use crate::sql::types::TIMESTAMP_FIELD; +use crate::schema::utils::add_timestamp_field; +use crate::types::TIMESTAMP_FIELD; // ----------------------------------------------------------------------------- // Constants & Identifiers // ----------------------------------------------------------------------------- -pub(crate) const EVENT_TIME_WATERMARK_NODE_NAME: &str = extension_node::EVENT_TIME_WATERMARK; +pub const EVENT_TIME_WATERMARK_NODE_NAME: &str = extension_node::EVENT_TIME_WATERMARK; const DEFAULT_WATERMARK_EMISSION_PERIOD_MICROS: u64 = 1_000_000; @@ -44,7 +44,7 @@ const DEFAULT_WATERMARK_EMISSION_PERIOD_MICROS: u64 = 1_000_000; /// Event-time watermark from a user strategy; drives time progress in stateful operators. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct EventTimeWatermarkNode { +pub struct EventTimeWatermarkNode { pub(crate) upstream_plan: LogicalPlan, pub(crate) namespace_qualifier: TableReference, pub(crate) watermark_strategy_expr: Expr, @@ -61,7 +61,7 @@ multifield_partial_ord!( ); impl EventTimeWatermarkNode { - pub(crate) fn try_new( + pub fn try_new( upstream_plan: LogicalPlan, namespace_qualifier: TableReference, watermark_strategy_expr: Expr, @@ -89,7 +89,7 @@ impl EventTimeWatermarkNode { }) } - pub(crate) fn generate_fs_schema(&self) -> FsSchema { + pub fn generate_fs_schema(&self) -> FsSchema { FsSchema::new_unkeyed( Arc::new(self.resolved_schema.as_ref().into()), self.internal_timestamp_offset, diff --git a/src/sql/logical_node/windows_function.rs b/src/streaming_planner/src/logical_node/windows_function.rs similarity index 92% rename from src/sql/logical_node/windows_function.rs rename to src/streaming_planner/src/logical_node/windows_function.rs index 9be37382..5fdfb9ce 100644 --- a/src/sql/logical_node/windows_function.rs +++ b/src/streaming_planner/src/logical_node/windows_function.rs @@ -21,12 +21,12 @@ use datafusion_proto::{physical_plan::AsExecutionPlan, protobuf::PhysicalPlanNod use prost::Message; use protocol::function_stream_graph::WindowFunctionOperator; -use crate::sql::common::constants::{extension_node, proto_operator_name, runtime_operator_kind}; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::physical::StreamingExtensionCodec; -use crate::sql::types::TIMESTAMP_FIELD; +use crate::common::constants::{extension_node, proto_operator_name, runtime_operator_kind}; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; +use crate::logical_planner::planner::{NamedNode, Planner}; +use crate::physical::StreamingExtensionCodec; +use crate::types::TIMESTAMP_FIELD; use super::{CompiledTopologyNode, StreamingOperatorBlueprint}; @@ -34,7 +34,7 @@ use super::{CompiledTopologyNode, StreamingOperatorBlueprint}; // Constants & Identifiers // ----------------------------------------------------------------------------- -pub(crate) const STREAMING_WINDOW_NODE_NAME: &str = extension_node::STREAMING_WINDOW_FUNCTION; +pub const STREAMING_WINDOW_NODE_NAME: &str = extension_node::STREAMING_WINDOW_FUNCTION; // ----------------------------------------------------------------------------- // Logical Node Definition @@ -42,7 +42,7 @@ pub(crate) const STREAMING_WINDOW_NODE_NAME: &str = extension_node::STREAMING_WI /// Stateful streaming window: temporal binning plus underlying window evaluation plan. #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd)] -pub(crate) struct StreamingWindowFunctionNode { +pub struct StreamingWindowFunctionNode { pub(crate) underlying_evaluation_plan: LogicalPlan, pub(crate) partition_key_indices: Vec, } diff --git a/src/sql/logical_planner/mod.rs b/src/streaming_planner/src/logical_planner/mod.rs similarity index 87% rename from src/sql/logical_planner/mod.rs rename to src/streaming_planner/src/logical_planner/mod.rs index 9ecfb676..d04ccce3 100644 --- a/src/sql/logical_planner/mod.rs +++ b/src/streaming_planner/src/logical_planner/mod.rs @@ -12,5 +12,5 @@ pub mod optimizers; -pub(crate) mod streaming_planner; -pub(crate) use streaming_planner as planner; +pub mod streaming_planner; +pub use streaming_planner as planner; diff --git a/src/sql/logical_planner/optimizers/chaining.rs b/src/streaming_planner/src/logical_planner/optimizers/chaining.rs similarity index 97% rename from src/sql/logical_planner/optimizers/chaining.rs rename to src/streaming_planner/src/logical_planner/optimizers/chaining.rs index ea7bd885..60b36115 100644 --- a/src/sql/logical_planner/optimizers/chaining.rs +++ b/src/streaming_planner/src/logical_planner/optimizers/chaining.rs @@ -14,7 +14,7 @@ use petgraph::prelude::*; use petgraph::visit::NodeIndexable; use tracing::debug; -use crate::sql::logical_node::logical::{LogicalEdgeType, LogicalGraph, Optimizer}; +use crate::logical_node::logical::{LogicalEdgeType, LogicalGraph, Optimizer}; pub struct ChainingOptimizer {} @@ -119,8 +119,8 @@ mod tests { use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; - use crate::sql::common::FsSchema; - use crate::sql::logical_node::logical::{ + use crate::common::FsSchema; + use crate::logical_node::logical::{ LogicalEdge, LogicalEdgeType, LogicalGraph, LogicalNode, OperatorName, Optimizer, }; diff --git a/src/sql/logical_planner/optimizers/mod.rs b/src/streaming_planner/src/logical_planner/optimizers/mod.rs similarity index 100% rename from src/sql/logical_planner/optimizers/mod.rs rename to src/streaming_planner/src/logical_planner/optimizers/mod.rs diff --git a/src/sql/logical_planner/optimizers/optimized_plan.rs b/src/streaming_planner/src/logical_planner/optimizers/optimized_plan.rs similarity index 98% rename from src/sql/logical_planner/optimizers/optimized_plan.rs rename to src/streaming_planner/src/logical_planner/optimizers/optimized_plan.rs index fbb64845..df380fe1 100644 --- a/src/sql/logical_planner/optimizers/optimized_plan.rs +++ b/src/streaming_planner/src/logical_planner/optimizers/optimized_plan.rs @@ -41,7 +41,7 @@ use datafusion::optimizer::simplify_expressions::SimplifyExpressions; use datafusion::sql::planner::SqlToRel; use datafusion::sql::sqlparser::ast::Statement; -use crate::sql::schema::StreamSchemaProvider; +use crate::schema::StreamSchemaProvider; /// Converts a SQL statement into an optimized DataFusion logical plan. /// diff --git a/src/sql/logical_planner/streaming_planner.rs b/src/streaming_planner/src/logical_planner/streaming_planner.rs similarity index 90% rename from src/sql/logical_planner/streaming_planner.rs rename to src/streaming_planner/src/logical_planner/streaming_planner.rs index 1e999c2a..a5f3d5f6 100644 --- a/src/sql/logical_planner/streaming_planner.rs +++ b/src/streaming_planner/src/logical_planner/streaming_planner.rs @@ -42,35 +42,33 @@ use datafusion_common::TableReference; use datafusion_proto::physical_plan::DefaultPhysicalExtensionCodec; use datafusion_proto::physical_plan::to_proto::serialize_physical_expr; -use crate::sql::common::constants::sql_planning_default; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::debezium::{ - PACK_NODE_NAME, UNROLL_NODE_NAME, UnrollDebeziumPayloadNode, -}; -use crate::sql::logical_node::key_calculation::KeyExtractionNode; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalGraph, LogicalNode}; -use crate::sql::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; -use crate::sql::physical::{ +use crate::common::constants::sql_planning_default; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::debezium::{PACK_NODE_NAME, UNROLL_NODE_NAME, UnrollDebeziumPayloadNode}; +use crate::logical_node::key_calculation::KeyExtractionNode; +use crate::logical_node::logical::{LogicalEdge, LogicalGraph, LogicalNode}; +use crate::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; +use crate::physical::{ CdcDebeziumPackExec, CdcDebeziumUnrollExec, FsMemExec, StreamingDecodingContext, StreamingExtensionCodec, }; -use crate::sql::schema::StreamSchemaProvider; -use crate::sql::schema::utils::add_timestamp_field_arrow; +use crate::schema::StreamSchemaProvider; +use crate::schema::utils::add_timestamp_field_arrow; -pub(crate) struct SplitPlanOutput { +pub struct SplitPlanOutput { pub(crate) partial_aggregation_plan: PhysicalPlanNode, pub(crate) partial_schema: FsSchema, pub(crate) finish_plan: PhysicalPlanNode, } #[derive(Eq, Hash, PartialEq, Debug)] -pub(crate) enum NamedNode { +pub enum NamedNode { Source(TableReference), Watermark(TableReference), RemoteTable(TableReference), Sink(TableReference), } -pub(crate) struct PlanToGraphVisitor<'a> { +pub struct PlanToGraphVisitor<'a> { graph: DiGraph, output_schemas: HashMap, named_nodes: HashMap, @@ -90,7 +88,7 @@ impl<'a> PlanToGraphVisitor<'a> { } } -pub(crate) struct Planner<'a> { +pub struct Planner<'a> { schema_provider: &'a StreamSchemaProvider, planner: DefaultPhysicalPlanner, session_state: &'a SessionState, @@ -98,25 +96,22 @@ pub(crate) struct Planner<'a> { impl<'a> Planner<'a> { #[inline] - pub(crate) fn default_parallelism(&self) -> usize { + pub fn default_parallelism(&self) -> usize { self.schema_provider.default_parallelism() } #[inline] - pub(crate) fn key_by_parallelism(&self) -> usize { + pub fn key_by_parallelism(&self) -> usize { self.schema_provider.key_by_parallelism() } /// Parallelism for operators that consume a keyed shuffle (non-empty partition keys). #[inline] - pub(crate) fn keyed_aggregate_parallelism(&self) -> usize { + pub fn keyed_aggregate_parallelism(&self) -> usize { sql_planning_default::KEYED_AGGREGATE_DEFAULT_PARALLELISM } - pub(crate) fn new( - schema_provider: &'a StreamSchemaProvider, - session_state: &'a SessionState, - ) -> Self { + pub fn new(schema_provider: &'a StreamSchemaProvider, session_state: &'a SessionState) -> Self { let planner = DefaultPhysicalPlanner::with_extension_planners(vec![Arc::new(FsExtensionPlanner {})]); Self { @@ -126,7 +121,7 @@ impl<'a> Planner<'a> { } } - pub(crate) fn sync_plan(&self, plan: &LogicalPlan) -> Result> { + pub fn sync_plan(&self, plan: &LogicalPlan) -> Result> { let fut = self.planner.create_physical_plan(plan, self.session_state); let (tx, mut rx) = oneshot::channel(); thread::scope(|s| { @@ -150,7 +145,7 @@ impl<'a> Planner<'a> { rx.try_recv().unwrap() } - pub(crate) fn create_physical_expr( + pub fn create_physical_expr( &self, expr: &Expr, input_dfschema: &DFSchema, @@ -159,17 +154,13 @@ impl<'a> Planner<'a> { .create_physical_expr(expr, input_dfschema, self.session_state) } - pub(crate) fn serialize_as_physical_expr( - &self, - expr: &Expr, - schema: &DFSchema, - ) -> Result> { + pub fn serialize_as_physical_expr(&self, expr: &Expr, schema: &DFSchema) -> Result> { let physical = self.create_physical_expr(expr, schema)?; let proto = serialize_physical_expr(&physical, &DefaultPhysicalExtensionCodec {})?; Ok(proto.encode_to_vec()) } - pub(crate) fn split_physical_plan( + pub fn split_physical_plan( &self, key_indices: Vec, aggregate: &LogicalPlan, @@ -313,7 +304,7 @@ impl PlanToGraphVisitor<'_> { } } - pub(crate) fn add_plan(&mut self, plan: LogicalPlan) -> Result<()> { + pub fn add_plan(&mut self, plan: LogicalPlan) -> Result<()> { self.traversal.clear(); plan.visit(self)?; Ok(()) diff --git a/src/streaming_planner/src/parse.rs b/src/streaming_planner/src/parse.rs new file mode 100644 index 00000000..563214aa --- /dev/null +++ b/src/streaming_planner/src/parse.rs @@ -0,0 +1,82 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! FunctionStream SQL parsing (`parse_sql`). +//! +//! This module only performs lexical/syntactic parsing into sqlparser +//! [`Statement`](datafusion::sql::sqlparser::ast::Statement) values using +//! [`FunctionStreamDialect`]. Mapping those AST nodes to coordinator `Statement` +//! values is done by `classify_statement` in the `function-stream` coordinator module. +//! +//! **Data-definition / pipeline shape (supported forms in the dialect)** +//! - **`CREATE TABLE ... (cols [, WATERMARK FOR ...]) WITH (...)`** — connector-backed source DDL +//! - **`CREATE TABLE ...`** other forms (including `AS SELECT` where the dialect accepts it) +//! - **`CREATE STREAMING TABLE ... WITH (...) AS SELECT ...`** +//! - **`DROP TABLE`** / **`DROP STREAMING TABLE`** +//! - **`SHOW TABLES`**, **`SHOW STREAMING TABLE(S)`**, **`SHOW CREATE TABLE`**, **`SHOW CREATE STREAMING TABLE`** +//! +//! **`INSERT` is not supported** at the coordinator layer — use `CREATE TABLE ... AS SELECT` or +//! `CREATE STREAMING TABLE ... AS SELECT` instead (see coordinator classification). + +use datafusion::common::{Result, plan_err}; +use datafusion::error::DataFusionError; +use datafusion::sql::sqlparser::ast::Statement as DFStatement; +use datafusion::sql::sqlparser::dialect::FunctionStreamDialect; +use datafusion::sql::sqlparser::parser::Parser; + +/// Parse SQL text into zero or more dialect [`Statement`](DFStatement) nodes. +pub fn parse_sql(query: &str) -> Result> { + let trimmed = query.trim(); + if trimmed.is_empty() { + return plan_err!("Query is empty"); + } + + let dialect = FunctionStreamDialect {}; + let statements = Parser::parse_sql(&dialect, trimmed) + .map_err(|e| DataFusionError::Plan(format!("SQL parse error: {e}")))?; + + if statements.is_empty() { + return plan_err!("No SQL statements found"); + } + + Ok(statements) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_multiple_statements_ast() { + let sql = concat!( + "CREATE TABLE t1 (id INT); ", + "CREATE STREAMING TABLE sk WITH ('connector' = 'kafka') AS SELECT id FROM t1", + ); + let stmts = parse_sql(sql).unwrap(); + assert_eq!(stmts.len(), 2); + assert!(matches!(stmts[0], DFStatement::CreateTable(_))); + assert!(matches!(stmts[1], DFStatement::CreateStreamingTable { .. })); + } + + #[test] + fn test_parse_empty() { + assert!(parse_sql("").is_err()); + assert!(parse_sql(" ").is_err()); + } + + #[test] + fn test_parse_select_yields_query_ast() { + let stmts = parse_sql("SELECT 1").unwrap(); + assert_eq!(stmts.len(), 1); + assert!(matches!(stmts[0], DFStatement::Query(_))); + } +} diff --git a/src/sql/physical/cdc/encode.rs b/src/streaming_planner/src/physical/cdc/encode.rs similarity index 97% rename from src/sql/physical/cdc/encode.rs rename to src/streaming_planner/src/physical/cdc/encode.rs index 65ec758d..8457724c 100644 --- a/src/sql/physical/cdc/encode.rs +++ b/src/streaming_planner/src/physical/cdc/encode.rs @@ -28,9 +28,9 @@ use datafusion::execution::{RecordBatchStream, SendableRecordBatchStream, TaskCo use datafusion::physical_plan::{DisplayAs, ExecutionPlan, PlanProperties}; use futures::{StreamExt, ready, stream::Stream}; -use crate::sql::common::constants::{cdc, debezium_op_short, physical_plan_node_name}; -use crate::sql::common::{TIMESTAMP_FIELD, UPDATING_META_FIELD}; -use crate::sql::physical::source_exec::make_stream_properties; +use crate::common::constants::{cdc, debezium_op_short, physical_plan_node_name}; +use crate::common::{TIMESTAMP_FIELD, UPDATING_META_FIELD}; +use crate::physical::source_exec::make_stream_properties; // ============================================================================ // CdcDebeziumPackExec (Execution Plan Node) @@ -85,7 +85,7 @@ impl CdcDebeziumPackExec { }) } - pub(crate) fn from_decoded_parts(input: Arc, schema: SchemaRef) -> Self { + pub fn from_decoded_parts(input: Arc, schema: SchemaRef) -> Self { Self { properties: make_stream_properties(schema.clone()), input, diff --git a/src/sql/physical/cdc/mod.rs b/src/streaming_planner/src/physical/cdc/mod.rs similarity index 100% rename from src/sql/physical/cdc/mod.rs rename to src/streaming_planner/src/physical/cdc/mod.rs diff --git a/src/sql/physical/cdc/unroll.rs b/src/streaming_planner/src/physical/cdc/unroll.rs similarity index 96% rename from src/sql/physical/cdc/unroll.rs rename to src/streaming_planner/src/physical/cdc/unroll.rs index 10c62c6c..963283cc 100644 --- a/src/sql/physical/cdc/unroll.rs +++ b/src/streaming_planner/src/physical/cdc/unroll.rs @@ -29,11 +29,11 @@ use datafusion::logical_expr::ColumnarValue; use datafusion::physical_plan::{DisplayAs, ExecutionPlan, PlanProperties}; use futures::{StreamExt, ready, stream::Stream}; -use crate::sql::common::TIMESTAMP_FIELD; -use crate::sql::common::constants::{cdc, debezium_op_short, physical_plan_node_name}; -use crate::sql::functions::MultiHashFunction; -use crate::sql::physical::meta::{updating_meta_field, updating_meta_fields}; -use crate::sql::physical::source_exec::make_stream_properties; +use crate::common::TIMESTAMP_FIELD; +use crate::common::constants::{cdc, debezium_op_short, physical_plan_node_name}; +use crate::functions::MultiHashFunction; +use crate::physical::meta::{updating_meta_field, updating_meta_fields}; +use crate::physical::source_exec::make_stream_properties; // ============================================================================ // CdcDebeziumUnrollExec (Execution Plan Node) @@ -103,7 +103,7 @@ impl CdcDebeziumUnrollExec { } /// Used when deserializing a plan with a pre-baked output schema (see [`StreamingExtensionCodec`]). - pub(crate) fn from_decoded_parts( + pub fn from_decoded_parts( input: Arc, schema: SchemaRef, primary_key_indices: Vec, diff --git a/src/sql/physical/codec.rs b/src/streaming_planner/src/physical/codec.rs similarity index 97% rename from src/sql/physical/codec.rs rename to src/streaming_planner/src/physical/codec.rs index 1b96a9d6..2fb6fcfc 100644 --- a/src/sql/physical/codec.rs +++ b/src/streaming_planner/src/physical/codec.rs @@ -27,13 +27,13 @@ use protocol::function_stream_graph::{ }; use tokio::sync::mpsc::UnboundedReceiver; -use crate::sql::analysis::UNNESTED_COL; -use crate::sql::common::constants::{mem_exec_join_side, window_function_udf}; -use crate::sql::physical::cdc::{CdcDebeziumPackExec, CdcDebeziumUnrollExec}; -use crate::sql::physical::source_exec::{ +use crate::analysis::UNNESTED_COL; +use crate::common::constants::{mem_exec_join_side, window_function_udf}; +use crate::physical::cdc::{CdcDebeziumPackExec, CdcDebeziumUnrollExec}; +use crate::physical::source_exec::{ BufferedBatchesExec, InjectableSingleBatchExec, MpscReceiverStreamExec, PlanningPlaceholderExec, }; -use crate::sql::physical::udfs::window; +use crate::physical::udfs::window; // ============================================================================ // StreamingExtensionCodec & StreamingDecodingContext diff --git a/src/sql/physical/meta.rs b/src/streaming_planner/src/physical/meta.rs similarity index 93% rename from src/sql/physical/meta.rs rename to src/streaming_planner/src/physical/meta.rs index 1387482c..ced37656 100644 --- a/src/sql/physical/meta.rs +++ b/src/streaming_planner/src/physical/meta.rs @@ -14,8 +14,8 @@ use std::sync::{Arc, OnceLock}; use datafusion::arrow::datatypes::{DataType, Field, Fields}; -use crate::sql::common::UPDATING_META_FIELD; -use crate::sql::common::constants::updating_state_field; +use crate::common::UPDATING_META_FIELD; +use crate::common::constants::updating_state_field; pub fn updating_meta_fields() -> Fields { static FIELDS: OnceLock = OnceLock::new(); diff --git a/src/sql/physical/mod.rs b/src/streaming_planner/src/physical/mod.rs similarity index 100% rename from src/sql/physical/mod.rs rename to src/streaming_planner/src/physical/mod.rs diff --git a/src/sql/physical/source_exec.rs b/src/streaming_planner/src/physical/source_exec.rs similarity index 96% rename from src/sql/physical/source_exec.rs rename to src/streaming_planner/src/physical/source_exec.rs index fa65cbfd..c94c1ca3 100644 --- a/src/sql/physical/source_exec.rs +++ b/src/streaming_planner/src/physical/source_exec.rs @@ -29,11 +29,11 @@ use futures::StreamExt; use tokio::sync::mpsc::UnboundedReceiver; use tokio_stream::wrappers::UnboundedReceiverStream; -use crate::sql::common::constants::physical_plan_node_name; +use crate::common::constants::physical_plan_node_name; /// Standard [`PlanProperties`] for a continuous, unbounded stream: incremental emission, /// unknown partitioning, and unbounded boundedness (without requiring infinite memory). -pub(crate) fn create_unbounded_stream_properties(schema: SchemaRef) -> PlanProperties { +pub fn create_unbounded_stream_properties(schema: SchemaRef) -> PlanProperties { PlanProperties::new( EquivalenceProperties::new(schema), Partitioning::UnknownPartitioning(1), @@ -45,7 +45,7 @@ pub(crate) fn create_unbounded_stream_properties(schema: SchemaRef) -> PlanPrope } /// Alias for call sites that still use the older name. -pub(crate) fn make_stream_properties(schema: SchemaRef) -> PlanProperties { +pub fn make_stream_properties(schema: SchemaRef) -> PlanProperties { create_unbounded_stream_properties(schema) } @@ -58,14 +58,14 @@ pub(crate) fn make_stream_properties(schema: SchemaRef) -> PlanProperties { /// For event-driven loops that receive a single batch from the network and run a DataFusion /// plan over it, the batch is stored in the lock until execution starts. #[derive(Debug)] -pub(crate) struct InjectableSingleBatchExec { +pub struct InjectableSingleBatchExec { schema: SchemaRef, injected_batch: Arc>>, properties: PlanProperties, } impl InjectableSingleBatchExec { - pub(crate) fn new( + pub fn new( schema: SchemaRef, injected_batch: Arc>>, ) -> Self { @@ -152,14 +152,14 @@ impl ExecutionPlan for InjectableSingleBatchExec { /// /// Bridges async producers (e.g. network threads) into a DataFusion pipeline. #[derive(Debug)] -pub(crate) struct MpscReceiverStreamExec { +pub struct MpscReceiverStreamExec { schema: SchemaRef, channel_receiver: Arc>>>, properties: PlanProperties, } impl MpscReceiverStreamExec { - pub(crate) fn new( + pub fn new( schema: SchemaRef, channel_receiver: Arc>>>, ) -> Self { @@ -243,14 +243,14 @@ impl ExecutionPlan for MpscReceiverStreamExec { /// Drains a growable, locked `Vec` when `execute` runs (micro-batching). #[derive(Debug)] -pub(crate) struct BufferedBatchesExec { +pub struct BufferedBatchesExec { schema: SchemaRef, buffered_batches: Arc>>, properties: PlanProperties, } impl BufferedBatchesExec { - pub(crate) fn new( + pub fn new( schema: SchemaRef, buffered_batches: Arc>>, ) -> Self { diff --git a/src/sql/physical/udfs.rs b/src/streaming_planner/src/physical/udfs.rs similarity index 97% rename from src/sql/physical/udfs.rs rename to src/streaming_planner/src/physical/udfs.rs index 767abf06..0a11ba4f 100644 --- a/src/sql/physical/udfs.rs +++ b/src/streaming_planner/src/physical/udfs.rs @@ -20,9 +20,9 @@ use datafusion::logical_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility, }; +use crate::common::constants::window_function_udf; use crate::make_udf_function; -use crate::sql::common::constants::window_function_udf; -use crate::sql::schema::utils::window_arrow_struct; +use crate::schema::utils::window_arrow_struct; // ============================================================================ // WindowFunctionUdf (User-Defined Scalar Function) diff --git a/src/sql/planning_runtime.rs b/src/streaming_planner/src/planning_runtime.rs similarity index 85% rename from src/sql/planning_runtime.rs rename to src/streaming_planner/src/planning_runtime.rs index dc4749ad..cb2ab93e 100644 --- a/src/sql/planning_runtime.rs +++ b/src/streaming_planner/src/planning_runtime.rs @@ -14,9 +14,9 @@ use std::sync::OnceLock; -use crate::config::streaming_job::ResolvedStreamingJobConfig; -use crate::sql::common::constants::sql_planning_default; -use crate::sql::types::SqlConfig; +use crate::common::constants::sql_planning_default; +use crate::types::SqlConfig; +use function_stream_config::streaming_job::ResolvedStreamingJobConfig; static SQL_PLANNING: OnceLock = OnceLock::new(); @@ -30,6 +30,6 @@ pub fn install_sql_planning_from_streaming_job(job: &ResolvedStreamingJobConfig) let _ = SQL_PLANNING.set(cfg).ok(); } -pub(crate) fn sql_planning_snapshot() -> SqlConfig { +pub fn sql_planning_snapshot() -> SqlConfig { SQL_PLANNING.get().cloned().unwrap_or_default() } diff --git a/src/sql/schema/catalog.rs b/src/streaming_planner/src/schema/catalog.rs similarity index 99% rename from src/sql/schema/catalog.rs rename to src/streaming_planner/src/schema/catalog.rs index 479df682..35966289 100644 --- a/src/sql/schema/catalog.rs +++ b/src/streaming_planner/src/schema/catalog.rs @@ -26,11 +26,11 @@ use super::column_descriptor::ColumnDescriptor; use super::data_encoding_format::DataEncodingFormat; use super::table::SqlSource; use super::temporal_pipeline_config::TemporalPipelineConfig; +use crate::common::constants::sql_field; +use crate::common::{Format, FsSchema}; +use crate::connector::config::ConnectorConfig; use crate::multifield_partial_ord; -use crate::sql::common::constants::sql_field; -use crate::sql::common::{Format, FsSchema}; -use crate::sql::connector::config::ConnectorConfig; -use crate::sql::types::ProcessingMode; +use crate::types::ProcessingMode; #[derive(Debug, Clone)] pub struct EngineDescriptor { diff --git a/src/sql/schema/column_descriptor.rs b/src/streaming_planner/src/schema/column_descriptor.rs similarity index 100% rename from src/sql/schema/column_descriptor.rs rename to src/streaming_planner/src/schema/column_descriptor.rs diff --git a/src/sql/schema/connection_type.rs b/src/streaming_planner/src/schema/connection_type.rs similarity index 100% rename from src/sql/schema/connection_type.rs rename to src/streaming_planner/src/schema/connection_type.rs diff --git a/src/sql/schema/data_encoding_format.rs b/src/streaming_planner/src/schema/data_encoding_format.rs similarity index 97% rename from src/sql/schema/data_encoding_format.rs rename to src/streaming_planner/src/schema/data_encoding_format.rs index 0b6f5e1d..b589d683 100644 --- a/src/sql/schema/data_encoding_format.rs +++ b/src/streaming_planner/src/schema/data_encoding_format.rs @@ -14,8 +14,8 @@ use datafusion::arrow::datatypes::{DataType, Field}; use datafusion::common::{Result, plan_err}; use super::column_descriptor::ColumnDescriptor; -use crate::sql::common::Format; -use crate::sql::common::constants::cdc; +use crate::common::Format; +use crate::common::constants::cdc; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] pub enum DataEncodingFormat { diff --git a/src/coordinator/plan/ddl_compiler.rs b/src/streaming_planner/src/schema/introspection/ddl_compiler.rs similarity index 89% rename from src/coordinator/plan/ddl_compiler.rs rename to src/streaming_planner/src/schema/introspection/ddl_compiler.rs index 5c5af39e..3a0dde74 100644 --- a/src/coordinator/plan/ddl_compiler.rs +++ b/src/streaming_planner/src/schema/introspection/ddl_compiler.rs @@ -22,21 +22,21 @@ use datafusion::sql::TableReference; use datafusion::sql::planner::{PlannerContext, SqlToRel}; use datafusion::sql::sqlparser::ast; use datafusion::sql::sqlparser::ast::CreateTable as SqlCreateTable; +use datafusion::sql::sqlparser::ast::Statement as DfStatement; use datafusion_expr::ExprSchemable; use tracing::warn; -use super::ast_utils::AstUtils; -use crate::coordinator::tool::ConnectorOptions; -use crate::sql::analysis::StreamSchemaProvider; -use crate::sql::common::constants::{connection_table_role, connector_type, sql_field}; -use crate::sql::common::with_option_keys as opt; -use crate::sql::common::{BadData, Format, Framing, JsonCompression, JsonFormat}; -use crate::sql::connector::registry::REGISTRY; -use crate::sql::schema::ColumnDescriptor; -use crate::sql::schema::catalog::{ExternalTable, LookupTable, SourceTable}; -use crate::sql::schema::data_encoding_format::DataEncodingFormat; -use crate::sql::schema::table_role::{apply_adapter_specific_rules, validate_adapter_availability}; -use crate::sql::schema::temporal_pipeline_config::TemporalPipelineConfig; +use crate::common::ast_utils::AstUtils; +use crate::common::constants::{connection_table_role, connector_type, sql_field}; +use crate::common::with_option_keys as opt; +use crate::common::{BadData, ConnectorOptions, Format, Framing, JsonCompression, JsonFormat}; +use crate::connector::registry::REGISTRY; +use crate::schema::ColumnDescriptor; +use crate::schema::catalog::{ExternalTable, LookupTable, SourceTable}; +use crate::schema::data_encoding_format::DataEncodingFormat; +use crate::schema::schema_provider::StreamSchemaProvider; +use crate::schema::table_role::{apply_adapter_specific_rules, validate_adapter_availability}; +use crate::schema::temporal_pipeline_config::TemporalPipelineConfig; pub struct DdlCompiler<'a> { schema_provider: &'a StreamSchemaProvider, @@ -293,6 +293,30 @@ impl<'a> DdlCompiler<'a> { } } +/// If `stmt` is `CREATE TABLE ... WITH (connector = ...)`, compile it to [`ExternalTable`]. +pub fn try_compile_connector_create_table( + schema_provider: &StreamSchemaProvider, + stmt: &DfStatement, +) -> Option> { + let DfStatement::CreateTable(ast_node) = stmt else { + return None; + }; + if ast_node.query.is_some() { + return None; + } + if !AstUtils::contains_connector_property(&ast_node.with_options) { + return None; + } + let declared_role = AstUtils::peek_table_role(&ast_node.with_options); + let if_not_exists = ast_node.if_not_exists; + let compiler = DdlCompiler::new(schema_provider); + Some( + compiler + .compile(ast_node, declared_role.as_deref()) + .map(|table| (table, if_not_exists)), + ) +} + fn resolve_source_watermark( table_identifier: &str, columns: &mut Vec, diff --git a/src/sql/schema/introspection/ddl_formatter.rs b/src/streaming_planner/src/schema/introspection/ddl_formatter.rs similarity index 99% rename from src/sql/schema/introspection/ddl_formatter.rs rename to src/streaming_planner/src/schema/introspection/ddl_formatter.rs index f4ce36e6..15d8f707 100644 --- a/src/sql/schema/introspection/ddl_formatter.rs +++ b/src/streaming_planner/src/schema/introspection/ddl_formatter.rs @@ -15,7 +15,7 @@ use std::fmt::{self, Write}; use datafusion::arrow::datatypes::{DataType, Schema, TimeUnit}; -use crate::sql::common::constants::sql_field; +use crate::common::constants::sql_field; pub struct DdlBuilder<'a> { table_name: &'a str, diff --git a/src/sql/schema/introspection/mod.rs b/src/streaming_planner/src/schema/introspection/mod.rs similarity index 90% rename from src/sql/schema/introspection/mod.rs rename to src/streaming_planner/src/schema/introspection/mod.rs index 1ba9c816..33efdf90 100644 --- a/src/sql/schema/introspection/mod.rs +++ b/src/streaming_planner/src/schema/introspection/mod.rs @@ -10,10 +10,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod ddl_compiler; pub mod ddl_formatter; pub mod show_formatter; pub mod stream_formatter; +pub use ddl_compiler::{DdlCompiler, try_compile_connector_create_table}; #[allow(unused_imports)] pub use ddl_formatter::{DdlBuilder, format_data_type, schema_columns_one_line}; pub use show_formatter::{catalog_table_row_detail, show_create_catalog_table}; diff --git a/src/sql/schema/introspection/show_formatter.rs b/src/streaming_planner/src/schema/introspection/show_formatter.rs similarity index 96% rename from src/sql/schema/introspection/show_formatter.rs rename to src/streaming_planner/src/schema/introspection/show_formatter.rs index 28a81ae9..3857ba21 100644 --- a/src/sql/schema/introspection/show_formatter.rs +++ b/src/streaming_planner/src/schema/introspection/show_formatter.rs @@ -10,9 +10,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sql::common::constants::connection_table_role; -use crate::sql::schema::catalog::ExternalTable; -use crate::sql::schema::table::CatalogEntity; +use crate::common::constants::connection_table_role; +use crate::schema::catalog::ExternalTable; +use crate::schema::table::CatalogEntity; use super::ddl_formatter::DdlBuilder; diff --git a/src/sql/schema/introspection/stream_formatter.rs b/src/streaming_planner/src/schema/introspection/stream_formatter.rs similarity index 95% rename from src/sql/schema/introspection/stream_formatter.rs rename to src/streaming_planner/src/schema/introspection/stream_formatter.rs index ebb02330..e3239ce8 100644 --- a/src/sql/schema/introspection/stream_formatter.rs +++ b/src/streaming_planner/src/schema/introspection/stream_formatter.rs @@ -14,9 +14,9 @@ use std::sync::Arc; use datafusion::arrow::datatypes::Schema; -use crate::sql::common::constants::connection_table_role; -use crate::sql::logical_node::logical::LogicalProgram; -use crate::sql::schema::schema_provider::StreamTable; +use crate::common::constants::connection_table_role; +use crate::logical_node::logical::LogicalProgram; +use crate::schema::schema_provider::StreamTable; use super::ddl_formatter::DdlBuilder; diff --git a/src/sql/schema/mod.rs b/src/streaming_planner/src/schema/mod.rs similarity index 94% rename from src/sql/schema/mod.rs rename to src/streaming_planner/src/schema/mod.rs index e11e4808..75546636 100644 --- a/src/sql/schema/mod.rs +++ b/src/streaming_planner/src/schema/mod.rs @@ -24,7 +24,8 @@ pub mod utils; pub use catalog::{ExternalTable, LookupTable, SinkTable, SourceTable}; pub use column_descriptor::ColumnDescriptor; pub use introspection::{ - catalog_table_row_detail, schema_columns_one_line, show_create_catalog_table, + catalog_table_row_detail, schema_columns_one_line, show_create_catalog_table, DdlCompiler, + try_compile_connector_create_table, }; pub use schema_provider::{ObjectName, StreamPlanningContext, StreamSchemaProvider, StreamTable}; pub use table::CatalogEntity; diff --git a/src/sql/schema/schema_provider.rs b/src/streaming_planner/src/schema/schema_provider.rs similarity index 96% rename from src/sql/schema/schema_provider.rs rename to src/streaming_planner/src/schema/schema_provider.rs index 15cd58ee..75026c3f 100644 --- a/src/sql/schema/schema_provider.rs +++ b/src/streaming_planner/src/schema/schema_provider.rs @@ -27,11 +27,11 @@ use thiserror::Error; use tracing::{debug, error, info}; use unicase::UniCase; -use crate::sql::common::constants::{planning_placeholder_udf, window_fn}; -use crate::sql::logical_node::logical::{DylibUdfConfig, LogicalProgram}; -use crate::sql::schema::table::CatalogEntity; -use crate::sql::schema::utils::window_arrow_struct; -use crate::sql::types::{PlanningOptions, PlanningPlaceholderUdf, SqlConfig}; +use crate::common::constants::{planning_placeholder_udf, window_fn}; +use crate::logical_node::logical::{DylibUdfConfig, LogicalProgram}; +use crate::schema::table::CatalogEntity; +use crate::schema::utils::window_arrow_struct; +use crate::types::{PlanningOptions, PlanningPlaceholderUdf, SqlConfig}; pub type ObjectName = UniCase; @@ -119,7 +119,7 @@ impl TableProvider for LogicalBatchInput { _filters: &[Expr], _limit: Option, ) -> DataFusionResult> { - Ok(Arc::new(crate::sql::physical::FsMemExec::new( + Ok(Arc::new(crate::physical::FsMemExec::new( self.table_name.clone(), Arc::clone(&self.schema), ))) @@ -181,7 +181,7 @@ impl StreamPlanningContext { } pub fn new() -> Self { - let config = crate::sql::planning_runtime::sql_planning_snapshot(); + let config = crate::planning_runtime::sql_planning_snapshot(); Self::try_new(config).expect("StreamPlanningContext bootstrap") } @@ -264,7 +264,7 @@ impl StreamPlanningContext { .map(Arc::make_mut) } - pub fn get_async_udf_options(&self, _name: &str) -> Option { + pub fn get_async_udf_options(&self, _name: &str) -> Option { None } diff --git a/src/sql/schema/table.rs b/src/streaming_planner/src/schema/table.rs similarity index 94% rename from src/sql/schema/table.rs rename to src/streaming_planner/src/schema/table.rs index 6c001d9c..912e9b00 100644 --- a/src/sql/schema/table.rs +++ b/src/streaming_planner/src/schema/table.rs @@ -10,12 +10,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sql::analysis::rewrite_plan; -use crate::sql::logical_node::remote_table::RemoteTableBoundaryNode; -use crate::sql::logical_planner::optimizers::produce_optimized_plan; -use crate::sql::schema::StreamSchemaProvider; -use crate::sql::schema::catalog::ExternalTable; -use crate::sql::types::{ProcessingMode, QualifiedField}; +use crate::analysis::rewrite_plan; +use crate::logical_node::remote_table::RemoteTableBoundaryNode; +use crate::logical_planner::optimizers::produce_optimized_plan; +use crate::schema::StreamSchemaProvider; +use crate::schema::catalog::ExternalTable; +use crate::types::{ProcessingMode, QualifiedField}; use datafusion::arrow::datatypes::FieldRef; use datafusion::common::{Result, plan_err}; use datafusion::logical_expr::{Extension, LogicalPlan}; diff --git a/src/sql/schema/table_role.rs b/src/streaming_planner/src/schema/table_role.rs similarity index 97% rename from src/sql/schema/table_role.rs rename to src/streaming_planner/src/schema/table_role.rs index 7d301f9d..4574a6ea 100644 --- a/src/sql/schema/table_role.rs +++ b/src/streaming_planner/src/schema/table_role.rs @@ -18,10 +18,10 @@ use datafusion::error::DataFusionError; use super::column_descriptor::ColumnDescriptor; use super::connection_type::ConnectionType; -use crate::sql::common::constants::{ +use crate::common::constants::{ SUPPORTED_CONNECTOR_ADAPTERS, connection_table_role, connector_type, }; -use crate::sql::common::with_option_keys as opt; +use crate::common::with_option_keys as opt; /// Role of a connector-backed table in the pipeline (ingest / egress / lookup). #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] diff --git a/src/sql/schema/temporal_pipeline_config.rs b/src/streaming_planner/src/schema/temporal_pipeline_config.rs similarity index 97% rename from src/sql/schema/temporal_pipeline_config.rs rename to src/streaming_planner/src/schema/temporal_pipeline_config.rs index f672e552..db751b1c 100644 --- a/src/sql/schema/temporal_pipeline_config.rs +++ b/src/streaming_planner/src/schema/temporal_pipeline_config.rs @@ -16,7 +16,7 @@ use datafusion::common::{Result, plan_err}; use datafusion::logical_expr::Expr; use super::column_descriptor::ColumnDescriptor; -use crate::sql::common::constants::sql_field; +use crate::common::constants::sql_field; /// Event-time and watermark configuration for streaming tables. #[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] diff --git a/src/sql/schema/utils.rs b/src/streaming_planner/src/schema/utils.rs similarity index 95% rename from src/sql/schema/utils.rs rename to src/streaming_planner/src/schema/utils.rs index 45254e5f..be5efbab 100644 --- a/src/sql/schema/utils.rs +++ b/src/streaming_planner/src/schema/utils.rs @@ -16,8 +16,8 @@ use std::sync::Arc; use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit}; use datafusion::common::{DFSchema, DFSchemaRef, Result as DFResult, TableReference}; -use crate::sql::common::constants::window_interval_field; -use crate::sql::types::{QualifiedField, TIMESTAMP_FIELD}; +use crate::common::constants::window_interval_field; +use crate::types::{QualifiedField, TIMESTAMP_FIELD}; /// Returns the Arrow struct type for a window (start, end) pair. pub fn window_arrow_struct() -> DataType { diff --git a/src/sql/types/data_type.rs b/src/streaming_planner/src/types/data_type.rs similarity index 98% rename from src/sql/types/data_type.rs rename to src/streaming_planner/src/types/data_type.rs index 387a4190..44bb8087 100644 --- a/src/sql/types/data_type.rs +++ b/src/streaming_planner/src/types/data_type.rs @@ -17,8 +17,8 @@ use datafusion::arrow::datatypes::{ }; use datafusion::common::{Result, plan_datafusion_err, plan_err}; -use crate::sql::common::FsExtensionType; -use crate::sql::common::constants::planning_placeholder_udf; +use crate::common::FsExtensionType; +use crate::common::constants::planning_placeholder_udf; pub fn convert_data_type( sql_type: &datafusion::sql::sqlparser::ast::DataType, diff --git a/src/sql/types/df_field.rs b/src/streaming_planner/src/types/df_field.rs similarity index 100% rename from src/sql/types/df_field.rs rename to src/streaming_planner/src/types/df_field.rs diff --git a/src/sql/types/mod.rs b/src/streaming_planner/src/types/mod.rs similarity index 83% rename from src/sql/types/mod.rs rename to src/streaming_planner/src/types/mod.rs index d5124bcc..a796bfea 100644 --- a/src/sql/types/mod.rs +++ b/src/streaming_planner/src/types/mod.rs @@ -12,22 +12,22 @@ mod data_type; mod df_field; -pub(crate) mod placeholder_udf; +pub mod placeholder_udf; mod stream_schema; mod window; use std::time::Duration; -use crate::sql::common::constants::sql_planning_default; +use crate::common::constants::sql_planning_default; pub use df_field::{ QualifiedField, build_df_schema, build_df_schema_with_metadata, extract_qualified_fields, }; -pub(crate) use placeholder_udf::PlanningPlaceholderUdf; -pub(crate) use window::WindowBehavior; +pub use placeholder_udf::PlanningPlaceholderUdf; +pub use window::WindowBehavior; pub use window::{WindowType, extract_window_type}; -pub use crate::sql::common::constants::sql_field::TIMESTAMP_FIELD; +pub use crate::common::constants::sql_field::TIMESTAMP_FIELD; #[derive(Clone, Debug, Eq, PartialEq)] pub enum ProcessingMode { @@ -38,7 +38,7 @@ pub enum ProcessingMode { #[derive(Clone, Debug)] pub struct SqlConfig { pub default_parallelism: usize, - /// Physical pipeline parallelism for [`KeyExtractionNode`](crate::sql::logical_node::key_calculation::KeyExtractionNode) / KeyBy. + /// Physical pipeline parallelism for [`KeyExtractionNode`](crate::logical_node::key_calculation::KeyExtractionNode) / KeyBy. pub key_by_parallelism: usize, } diff --git a/src/sql/types/placeholder_udf.rs b/src/streaming_planner/src/types/placeholder_udf.rs similarity index 98% rename from src/sql/types/placeholder_udf.rs rename to src/streaming_planner/src/types/placeholder_udf.rs index 059637e9..2f4f3f3c 100644 --- a/src/sql/types/placeholder_udf.rs +++ b/src/streaming_planner/src/types/placeholder_udf.rs @@ -25,7 +25,7 @@ use datafusion::logical_expr::{ // ============================================================================ /// Logical-planning-only UDF: satisfies type checking until real functions are wired in. -pub(crate) struct PlanningPlaceholderUdf { +pub struct PlanningPlaceholderUdf { name: String, signature: Signature, return_type: DataType, diff --git a/src/sql/types/stream_schema.rs b/src/streaming_planner/src/types/stream_schema.rs similarity index 100% rename from src/sql/types/stream_schema.rs rename to src/streaming_planner/src/types/stream_schema.rs diff --git a/src/sql/types/window.rs b/src/streaming_planner/src/types/window.rs similarity index 98% rename from src/sql/types/window.rs rename to src/streaming_planner/src/types/window.rs index 1aa05f42..69401f51 100644 --- a/src/sql/types/window.rs +++ b/src/streaming_planner/src/types/window.rs @@ -16,7 +16,7 @@ use datafusion::common::{Result, ScalarValue, not_impl_err, plan_err}; use datafusion::logical_expr::Expr; use datafusion::logical_expr::expr::{Alias, ScalarFunction}; -use crate::sql::common::constants::window_fn; +use crate::common::constants::window_fn; use super::QualifiedField; @@ -35,7 +35,7 @@ pub enum WindowType { /// How windowing is represented in the physical plan. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) enum WindowBehavior { +pub enum WindowBehavior { FromOperator { window: WindowType, window_field: QualifiedField, diff --git a/src/streaming_runtime/Cargo.toml b/src/streaming_runtime/Cargo.toml new file mode 100644 index 00000000..026b50e1 --- /dev/null +++ b/src/streaming_runtime/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "function-stream-streaming-runtime" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_streaming_runtime" +path = "src/lib.rs" diff --git a/src/lib.rs b/src/streaming_runtime/src/lib.rs similarity index 54% rename from src/lib.rs rename to src/streaming_runtime/src/lib.rs index a6bb4d28..aeb056d5 100644 --- a/src/lib.rs +++ b/src/streaming_runtime/src/lib.rs @@ -10,14 +10,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Library crate for function-stream +//! Streaming execution runtime. +//! +//! The streaming engine and shared runtime helpers (`streaming/`, `util/`) are +//! implemented under [`src/streaming`] and [`src/util`] in this package. They are +//! currently **compiled as part of the `function-stream` crate** via `#[path]` in +//! `src/lib.rs` / `src/main.rs`, sharing the root `crate::sql` name (re-exported streaming planner crate). -#![allow(dead_code)] - -pub mod config; -pub mod coordinator; -pub mod logging; -pub mod runtime; -pub mod server; -pub mod sql; -pub mod storage; +pub const CRATE_NAME: &str = "function-stream-streaming-runtime"; diff --git a/src/runtime/streaming/api/context.rs b/src/streaming_runtime/src/streaming/api/context.rs similarity index 96% rename from src/runtime/streaming/api/context.rs rename to src/streaming_runtime/src/streaming/api/context.rs index f2557e7a..a29d7ac2 100644 --- a/src/runtime/streaming/api/context.rs +++ b/src/streaming_runtime/src/streaming/api/context.rs @@ -19,11 +19,11 @@ use arrow_array::RecordBatch; use protocol::storage::SourceCheckpointInfo; use tokio::sync::mpsc; -use crate::runtime::memory::{MemoryBlock, MemoryPool, get_array_memory_size}; -use crate::runtime::streaming::network::endpoint::PhysicalSender; -use crate::runtime::streaming::protocol::control::JobMasterEvent; -use crate::runtime::streaming::protocol::event::{StreamEvent, TrackedEvent}; -use crate::runtime::streaming::state::IoManager; +use crate::memory::{MemoryBlock, MemoryPool, get_array_memory_size}; +use crate::streaming::network::endpoint::PhysicalSender; +use crate::streaming::protocol::control::JobMasterEvent; +use crate::streaming::protocol::event::{StreamEvent, TrackedEvent}; +use crate::streaming::state::IoManager; #[derive(Debug, Clone)] pub struct TaskContextConfig { diff --git a/src/runtime/streaming/api/mod.rs b/src/streaming_runtime/src/streaming/api/mod.rs similarity index 100% rename from src/runtime/streaming/api/mod.rs rename to src/streaming_runtime/src/streaming/api/mod.rs diff --git a/src/runtime/streaming/api/operator.rs b/src/streaming_runtime/src/streaming/api/operator.rs similarity index 93% rename from src/runtime/streaming/api/operator.rs rename to src/streaming_runtime/src/streaming/api/operator.rs index fc75e475..56cefcee 100644 --- a/src/runtime/streaming/api/operator.rs +++ b/src/streaming_runtime/src/streaming/api/operator.rs @@ -10,10 +10,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::source::SourceOperator; -use crate::runtime::streaming::protocol::event::StreamOutput; use crate::sql::common::{CheckpointBarrier, Watermark}; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::source::SourceOperator; +use crate::streaming::protocol::event::StreamOutput; use arrow_array::RecordBatch; use async_trait::async_trait; diff --git a/src/runtime/streaming/api/source.rs b/src/streaming_runtime/src/streaming/api/source.rs similarity index 98% rename from src/runtime/streaming/api/source.rs rename to src/streaming_runtime/src/streaming/api/source.rs index 9c531f2c..4bed912f 100644 --- a/src/runtime/streaming/api/source.rs +++ b/src/streaming_runtime/src/streaming/api/source.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::streaming::api::context::TaskContext; use crate::sql::common::{CheckpointBarrier, Watermark}; +use crate::streaming::api::context::TaskContext; use arrow_array::RecordBatch; use async_trait::async_trait; use protocol::storage::{ diff --git a/src/runtime/streaming/error.rs b/src/streaming_runtime/src/streaming/error.rs similarity index 100% rename from src/runtime/streaming/error.rs rename to src/streaming_runtime/src/streaming/error.rs diff --git a/src/runtime/streaming/execution/mod.rs b/src/streaming_runtime/src/streaming/execution/mod.rs similarity index 100% rename from src/runtime/streaming/execution/mod.rs rename to src/streaming_runtime/src/streaming/execution/mod.rs diff --git a/src/runtime/streaming/execution/operator_chain.rs b/src/streaming_runtime/src/streaming/execution/operator_chain.rs similarity index 98% rename from src/runtime/streaming/execution/operator_chain.rs rename to src/streaming_runtime/src/streaming/execution/operator_chain.rs index 88e8f441..38584c5d 100644 --- a/src/runtime/streaming/execution/operator_chain.rs +++ b/src/streaming_runtime/src/streaming/execution/operator_chain.rs @@ -13,14 +13,14 @@ use anyhow::anyhow; use async_trait::async_trait; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::error::RunError; -use crate::runtime::streaming::protocol::{ +use crate::sql::common::CheckpointBarrier; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::operator::{Collector, Operator}; +use crate::streaming::error::RunError; +use crate::streaming::protocol::{ control::{ControlCommand, StopMode}, event::{StreamEvent, StreamOutput, TrackedEvent}, }; -use crate::sql::common::CheckpointBarrier; // ============================================================================ // Core Traits diff --git a/src/runtime/streaming/execution/pipeline.rs b/src/streaming_runtime/src/streaming/execution/pipeline.rs similarity index 94% rename from src/runtime/streaming/execution/pipeline.rs rename to src/streaming_runtime/src/streaming/execution/pipeline.rs index 7c2ca17a..6b6b1878 100644 --- a/src/runtime/streaming/execution/pipeline.rs +++ b/src/streaming_runtime/src/streaming/execution/pipeline.rs @@ -14,20 +14,20 @@ use tokio::sync::mpsc::UnboundedReceiver; use tokio_stream::{StreamExt, StreamMap}; use tracing::{Instrument, info, info_span}; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::Operator; -use crate::runtime::streaming::error::RunError; -use crate::runtime::streaming::execution::operator_chain::{ChainBuilder, OperatorDrive}; -use crate::runtime::streaming::execution::tracker::{ +use crate::sql::common::Watermark; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::operator::Operator; +use crate::streaming::error::RunError; +use crate::streaming::execution::operator_chain::{ChainBuilder, OperatorDrive}; +use crate::streaming::execution::tracker::{ barrier_aligner::{AlignmentStatus, BarrierAligner}, watermark_tracker::WatermarkTracker, }; -use crate::runtime::streaming::network::endpoint::BoxedEventStream; -use crate::runtime::streaming::protocol::{ +use crate::streaming::network::endpoint::BoxedEventStream; +use crate::streaming::protocol::{ control::ControlCommand, event::{StreamEvent, TrackedEvent}, }; -use crate::sql::common::Watermark; pub struct Pipeline { chain_head: Box, diff --git a/src/runtime/streaming/execution/source_driver.rs b/src/streaming_runtime/src/streaming/execution/source_driver.rs similarity index 95% rename from src/runtime/streaming/execution/source_driver.rs rename to src/streaming_runtime/src/streaming/execution/source_driver.rs index 0118c4ee..72cc160e 100644 --- a/src/runtime/streaming/execution/source_driver.rs +++ b/src/streaming_runtime/src/streaming/execution/source_driver.rs @@ -14,15 +14,15 @@ use tokio::sync::mpsc::UnboundedReceiver; use tokio::time::{Instant, sleep}; use tracing::{Instrument, info, info_span, warn}; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::source::{SourceCheckpointReport, SourceEvent, SourceOperator}; -use crate::runtime::streaming::error::RunError; -use crate::runtime::streaming::execution::OperatorDrive; -use crate::runtime::streaming::protocol::{ +use crate::sql::common::CheckpointBarrier; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::source::{SourceCheckpointReport, SourceEvent, SourceOperator}; +use crate::streaming::error::RunError; +use crate::streaming::execution::OperatorDrive; +use crate::streaming::protocol::{ control::ControlCommand, event::{StreamEvent, TrackedEvent}, }; -use crate::sql::common::CheckpointBarrier; pub struct SourceDriver { operator: Box, diff --git a/src/runtime/streaming/execution/tracker/barrier_aligner.rs b/src/streaming_runtime/src/streaming/execution/tracker/barrier_aligner.rs similarity index 100% rename from src/runtime/streaming/execution/tracker/barrier_aligner.rs rename to src/streaming_runtime/src/streaming/execution/tracker/barrier_aligner.rs diff --git a/src/runtime/streaming/execution/tracker/mod.rs b/src/streaming_runtime/src/streaming/execution/tracker/mod.rs similarity index 100% rename from src/runtime/streaming/execution/tracker/mod.rs rename to src/streaming_runtime/src/streaming/execution/tracker/mod.rs diff --git a/src/runtime/streaming/execution/tracker/watermark_tracker.rs b/src/streaming_runtime/src/streaming/execution/tracker/watermark_tracker.rs similarity index 97% rename from src/runtime/streaming/execution/tracker/watermark_tracker.rs rename to src/streaming_runtime/src/streaming/execution/tracker/watermark_tracker.rs index af6fd0bc..ec44726d 100644 --- a/src/runtime/streaming/execution/tracker/watermark_tracker.rs +++ b/src/streaming_runtime/src/streaming/execution/tracker/watermark_tracker.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::streaming::protocol::event::{merge_watermarks, watermark_strictly_advances}; use crate::sql::common::Watermark; +use crate::streaming::protocol::event::{merge_watermarks, watermark_strictly_advances}; #[derive(Debug)] pub struct WatermarkTracker { diff --git a/src/runtime/streaming/factory/connector/delta.rs b/src/streaming_runtime/src/streaming/factory/connector/delta.rs similarity index 87% rename from src/runtime/streaming/factory/connector/delta.rs rename to src/streaming_runtime/src/streaming/factory/connector/delta.rs index 726f87ef..0c684e94 100644 --- a/src/runtime/streaming/factory/connector/delta.rs +++ b/src/streaming_runtime/src/streaming/factory/connector/delta.rs @@ -18,16 +18,16 @@ use prost::Message; use protocol::function_stream_graph::ConnectorOp; use protocol::function_stream_graph::connector_op::Config; -use crate::runtime::streaming::api::operator::ConstructedOperator; -use crate::runtime::streaming::factory::connector::sink_props_codec::{ - apply_common_sink_fields, normalized_props, parse_sink_memory_bytes, -}; -use crate::runtime::streaming::factory::global::Registry; -use crate::runtime::streaming::factory::operator_constructor::OperatorConstructor; -use crate::runtime::streaming::operators::sink::delta::{DeltaFormat, DeltaSinkOperator}; -use crate::runtime::streaming::operators::sink::filesystem::compression_from_str; use crate::sql::common::constants::connection_format_value; use crate::sql::common::with_option_keys as opt; +use crate::streaming::api::operator::ConstructedOperator; +use crate::streaming::factory::connector::sink_props_codec::{ + apply_common_sink_fields, normalized_props, parse_sink_memory_bytes, +}; +use crate::streaming::factory::global::Registry; +use crate::streaming::factory::operator_constructor::OperatorConstructor; +use crate::streaming::operators::sink::delta::{DeltaFormat, DeltaSinkOperator}; +use crate::streaming::operators::sink::filesystem::compression_from_str; pub struct DeltaSinkDispatcher; diff --git a/src/runtime/streaming/factory/connector/dispatchers.rs b/src/streaming_runtime/src/streaming/factory/connector/dispatchers.rs similarity index 92% rename from src/runtime/streaming/factory/connector/dispatchers.rs rename to src/streaming_runtime/src/streaming/factory/connector/dispatchers.rs index 7d626600..bcd540a4 100644 --- a/src/runtime/streaming/factory/connector/dispatchers.rs +++ b/src/streaming_runtime/src/streaming/factory/connector/dispatchers.rs @@ -16,10 +16,10 @@ use anyhow::{Context, Result, bail}; use prost::Message; use protocol::function_stream_graph::ConnectorOp; -use crate::runtime::streaming::api::operator::ConstructedOperator; -use crate::runtime::streaming::factory::global::Registry; -use crate::runtime::streaming::factory::operator_constructor::OperatorConstructor; use crate::sql::common::constants::connector_type; +use crate::streaming::api::operator::ConstructedOperator; +use crate::streaming::factory::global::Registry; +use crate::streaming::factory::operator_constructor::OperatorConstructor; use super::{ DeltaSinkDispatcher, FilesystemSinkDispatcher, IcebergSinkDispatcher, LanceDbSinkDispatcher, diff --git a/src/runtime/streaming/factory/connector/filesystem.rs b/src/streaming_runtime/src/streaming/factory/connector/filesystem.rs similarity index 89% rename from src/runtime/streaming/factory/connector/filesystem.rs rename to src/streaming_runtime/src/streaming/factory/connector/filesystem.rs index 94101407..b680590c 100644 --- a/src/runtime/streaming/factory/connector/filesystem.rs +++ b/src/streaming_runtime/src/streaming/factory/connector/filesystem.rs @@ -18,17 +18,17 @@ use prost::Message; use protocol::function_stream_graph::ConnectorOp; use protocol::function_stream_graph::connector_op::Config; -use crate::runtime::streaming::api::operator::ConstructedOperator; -use crate::runtime::streaming::factory::connector::sink_props_codec::{ +use crate::sql::common::constants::connection_format_value; +use crate::sql::common::with_option_keys as opt; +use crate::streaming::api::operator::ConstructedOperator; +use crate::streaming::factory::connector::sink_props_codec::{ apply_common_sink_fields, normalized_props, parse_sink_memory_bytes, }; -use crate::runtime::streaming::factory::global::Registry; -use crate::runtime::streaming::factory::operator_constructor::OperatorConstructor; -use crate::runtime::streaming::operators::sink::filesystem::{ +use crate::streaming::factory::global::Registry; +use crate::streaming::factory::operator_constructor::OperatorConstructor; +use crate::streaming::operators::sink::filesystem::{ FilesystemFormat, FilesystemSinkOperator, compression_from_str, }; -use crate::sql::common::constants::connection_format_value; -use crate::sql::common::with_option_keys as opt; pub struct FilesystemSinkDispatcher; diff --git a/src/runtime/streaming/factory/connector/iceberg.rs b/src/streaming_runtime/src/streaming/factory/connector/iceberg.rs similarity index 86% rename from src/runtime/streaming/factory/connector/iceberg.rs rename to src/streaming_runtime/src/streaming/factory/connector/iceberg.rs index 58e0809f..a6285cbf 100644 --- a/src/runtime/streaming/factory/connector/iceberg.rs +++ b/src/streaming_runtime/src/streaming/factory/connector/iceberg.rs @@ -18,16 +18,16 @@ use prost::Message; use protocol::function_stream_graph::ConnectorOp; use protocol::function_stream_graph::connector_op::Config; -use crate::runtime::streaming::api::operator::ConstructedOperator; -use crate::runtime::streaming::factory::connector::sink_props_codec::{ - apply_common_sink_fields, normalized_props, parse_sink_memory_bytes, -}; -use crate::runtime::streaming::factory::global::Registry; -use crate::runtime::streaming::factory::operator_constructor::OperatorConstructor; -use crate::runtime::streaming::operators::sink::filesystem::compression_from_str; -use crate::runtime::streaming::operators::sink::iceberg::{IcebergFormat, IcebergSinkOperator}; use crate::sql::common::constants::connection_format_value; use crate::sql::common::with_option_keys as opt; +use crate::streaming::api::operator::ConstructedOperator; +use crate::streaming::factory::connector::sink_props_codec::{ + apply_common_sink_fields, normalized_props, parse_sink_memory_bytes, +}; +use crate::streaming::factory::global::Registry; +use crate::streaming::factory::operator_constructor::OperatorConstructor; +use crate::streaming::operators::sink::filesystem::compression_from_str; +use crate::streaming::operators::sink::iceberg::{IcebergFormat, IcebergSinkOperator}; pub struct IcebergSinkDispatcher; diff --git a/src/runtime/streaming/factory/connector/kafka.rs b/src/streaming_runtime/src/streaming/factory/connector/kafka.rs similarity index 94% rename from src/runtime/streaming/factory/connector/kafka.rs rename to src/streaming_runtime/src/streaming/factory/connector/kafka.rs index 17838e3e..ad77af06 100644 --- a/src/runtime/streaming/factory/connector/kafka.rs +++ b/src/streaming_runtime/src/streaming/factory/connector/kafka.rs @@ -24,19 +24,17 @@ use protocol::function_stream_graph::{ }; use tracing::info; -use crate::runtime::streaming::api::operator::ConstructedOperator; -use crate::runtime::streaming::api::source::SourceOffset; -use crate::runtime::streaming::factory::global::Registry; -use crate::runtime::streaming::factory::operator_constructor::OperatorConstructor; -use crate::runtime::streaming::format::{ +use crate::sql::common::FsSchema; +use crate::streaming::api::operator::ConstructedOperator; +use crate::streaming::api::source::SourceOffset; +use crate::streaming::factory::global::Registry; +use crate::streaming::factory::operator_constructor::OperatorConstructor; +use crate::streaming::format::{ BadDataPolicy as RtBadDataPolicy, DataSerializer, DecimalEncoding as RtDecimalEncoding, Format as RuntimeFormat, JsonFormat as RuntimeJsonFormat, TimestampFormat as RtTimestampFormat, }; -use crate::runtime::streaming::operators::sink::kafka::{ConsistencyMode, KafkaSinkOperator}; -use crate::runtime::streaming::operators::source::kafka::{ - BufferedDeserializer, KafkaSourceOperator, -}; -use crate::sql::common::FsSchema; +use crate::streaming::operators::sink::kafka::{ConsistencyMode, KafkaSinkOperator}; +use crate::streaming::operators::source::kafka::{BufferedDeserializer, KafkaSourceOperator}; const DEFAULT_SOURCE_BATCH_SIZE: usize = 1024; diff --git a/src/runtime/streaming/factory/connector/lancedb.rs b/src/streaming_runtime/src/streaming/factory/connector/lancedb.rs similarity index 93% rename from src/runtime/streaming/factory/connector/lancedb.rs rename to src/streaming_runtime/src/streaming/factory/connector/lancedb.rs index ad8bc246..3deb055d 100644 --- a/src/runtime/streaming/factory/connector/lancedb.rs +++ b/src/streaming_runtime/src/streaming/factory/connector/lancedb.rs @@ -18,15 +18,15 @@ use prost::Message; use protocol::function_stream_graph::ConnectorOp; use protocol::function_stream_graph::connector_op::Config; -use crate::runtime::streaming::api::operator::ConstructedOperator; -use crate::runtime::streaming::factory::connector::sink_props_codec::{ - apply_common_sink_fields, normalized_props, -}; -use crate::runtime::streaming::factory::global::Registry; -use crate::runtime::streaming::factory::operator_constructor::OperatorConstructor; -use crate::runtime::streaming::operators::sink::lancedb::LanceDbSinkOperator; use crate::sql::common::constants::connection_format_value; use crate::sql::common::with_option_keys as opt; +use crate::streaming::api::operator::ConstructedOperator; +use crate::streaming::factory::connector::sink_props_codec::{ + apply_common_sink_fields, normalized_props, +}; +use crate::streaming::factory::global::Registry; +use crate::streaming::factory::operator_constructor::OperatorConstructor; +use crate::streaming::operators::sink::lancedb::LanceDbSinkOperator; pub struct LanceDbSinkDispatcher; diff --git a/src/runtime/streaming/factory/connector/mod.rs b/src/streaming_runtime/src/streaming/factory/connector/mod.rs similarity index 100% rename from src/runtime/streaming/factory/connector/mod.rs rename to src/streaming_runtime/src/streaming/factory/connector/mod.rs diff --git a/src/runtime/streaming/factory/connector/s3.rs b/src/streaming_runtime/src/streaming/factory/connector/s3.rs similarity index 89% rename from src/runtime/streaming/factory/connector/s3.rs rename to src/streaming_runtime/src/streaming/factory/connector/s3.rs index 4b67fb9e..462b25d9 100644 --- a/src/runtime/streaming/factory/connector/s3.rs +++ b/src/streaming_runtime/src/streaming/factory/connector/s3.rs @@ -18,17 +18,15 @@ use prost::Message; use protocol::function_stream_graph::ConnectorOp; use protocol::function_stream_graph::connector_op::Config; -use crate::runtime::streaming::api::operator::ConstructedOperator; -use crate::runtime::streaming::factory::connector::sink_props_codec::{ - apply_common_sink_fields, normalized_props, -}; -use crate::runtime::streaming::factory::global::Registry; -use crate::runtime::streaming::factory::operator_constructor::OperatorConstructor; -use crate::runtime::streaming::operators::sink::s3::{ - S3Format, S3SinkOperator, compression_from_str, -}; use crate::sql::common::constants::connection_format_value; use crate::sql::common::with_option_keys as opt; +use crate::streaming::api::operator::ConstructedOperator; +use crate::streaming::factory::connector::sink_props_codec::{ + apply_common_sink_fields, normalized_props, +}; +use crate::streaming::factory::global::Registry; +use crate::streaming::factory::operator_constructor::OperatorConstructor; +use crate::streaming::operators::sink::s3::{S3Format, S3SinkOperator, compression_from_str}; pub struct S3SinkDispatcher; diff --git a/src/runtime/streaming/factory/connector/sink_props_codec.rs b/src/streaming_runtime/src/streaming/factory/connector/sink_props_codec.rs similarity index 100% rename from src/runtime/streaming/factory/connector/sink_props_codec.rs rename to src/streaming_runtime/src/streaming/factory/connector/sink_props_codec.rs diff --git a/src/runtime/streaming/factory/global/mod.rs b/src/streaming_runtime/src/streaming/factory/global/mod.rs similarity index 100% rename from src/runtime/streaming/factory/global/mod.rs rename to src/streaming_runtime/src/streaming/factory/global/mod.rs diff --git a/src/runtime/streaming/factory/global/session_registry.rs b/src/streaming_runtime/src/streaming/factory/global/session_registry.rs similarity index 100% rename from src/runtime/streaming/factory/global/session_registry.rs rename to src/streaming_runtime/src/streaming/factory/global/session_registry.rs diff --git a/src/runtime/streaming/factory/mod.rs b/src/streaming_runtime/src/streaming/factory/mod.rs similarity index 100% rename from src/runtime/streaming/factory/mod.rs rename to src/streaming_runtime/src/streaming/factory/mod.rs diff --git a/src/runtime/streaming/factory/operator_constructor.rs b/src/streaming_runtime/src/streaming/factory/operator_constructor.rs similarity index 86% rename from src/runtime/streaming/factory/operator_constructor.rs rename to src/streaming_runtime/src/streaming/factory/operator_constructor.rs index 5d0ff7d7..2cd96e6d 100644 --- a/src/runtime/streaming/factory/operator_constructor.rs +++ b/src/streaming_runtime/src/streaming/factory/operator_constructor.rs @@ -13,8 +13,8 @@ use anyhow::Result; use std::sync::Arc; -use crate::runtime::streaming::api::operator::ConstructedOperator; -use crate::runtime::streaming::factory::global::Registry; +use crate::streaming::api::operator::ConstructedOperator; +use crate::streaming::factory::global::Registry; /// Builds a [`ConstructedOperator`] from serialized configuration and a [`Registry`]. pub trait OperatorConstructor: Send + Sync { diff --git a/src/runtime/streaming/factory/operator_factory.rs b/src/streaming_runtime/src/streaming/factory/operator_factory.rs similarity index 93% rename from src/runtime/streaming/factory/operator_factory.rs rename to src/streaming_runtime/src/streaming/factory/operator_factory.rs index 1ce04eeb..10b9d3bf 100644 --- a/src/runtime/streaming/factory/operator_factory.rs +++ b/src/streaming_runtime/src/streaming/factory/operator_factory.rs @@ -11,27 +11,23 @@ // limitations under the License. use super::operator_constructor::OperatorConstructor; -use crate::runtime::streaming::api::operator::ConstructedOperator; -use crate::runtime::streaming::factory::connector::{ - ConnectorSinkDispatcher, ConnectorSourceDispatcher, -}; -use crate::runtime::streaming::factory::global::Registry; -use crate::runtime::streaming::operators::grouping::IncrementalAggregatingConstructor; -use crate::runtime::streaming::operators::joins::{ - InstantJoinConstructor, JoinWithExpirationConstructor, -}; +use crate::streaming::api::operator::ConstructedOperator; +use crate::streaming::factory::connector::{ConnectorSinkDispatcher, ConnectorSourceDispatcher}; +use crate::streaming::factory::global::Registry; +use crate::streaming::operators::grouping::IncrementalAggregatingConstructor; +use crate::streaming::operators::joins::{InstantJoinConstructor, JoinWithExpirationConstructor}; use anyhow::{Result, anyhow}; use prost::Message; use protocol::function_stream_graph::ProjectionOperator as ProjectionOperatorProto; use std::collections::HashMap; use std::sync::Arc; -use crate::runtime::streaming::operators::watermark::WatermarkGeneratorConstructor; -use crate::runtime::streaming::operators::windows::{ +use crate::streaming::operators::watermark::WatermarkGeneratorConstructor; +use crate::streaming::operators::windows::{ SessionAggregatingWindowConstructor, SlidingAggregatingWindowConstructor, TumblingAggregateWindowConstructor, WindowFunctionConstructor, }; -use crate::runtime::streaming::operators::{ +use crate::streaming::operators::{ KeyExecutionOperator, ProjectionOperator, StatelessPhysicalExecutor, ValueExecutionOperator, }; use protocol::function_stream_graph::{ @@ -125,7 +121,7 @@ impl OperatorFactory { ); self.register_named(OperatorName::ConnectorSink, Box::new(ConnectorSinkBridge)); - crate::runtime::streaming::factory::register_kafka_connector_plugins(self); + crate::streaming::factory::register_kafka_connector_plugins(self); } } diff --git a/src/runtime/streaming/format/config.rs b/src/streaming_runtime/src/streaming/format/config.rs similarity index 100% rename from src/runtime/streaming/format/config.rs rename to src/streaming_runtime/src/streaming/format/config.rs diff --git a/src/runtime/streaming/format/deserializer.rs b/src/streaming_runtime/src/streaming/format/deserializer.rs similarity index 100% rename from src/runtime/streaming/format/deserializer.rs rename to src/streaming_runtime/src/streaming/format/deserializer.rs diff --git a/src/runtime/streaming/format/encoder.rs b/src/streaming_runtime/src/streaming/format/encoder.rs similarity index 100% rename from src/runtime/streaming/format/encoder.rs rename to src/streaming_runtime/src/streaming/format/encoder.rs diff --git a/src/runtime/streaming/format/json_encoder.rs b/src/streaming_runtime/src/streaming/format/json_encoder.rs similarity index 100% rename from src/runtime/streaming/format/json_encoder.rs rename to src/streaming_runtime/src/streaming/format/json_encoder.rs diff --git a/src/runtime/streaming/format/mod.rs b/src/streaming_runtime/src/streaming/format/mod.rs similarity index 100% rename from src/runtime/streaming/format/mod.rs rename to src/streaming_runtime/src/streaming/format/mod.rs diff --git a/src/runtime/streaming/format/serializer.rs b/src/streaming_runtime/src/streaming/format/serializer.rs similarity index 100% rename from src/runtime/streaming/format/serializer.rs rename to src/streaming_runtime/src/streaming/format/serializer.rs diff --git a/src/runtime/streaming/job/edge_manager.rs b/src/streaming_runtime/src/streaming/job/edge_manager.rs similarity index 98% rename from src/runtime/streaming/job/edge_manager.rs rename to src/streaming_runtime/src/streaming/job/edge_manager.rs index 00c94485..32bb3722 100644 --- a/src/runtime/streaming/job/edge_manager.rs +++ b/src/streaming_runtime/src/streaming/job/edge_manager.rs @@ -16,7 +16,7 @@ use anyhow::{Result, anyhow}; use tokio::sync::mpsc; use tracing::{debug, info, warn}; -use crate::runtime::streaming::protocol::event::TrackedEvent; +use crate::streaming::protocol::event::TrackedEvent; use protocol::function_stream_graph::{FsEdge, FsNode}; const DEFAULT_CHANNEL_CAPACITY: usize = 2048; diff --git a/src/runtime/streaming/job/job_manager.rs b/src/streaming_runtime/src/streaming/job/job_manager.rs similarity index 96% rename from src/runtime/streaming/job/job_manager.rs rename to src/streaming_runtime/src/streaming/job/job_manager.rs index e4b9916b..c4c0a826 100644 --- a/src/runtime/streaming/job/job_manager.rs +++ b/src/streaming_runtime/src/streaming/job/job_manager.rs @@ -30,22 +30,22 @@ use crate::config::{ DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_OPERATOR_STATE_STORE_MEMORY_BYTES, DEFAULT_PIPELINE_PARALLELISM, }; -use crate::runtime::memory::global_memory_pool; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{ConstructedOperator, Operator}; -use crate::runtime::streaming::api::source::SourceOperator; -use crate::runtime::streaming::execution::{ChainBuilder, Pipeline, SourceDriver}; -use crate::runtime::streaming::factory::OperatorFactory; -use crate::runtime::streaming::job::edge_manager::EdgeManager; -use crate::runtime::streaming::job::models::{ +use crate::memory::global_memory_pool; +use crate::sql::logical_node::logical::OperatorName; +use crate::stream_catalog::CatalogManager; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::operator::{ConstructedOperator, Operator}; +use crate::streaming::api::source::SourceOperator; +use crate::streaming::execution::{ChainBuilder, Pipeline, SourceDriver}; +use crate::streaming::factory::OperatorFactory; +use crate::streaming::job::edge_manager::EdgeManager; +use crate::streaming::job::models::{ PhysicalExecutionGraph, PhysicalPipeline, PipelineStatus, StreamingJobRollupStatus, }; -use crate::runtime::streaming::network::endpoint::{BoxedEventStream, PhysicalSender}; -use crate::runtime::streaming::protocol::control::{ControlCommand, JobMasterEvent, StopMode}; -use crate::runtime::streaming::protocol::event::CheckpointBarrier; -use crate::runtime::streaming::state::{IoManager, IoPool, NoopMetricsCollector}; -use crate::sql::logical_node::logical::OperatorName; -use crate::storage::stream_catalog::CatalogManager; +use crate::streaming::network::endpoint::{BoxedEventStream, PhysicalSender}; +use crate::streaming::protocol::control::{ControlCommand, JobMasterEvent, StopMode}; +use crate::streaming::protocol::event::CheckpointBarrier; +use crate::streaming::state::{IoManager, IoPool, NoopMetricsCollector}; #[derive(Debug, Clone)] pub struct StreamingJobSummary { @@ -80,7 +80,7 @@ pub struct StateConfig { pub pipeline_parallelism: u32, pub job_manager_control_plane_threads: u32, pub job_manager_data_plane_threads: u32, - /// Total bytes shared by all [`crate::runtime::streaming::state::OperatorStateStore`] (global pool). + /// Total bytes shared by all [`crate::streaming::state::OperatorStateStore`] (global pool). pub per_operator_memory_bytes: u64, } @@ -103,7 +103,7 @@ impl Default for StateConfig { static GLOBAL_JOB_MANAGER: OnceLock> = OnceLock::new(); -/// Operators that create an [`crate::runtime::streaming::state::OperatorStateStore`] at runtime. +/// Operators that create an [`crate::streaming::state::OperatorStateStore`] at runtime. fn pipeline_state_store_operator_count(operators: &[ChainedOperator]) -> usize { operators .iter() @@ -159,7 +159,7 @@ struct CheckpointCoordinatorConfig { } impl PipelineRunner { - async fn run(self) -> Result<(), crate::runtime::streaming::error::RunError> { + async fn run(self) -> Result<(), crate::streaming::error::RunError> { match self { PipelineRunner::Source(driver) => driver.run().await, PipelineRunner::Standard(pipeline) => pipeline.run().await, diff --git a/src/runtime/streaming/job/mod.rs b/src/streaming_runtime/src/streaming/job/mod.rs similarity index 100% rename from src/runtime/streaming/job/mod.rs rename to src/streaming_runtime/src/streaming/job/mod.rs diff --git a/src/runtime/streaming/job/models.rs b/src/streaming_runtime/src/streaming/job/models.rs similarity index 98% rename from src/runtime/streaming/job/models.rs rename to src/streaming_runtime/src/streaming/job/models.rs index e81649f2..a6d226bd 100644 --- a/src/runtime/streaming/job/models.rs +++ b/src/streaming_runtime/src/streaming/job/models.rs @@ -19,7 +19,7 @@ use protocol::function_stream_graph::FsProgram; use tokio::sync::mpsc; use tokio::task::JoinHandle; -use crate::runtime::streaming::protocol::control::ControlCommand; +use crate::streaming::protocol::control::ControlCommand; #[derive(Debug, Clone, PartialEq)] pub enum PipelineStatus { diff --git a/src/runtime/streaming/mod.rs b/src/streaming_runtime/src/streaming/mod.rs similarity index 100% rename from src/runtime/streaming/mod.rs rename to src/streaming_runtime/src/streaming/mod.rs diff --git a/src/runtime/streaming/network/endpoint.rs b/src/streaming_runtime/src/streaming/network/endpoint.rs similarity index 94% rename from src/runtime/streaming/network/endpoint.rs rename to src/streaming_runtime/src/streaming/network/endpoint.rs index ae75e6fc..12550977 100644 --- a/src/runtime/streaming/network/endpoint.rs +++ b/src/streaming_runtime/src/streaming/network/endpoint.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::streaming::protocol::event::StreamEvent; -use crate::runtime::streaming::protocol::event::TrackedEvent; +use crate::streaming::protocol::event::StreamEvent; +use crate::streaming::protocol::event::TrackedEvent; use anyhow::{Result, anyhow}; use std::pin::Pin; use tokio::sync::mpsc; diff --git a/src/runtime/streaming/network/environment.rs b/src/streaming_runtime/src/streaming/network/environment.rs similarity index 100% rename from src/runtime/streaming/network/environment.rs rename to src/streaming_runtime/src/streaming/network/environment.rs diff --git a/src/runtime/streaming/network/mod.rs b/src/streaming_runtime/src/streaming/network/mod.rs similarity index 100% rename from src/runtime/streaming/network/mod.rs rename to src/streaming_runtime/src/streaming/network/mod.rs diff --git a/src/runtime/streaming/operators/grouping/incremental_aggregate.rs b/src/streaming_runtime/src/streaming/operators/grouping/incremental_aggregate.rs similarity index 99% rename from src/runtime/streaming/operators/grouping/incremental_aggregate.rs rename to src/streaming_runtime/src/streaming/operators/grouping/incremental_aggregate.rs index ff997c11..55ec2006 100644 --- a/src/runtime/streaming/operators/grouping/incremental_aggregate.rs +++ b/src/streaming_runtime/src/streaming/operators/grouping/incremental_aggregate.rs @@ -41,17 +41,17 @@ use std::{collections::HashMap, mem, sync::Arc}; use tracing::{debug, info, warn}; // ========================================================================= // ========================================================================= -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::factory::Registry; -use crate::runtime::streaming::operators::{Key, UpdatingCache}; -use crate::runtime::streaming::state::OperatorStateStore; -use crate::runtime::util::decode_aggregate; use crate::sql::common::{ CheckpointBarrier, FsSchema, TIMESTAMP_FIELD, UPDATING_META_FIELD, Watermark, to_nanos, }; use crate::sql::physical::updating_meta_fields; +use crate::streaming::StreamOutput; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::operator::{Collector, Operator}; +use crate::streaming::factory::Registry; +use crate::streaming::operators::{Key, UpdatingCache}; +use crate::streaming::state::OperatorStateStore; +use crate::util::decode_aggregate; #[derive(Debug, Copy, Clone)] struct BatchData { diff --git a/src/runtime/streaming/operators/grouping/mod.rs b/src/streaming_runtime/src/streaming/operators/grouping/mod.rs similarity index 100% rename from src/runtime/streaming/operators/grouping/mod.rs rename to src/streaming_runtime/src/streaming/operators/grouping/mod.rs diff --git a/src/runtime/streaming/operators/grouping/updating_cache.rs b/src/streaming_runtime/src/streaming/operators/grouping/updating_cache.rs similarity index 100% rename from src/runtime/streaming/operators/grouping/updating_cache.rs rename to src/streaming_runtime/src/streaming/operators/grouping/updating_cache.rs diff --git a/src/runtime/streaming/operators/joins/join_instance.rs b/src/streaming_runtime/src/streaming/operators/joins/join_instance.rs similarity index 98% rename from src/runtime/streaming/operators/joins/join_instance.rs rename to src/streaming_runtime/src/streaming/operators/joins/join_instance.rs index 5c04c1fb..232c2985 100644 --- a/src/runtime/streaming/operators/joins/join_instance.rs +++ b/src/streaming_runtime/src/streaming/operators/joins/join_instance.rs @@ -25,13 +25,13 @@ use std::sync::{Arc, RwLock}; use std::time::UNIX_EPOCH; use tracing::{info, warn}; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::factory::Registry; -use crate::runtime::streaming::state::OperatorStateStore; use crate::sql::common::{CheckpointBarrier, FsSchema, FsSchemaRef, Watermark}; use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; +use crate::streaming::StreamOutput; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::operator::{Collector, Operator}; +use crate::streaming::factory::Registry; +use crate::streaming::state::OperatorStateStore; use async_trait::async_trait; use protocol::function_stream_graph::JoinOperator; diff --git a/src/runtime/streaming/operators/joins/join_with_expiration.rs b/src/streaming_runtime/src/streaming/operators/joins/join_with_expiration.rs similarity index 97% rename from src/runtime/streaming/operators/joins/join_with_expiration.rs rename to src/streaming_runtime/src/streaming/operators/joins/join_with_expiration.rs index 5ed8dfa3..139e88cd 100644 --- a/src/runtime/streaming/operators/joins/join_with_expiration.rs +++ b/src/streaming_runtime/src/streaming/operators/joins/join_with_expiration.rs @@ -24,13 +24,13 @@ use std::sync::{Arc, RwLock}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tracing::{info, warn}; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::factory::Registry; -use crate::runtime::streaming::state::OperatorStateStore; use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark}; use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; +use crate::streaming::StreamOutput; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::operator::{Collector, Operator}; +use crate::streaming::factory::Registry; +use crate::streaming::state::OperatorStateStore; use async_trait::async_trait; use protocol::function_stream_graph::JoinOperator; diff --git a/src/runtime/streaming/operators/joins/mod.rs b/src/streaming_runtime/src/streaming/operators/joins/mod.rs similarity index 100% rename from src/runtime/streaming/operators/joins/mod.rs rename to src/streaming_runtime/src/streaming/operators/joins/mod.rs diff --git a/src/runtime/streaming/operators/key_by.rs b/src/streaming_runtime/src/streaming/operators/key_by.rs similarity index 96% rename from src/runtime/streaming/operators/key_by.rs rename to src/streaming_runtime/src/streaming/operators/key_by.rs index 90c55d08..f1e2831d 100644 --- a/src/runtime/streaming/operators/key_by.rs +++ b/src/streaming_runtime/src/streaming/operators/key_by.rs @@ -19,10 +19,10 @@ use datafusion_common::hash_utils::create_hashes; use datafusion_physical_expr::expressions::Column; use std::sync::Arc; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; use crate::sql::common::{CheckpointBarrier, Watermark}; +use crate::streaming::StreamOutput; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::operator::{Collector, Operator}; use protocol::function_stream_graph::KeyPlanOperator; diff --git a/src/runtime/streaming/operators/key_operator.rs b/src/streaming_runtime/src/streaming/operators/key_operator.rs similarity index 94% rename from src/runtime/streaming/operators/key_operator.rs rename to src/streaming_runtime/src/streaming/operators/key_operator.rs index 7a89d2f2..a8064d2e 100644 --- a/src/runtime/streaming/operators/key_operator.rs +++ b/src/streaming_runtime/src/streaming/operators/key_operator.rs @@ -13,13 +13,13 @@ //! Key-by over the physical plan output: key column(s) are **values** projected by the plan //! (e.g. `_key_user_id`); **shuffle / `StreamOutput::Keyed` uses `u64` hashes** computed by //! [`datafusion_common::hash_utils::create_hashes`] on those columns — same mechanism as -//! [`crate::runtime::streaming::operators::key_by::KeyByOperator`]. +//! [`crate::streaming::operators::key_by::KeyByOperator`]. -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::operators::StatelessPhysicalExecutor; use crate::sql::common::{CheckpointBarrier, Watermark}; +use crate::streaming::StreamOutput; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::operator::{Collector, Operator}; +use crate::streaming::operators::StatelessPhysicalExecutor; use ahash::RandomState; use anyhow::{Result, anyhow}; use arrow::compute::{sort_to_indices, take}; diff --git a/src/runtime/streaming/operators/mod.rs b/src/streaming_runtime/src/streaming/operators/mod.rs similarity index 100% rename from src/runtime/streaming/operators/mod.rs rename to src/streaming_runtime/src/streaming/operators/mod.rs diff --git a/src/runtime/streaming/operators/projection.rs b/src/streaming_runtime/src/streaming/operators/projection.rs similarity index 94% rename from src/runtime/streaming/operators/projection.rs rename to src/streaming_runtime/src/streaming/operators/projection.rs index b84d74aa..54882547 100644 --- a/src/runtime/streaming/operators/projection.rs +++ b/src/streaming_runtime/src/streaming/operators/projection.rs @@ -22,12 +22,12 @@ use std::sync::Arc; use protocol::function_stream_graph::ProjectionOperator as ProjectionOperatorProto; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::factory::global::Registry; use crate::sql::common::{CheckpointBarrier, FsSchema, FsSchemaRef, Watermark}; use crate::sql::logical_node::logical::OperatorName; +use crate::streaming::StreamOutput; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::operator::{Collector, Operator}; +use crate::streaming::factory::global::Registry; pub struct ProjectionOperator { name: String, diff --git a/src/runtime/streaming/operators/sink/delta/mod.rs b/src/streaming_runtime/src/streaming/operators/sink/delta/mod.rs similarity index 96% rename from src/runtime/streaming/operators/sink/delta/mod.rs rename to src/streaming_runtime/src/streaming/operators/sink/delta/mod.rs index 4df6b3b5..b2d16c60 100644 --- a/src/runtime/streaming/operators/sink/delta/mod.rs +++ b/src/streaming_runtime/src/streaming/operators/sink/delta/mod.rs @@ -26,14 +26,14 @@ use parquet::basic::Compression; use tokio::io::AsyncWriteExt; use tracing::{debug, info, warn}; -use crate::runtime::memory::{MemoryBlock, try_global_memory_pool}; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::format::encoder::FormatEncoder; +use crate::memory::{MemoryBlock, try_global_memory_pool}; use crate::sql::common::constants::factory_operator_name; use crate::sql::common::with_option_keys as opt; use crate::sql::common::{CheckpointBarrier, Watermark}; +use crate::streaming::StreamOutput; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::operator::{Collector, Operator}; +use crate::streaming::format::encoder::FormatEncoder; /// Flush early when buffered batches exceed this size. const DEFAULT_MAX_BUFFER_BYTES: usize = 64 * 1024 * 1024; diff --git a/src/runtime/streaming/operators/sink/filesystem/mod.rs b/src/streaming_runtime/src/streaming/operators/sink/filesystem/mod.rs similarity index 96% rename from src/runtime/streaming/operators/sink/filesystem/mod.rs rename to src/streaming_runtime/src/streaming/operators/sink/filesystem/mod.rs index a865a752..3461c4a4 100644 --- a/src/runtime/streaming/operators/sink/filesystem/mod.rs +++ b/src/streaming_runtime/src/streaming/operators/sink/filesystem/mod.rs @@ -20,13 +20,13 @@ use parquet::basic::Compression; use tokio::io::AsyncWriteExt; use tracing::{debug, info, warn}; -use crate::runtime::memory::{MemoryBlock, try_global_memory_pool}; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::format::encoder::FormatEncoder; +use crate::memory::{MemoryBlock, try_global_memory_pool}; use crate::sql::common::constants::factory_operator_name; use crate::sql::common::{CheckpointBarrier, Watermark}; +use crate::streaming::StreamOutput; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::operator::{Collector, Operator}; +use crate::streaming::format::encoder::FormatEncoder; const DEFAULT_MAX_BUFFER_BYTES: usize = 64 * 1024 * 1024; diff --git a/src/runtime/streaming/operators/sink/iceberg/mod.rs b/src/streaming_runtime/src/streaming/operators/sink/iceberg/mod.rs similarity index 96% rename from src/runtime/streaming/operators/sink/iceberg/mod.rs rename to src/streaming_runtime/src/streaming/operators/sink/iceberg/mod.rs index b6c17414..51715aa2 100644 --- a/src/runtime/streaming/operators/sink/iceberg/mod.rs +++ b/src/streaming_runtime/src/streaming/operators/sink/iceberg/mod.rs @@ -26,14 +26,14 @@ use parquet::basic::Compression; use tokio::io::AsyncWriteExt; use tracing::{debug, info, warn}; -use crate::runtime::memory::{MemoryBlock, try_global_memory_pool}; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::format::encoder::FormatEncoder; +use crate::memory::{MemoryBlock, try_global_memory_pool}; use crate::sql::common::constants::factory_operator_name; use crate::sql::common::with_option_keys as opt; use crate::sql::common::{CheckpointBarrier, Watermark}; +use crate::streaming::StreamOutput; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::operator::{Collector, Operator}; +use crate::streaming::format::encoder::FormatEncoder; const DEFAULT_MAX_BUFFER_BYTES: usize = 64 * 1024 * 1024; diff --git a/src/runtime/streaming/operators/sink/kafka/mod.rs b/src/streaming_runtime/src/streaming/operators/sink/kafka/mod.rs similarity index 98% rename from src/runtime/streaming/operators/sink/kafka/mod.rs rename to src/streaming_runtime/src/streaming/operators/sink/kafka/mod.rs index b30bc572..1266ebdd 100644 --- a/src/runtime/streaming/operators/sink/kafka/mod.rs +++ b/src/streaming_runtime/src/streaming/operators/sink/kafka/mod.rs @@ -36,12 +36,12 @@ use std::time::Duration; use tokio::time::sleep; use tracing::{info, warn}; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::format::DataSerializer; use crate::sql::common::constants::factory_operator_name; use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark}; +use crate::streaming::StreamOutput; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::operator::{Collector, Operator}; +use crate::streaming::format::DataSerializer; // ============================================================================ // ============================================================================ diff --git a/src/runtime/streaming/operators/sink/lancedb/mod.rs b/src/streaming_runtime/src/streaming/operators/sink/lancedb/mod.rs similarity index 96% rename from src/runtime/streaming/operators/sink/lancedb/mod.rs rename to src/streaming_runtime/src/streaming/operators/sink/lancedb/mod.rs index 4e8d8309..e69a59ee 100644 --- a/src/runtime/streaming/operators/sink/lancedb/mod.rs +++ b/src/streaming_runtime/src/streaming/operators/sink/lancedb/mod.rs @@ -23,11 +23,11 @@ use lance::dataset::{WriteMode, WriteParams}; use lance::io::{ObjectStoreParams, StorageOptionsAccessor}; use tracing::{info, warn}; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; use crate::sql::common::constants::factory_operator_name; use crate::sql::common::{CheckpointBarrier, Watermark}; +use crate::streaming::StreamOutput; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::operator::{Collector, Operator}; pub struct LanceDbSinkOperator { table_name: String, diff --git a/src/runtime/streaming/operators/sink/mod.rs b/src/streaming_runtime/src/streaming/operators/sink/mod.rs similarity index 100% rename from src/runtime/streaming/operators/sink/mod.rs rename to src/streaming_runtime/src/streaming/operators/sink/mod.rs diff --git a/src/runtime/streaming/operators/sink/s3/mod.rs b/src/streaming_runtime/src/streaming/operators/sink/s3/mod.rs similarity index 97% rename from src/runtime/streaming/operators/sink/s3/mod.rs rename to src/streaming_runtime/src/streaming/operators/sink/s3/mod.rs index 715b5b86..f374b95c 100644 --- a/src/runtime/streaming/operators/sink/s3/mod.rs +++ b/src/streaming_runtime/src/streaming/operators/sink/s3/mod.rs @@ -26,12 +26,12 @@ use parquet::basic::Compression; use parquet::file::properties::WriterProperties; use tracing::{info, warn}; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; use crate::sql::common::constants::factory_operator_name; use crate::sql::common::with_option_keys as opt; use crate::sql::common::{CheckpointBarrier, Watermark}; +use crate::streaming::StreamOutput; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::operator::{Collector, Operator}; #[derive(Debug, Clone, Copy)] pub enum S3Format { diff --git a/src/runtime/streaming/operators/source/kafka/mod.rs b/src/streaming_runtime/src/streaming/operators/source/kafka/mod.rs similarity index 98% rename from src/runtime/streaming/operators/source/kafka/mod.rs rename to src/streaming_runtime/src/streaming/operators/source/kafka/mod.rs index 75edb968..7f5e09d4 100644 --- a/src/runtime/streaming/operators/source/kafka/mod.rs +++ b/src/streaming_runtime/src/streaming/operators/source/kafka/mod.rs @@ -31,13 +31,13 @@ use std::num::NonZeroU32; use std::time::{Duration, Instant}; use tracing::{debug, error, info, warn}; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::source::{ - SourceCheckpointReport, SourceEvent, SourceOffset, SourceOperator, -}; -use crate::runtime::streaming::format::{BadDataPolicy, DataDeserializer, Format}; use crate::sql::common::fs_schema::FieldValueType; use crate::sql::common::{CheckpointBarrier, MetadataField}; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::source::{ + SourceCheckpointReport, SourceEvent, SourceOffset, SourceOperator, +}; +use crate::streaming::format::{BadDataPolicy, DataDeserializer, Format}; pub trait BatchDeserializer: Send + 'static { fn deserialize_slice( diff --git a/src/runtime/wasm/input/protocol/mod.rs b/src/streaming_runtime/src/streaming/operators/source/mod.rs similarity index 100% rename from src/runtime/wasm/input/protocol/mod.rs rename to src/streaming_runtime/src/streaming/operators/source/mod.rs diff --git a/src/runtime/streaming/operators/stateless_physical_executor.rs b/src/streaming_runtime/src/streaming/operators/stateless_physical_executor.rs similarity index 98% rename from src/runtime/streaming/operators/stateless_physical_executor.rs rename to src/streaming_runtime/src/streaming/operators/stateless_physical_executor.rs index eb595d31..560ae8c5 100644 --- a/src/runtime/streaming/operators/stateless_physical_executor.rs +++ b/src/streaming_runtime/src/streaming/operators/stateless_physical_executor.rs @@ -24,8 +24,8 @@ use datafusion_proto::protobuf::PhysicalPlanNode; use futures::StreamExt; use prost::Message; -use crate::runtime::streaming::factory::Registry; use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; +use crate::streaming::factory::Registry; pub struct StatelessPhysicalExecutor { batch: Arc>>, diff --git a/src/runtime/streaming/operators/value_execution.rs b/src/streaming_runtime/src/streaming/operators/value_execution.rs similarity index 89% rename from src/runtime/streaming/operators/value_execution.rs rename to src/streaming_runtime/src/streaming/operators/value_execution.rs index b93cd78b..d971668f 100644 --- a/src/runtime/streaming/operators/value_execution.rs +++ b/src/streaming_runtime/src/streaming/operators/value_execution.rs @@ -15,11 +15,11 @@ use arrow_array::RecordBatch; use async_trait::async_trait; use futures::StreamExt; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::operators::StatelessPhysicalExecutor; use crate::sql::common::{CheckpointBarrier, Watermark}; +use crate::streaming::StreamOutput; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::operator::{Collector, Operator}; +use crate::streaming::operators::StatelessPhysicalExecutor; pub struct ValueExecutionOperator { name: String, diff --git a/src/runtime/streaming/operators/watermark/mod.rs b/src/streaming_runtime/src/streaming/operators/watermark/mod.rs similarity index 100% rename from src/runtime/streaming/operators/watermark/mod.rs rename to src/streaming_runtime/src/streaming/operators/watermark/mod.rs diff --git a/src/runtime/streaming/operators/watermark/watermark_generator.rs b/src/streaming_runtime/src/streaming/operators/watermark/watermark_generator.rs similarity index 96% rename from src/runtime/streaming/operators/watermark/watermark_generator.rs rename to src/streaming_runtime/src/streaming/operators/watermark/watermark_generator.rs index 497553eb..ea70b41b 100644 --- a/src/runtime/streaming/operators/watermark/watermark_generator.rs +++ b/src/streaming_runtime/src/streaming/operators/watermark/watermark_generator.rs @@ -25,11 +25,11 @@ use std::sync::Arc; use std::time::{Duration, SystemTime}; use tracing::debug; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::factory::Registry; use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark, from_nanos, to_millis}; +use crate::streaming::StreamOutput; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::operator::{Collector, Operator}; +use crate::streaming::factory::Registry; use async_trait::async_trait; use protocol::function_stream_graph::ExpressionWatermarkConfig; diff --git a/src/runtime/streaming/operators/windows/mod.rs b/src/streaming_runtime/src/streaming/operators/windows/mod.rs similarity index 100% rename from src/runtime/streaming/operators/windows/mod.rs rename to src/streaming_runtime/src/streaming/operators/windows/mod.rs diff --git a/src/runtime/streaming/operators/windows/session_aggregating_window.rs b/src/streaming_runtime/src/streaming/operators/windows/session_aggregating_window.rs similarity index 99% rename from src/runtime/streaming/operators/windows/session_aggregating_window.rs rename to src/streaming_runtime/src/streaming/operators/windows/session_aggregating_window.rs index 2056cdd9..b42fcc13 100644 --- a/src/runtime/streaming/operators/windows/session_aggregating_window.rs +++ b/src/streaming_runtime/src/streaming/operators/windows/session_aggregating_window.rs @@ -36,17 +36,17 @@ use std::time::{Duration, SystemTime}; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; use tracing::info; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::factory::Registry; -use crate::runtime::streaming::state::OperatorStateStore; use crate::sql::common::converter::Converter; use crate::sql::common::{ CheckpointBarrier, FsSchema, FsSchemaRef, Watermark, from_nanos, to_nanos, }; use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; use crate::sql::schema::utils::window_arrow_struct; +use crate::streaming::StreamOutput; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::operator::{Collector, Operator}; +use crate::streaming::factory::Registry; +use crate::streaming::state::OperatorStateStore; use async_trait::async_trait; use protocol::function_stream_graph::SessionWindowAggregateOperator; // ============================================================================ diff --git a/src/runtime/streaming/operators/windows/sliding_aggregating_window.rs b/src/streaming_runtime/src/streaming/operators/windows/sliding_aggregating_window.rs similarity index 98% rename from src/runtime/streaming/operators/windows/sliding_aggregating_window.rs rename to src/streaming_runtime/src/streaming/operators/windows/sliding_aggregating_window.rs index f18b3b14..b9c57835 100644 --- a/src/runtime/streaming/operators/windows/sliding_aggregating_window.rs +++ b/src/streaming_runtime/src/streaming/operators/windows/sliding_aggregating_window.rs @@ -33,13 +33,13 @@ use std::time::{Duration, SystemTime}; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; use tracing::info; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::factory::Registry; -use crate::runtime::streaming::state::OperatorStateStore; use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark, from_nanos, to_nanos}; use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; +use crate::streaming::StreamOutput; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::operator::{Collector, Operator}; +use crate::streaming::factory::Registry; +use crate::streaming::state::OperatorStateStore; use async_trait::async_trait; use protocol::function_stream_graph::SlidingWindowAggregateOperator; // ============================================================================ diff --git a/src/runtime/streaming/operators/windows/tumbling_aggregating_window.rs b/src/streaming_runtime/src/streaming/operators/windows/tumbling_aggregating_window.rs similarity index 98% rename from src/runtime/streaming/operators/windows/tumbling_aggregating_window.rs rename to src/streaming_runtime/src/streaming/operators/windows/tumbling_aggregating_window.rs index 5c805625..7d45a684 100644 --- a/src/runtime/streaming/operators/windows/tumbling_aggregating_window.rs +++ b/src/streaming_runtime/src/streaming/operators/windows/tumbling_aggregating_window.rs @@ -34,15 +34,15 @@ use std::time::{Duration, SystemTime}; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; use tracing::{info, warn}; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::factory::Registry; -use crate::runtime::streaming::state::OperatorStateStore; use crate::sql::common::time_utils::print_time; use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark, from_nanos, to_nanos}; use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; use crate::sql::schema::utils::add_timestamp_field_arrow; +use crate::streaming::StreamOutput; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::operator::{Collector, Operator}; +use crate::streaming::factory::Registry; +use crate::streaming::state::OperatorStateStore; use async_trait::async_trait; use protocol::function_stream_graph::TumblingWindowAggregateOperator; diff --git a/src/runtime/streaming/operators/windows/window_function.rs b/src/streaming_runtime/src/streaming/operators/windows/window_function.rs similarity index 97% rename from src/runtime/streaming/operators/windows/window_function.rs rename to src/streaming_runtime/src/streaming/operators/windows/window_function.rs index 37815b78..83f14551 100644 --- a/src/runtime/streaming/operators/windows/window_function.rs +++ b/src/streaming_runtime/src/streaming/operators/windows/window_function.rs @@ -26,16 +26,16 @@ use std::time::SystemTime; use tokio::sync::mpsc::{UnboundedReceiver, unbounded_channel}; use tracing::{info, warn}; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::factory::Registry; -use crate::runtime::streaming::state::OperatorStateStore; use crate::sql::common::time_utils::print_time; use crate::sql::common::{ CheckpointBarrier, FsSchema, FsSchemaRef, Watermark, from_nanos, to_nanos, }; use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; +use crate::streaming::StreamOutput; +use crate::streaming::api::context::TaskContext; +use crate::streaming::api::operator::{Collector, Operator}; +use crate::streaming::factory::Registry; +use crate::streaming::state::OperatorStateStore; use async_trait::async_trait; // ============================================================================ diff --git a/src/runtime/streaming/protocol/control.rs b/src/streaming_runtime/src/streaming/protocol/control.rs similarity index 100% rename from src/runtime/streaming/protocol/control.rs rename to src/streaming_runtime/src/streaming/protocol/control.rs diff --git a/src/streaming_runtime/src/streaming/protocol/event.rs b/src/streaming_runtime/src/streaming/protocol/event.rs new file mode 100644 index 00000000..0db2135d --- /dev/null +++ b/src/streaming_runtime/src/streaming/protocol/event.rs @@ -0,0 +1,59 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use arrow_array::RecordBatch; + +use crate::memory::MemoryTicket; + +pub use function_stream_runtime_common::streaming_protocol::{ + CheckpointBarrier, Watermark, merge_watermarks, watermark_strictly_advances, +}; + +#[derive(Debug, Clone)] +pub enum StreamEvent { + Data(RecordBatch), + Watermark(Watermark), + Barrier(CheckpointBarrier), + EndOfStream, +} + +#[derive(Debug, Clone)] +pub enum StreamOutput { + Forward(RecordBatch), + Keyed(u64, RecordBatch), + Broadcast(RecordBatch), + Watermark(Watermark), +} + +#[derive(Debug, Clone)] +pub struct TrackedEvent { + pub event: StreamEvent, + pub _ticket: Option>, +} + +impl TrackedEvent { + pub fn new(event: StreamEvent, ticket: Option) -> Self { + Self { + event, + _ticket: ticket.map(Arc::new), + } + } + + pub fn control(event: StreamEvent) -> Self { + Self { + event, + _ticket: None, + } + } +} diff --git a/src/runtime/streaming/protocol/mod.rs b/src/streaming_runtime/src/streaming/protocol/mod.rs similarity index 96% rename from src/runtime/streaming/protocol/mod.rs rename to src/streaming_runtime/src/streaming/protocol/mod.rs index 28fd85a4..d41d9fcf 100644 --- a/src/runtime/streaming/protocol/mod.rs +++ b/src/streaming_runtime/src/streaming/protocol/mod.rs @@ -15,4 +15,5 @@ pub mod event; #[allow(unused_imports)] pub use control::{ControlCommand, JobMasterEvent, StopMode}; +#[allow(unused_imports)] pub use event::{CheckpointBarrier, StreamOutput, Watermark}; diff --git a/src/runtime/streaming/state/error.rs b/src/streaming_runtime/src/streaming/state/error.rs similarity index 97% rename from src/runtime/streaming/state/error.rs rename to src/streaming_runtime/src/streaming/state/error.rs index 37bc6481..cab033db 100644 --- a/src/runtime/streaming/state/error.rs +++ b/src/streaming_runtime/src/streaming/state/error.rs @@ -13,7 +13,7 @@ use crossbeam_channel::TrySendError; use thiserror::Error; -use crate::runtime::memory::MemoryAllocationError; +use crate::memory::MemoryAllocationError; #[derive(Error, Debug)] pub enum StateEngineError { diff --git a/src/runtime/streaming/state/io_manager.rs b/src/streaming_runtime/src/streaming/state/io_manager.rs similarity index 100% rename from src/runtime/streaming/state/io_manager.rs rename to src/streaming_runtime/src/streaming/state/io_manager.rs diff --git a/src/runtime/streaming/state/metrics.rs b/src/streaming_runtime/src/streaming/state/metrics.rs similarity index 100% rename from src/runtime/streaming/state/metrics.rs rename to src/streaming_runtime/src/streaming/state/metrics.rs diff --git a/src/runtime/streaming/state/mod.rs b/src/streaming_runtime/src/streaming/state/mod.rs similarity index 100% rename from src/runtime/streaming/state/mod.rs rename to src/streaming_runtime/src/streaming/state/mod.rs diff --git a/src/runtime/streaming/state/operator_state.rs b/src/streaming_runtime/src/streaming/state/operator_state.rs similarity index 99% rename from src/runtime/streaming/state/operator_state.rs rename to src/streaming_runtime/src/streaming/state/operator_state.rs index a3514461..224671f2 100644 --- a/src/runtime/streaming/state/operator_state.rs +++ b/src/streaming_runtime/src/streaming/state/operator_state.rs @@ -13,7 +13,7 @@ use super::error::{Result, StateEngineError}; use super::io_manager::{CompactJob, IoManager, SpillJob}; use super::metrics::StateMetricsCollector; -use crate::runtime::memory::{MemoryBlock, MemoryTicket}; +use crate::memory::{MemoryBlock, MemoryTicket}; use arrow_array::builder::{BinaryBuilder, BooleanBuilder, UInt64Builder}; use arrow_array::{Array, BinaryArray, RecordBatch, UInt64Array}; use arrow_schema::{DataType, Field, Schema}; @@ -813,7 +813,7 @@ mod tests { use super::super::io_manager::IoPool; use super::super::metrics::NoopMetricsCollector; use super::*; - use crate::runtime::memory::{MemoryBlock, MemoryPool, global_memory_pool}; + use crate::memory::{MemoryBlock, MemoryPool, global_memory_pool}; use arrow_array::Int64Array; use tempfile::TempDir; @@ -836,7 +836,7 @@ mod tests { const TEST_OPERATOR_MEMORY: u64 = 2 * 1024 * 1024; fn ensure_global_memory_pool() { - use crate::runtime::memory::{init_global_memory_pool, try_global_memory_pool}; + use crate::memory::{init_global_memory_pool, try_global_memory_pool}; use std::sync::Once; static INIT: Once = Once::new(); INIT.call_once(|| { diff --git a/src/runtime/util/mod.rs b/src/streaming_runtime/src/util/mod.rs similarity index 100% rename from src/runtime/util/mod.rs rename to src/streaming_runtime/src/util/mod.rs diff --git a/src/runtime/util/physical_aggregate.rs b/src/streaming_runtime/src/util/physical_aggregate.rs similarity index 100% rename from src/runtime/util/physical_aggregate.rs rename to src/streaming_runtime/src/util/physical_aggregate.rs diff --git a/src/wasm_runtime/Cargo.toml b/src/wasm_runtime/Cargo.toml new file mode 100644 index 00000000..2f385240 --- /dev/null +++ b/src/wasm_runtime/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "function-stream-wasm-runtime" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_wasm_runtime" +path = "src/lib.rs" diff --git a/src/wasm_runtime/src/lib.rs b/src/wasm_runtime/src/lib.rs new file mode 100644 index 00000000..12b3ebee --- /dev/null +++ b/src/wasm_runtime/src/lib.rs @@ -0,0 +1,22 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! WebAssembly execution runtime. +//! +//! Implementation lives under `src/wasm/` in this package. It is currently **compiled as +//! part of the `function-stream` crate** via `#[path]` in `src/lib.rs` / `src/main.rs`, so paths +//! like `crate::sql` (streaming planner dependency) and `crate::memory` keep resolving. +//! +//! Operator state storage (`state_backend/`) also lives in this package and is compiled via +//! `#[path]` from the root crate as `crate::state_backend`. + +pub const CRATE_NAME: &str = "function-stream-wasm-runtime"; diff --git a/src/storage/state_backend/error.rs b/src/wasm_runtime/src/state_backend/error.rs similarity index 100% rename from src/storage/state_backend/error.rs rename to src/wasm_runtime/src/state_backend/error.rs diff --git a/src/storage/state_backend/factory.rs b/src/wasm_runtime/src/state_backend/factory.rs similarity index 82% rename from src/storage/state_backend/factory.rs rename to src/wasm_runtime/src/state_backend/factory.rs index 5dc38632..2bc0b55c 100644 --- a/src/storage/state_backend/factory.rs +++ b/src/wasm_runtime/src/state_backend/factory.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage::state_backend::error::BackendError; -use crate::storage::state_backend::store::StateStore; +use crate::state_backend::error::BackendError; +use crate::state_backend::store::StateStore; use std::path::Path; use std::sync::Arc; @@ -47,11 +47,11 @@ pub fn get_factory_for_task>( task_name: String, created_at: u64, base_dir: Option

, - rocksdb_config: Option, + rocksdb_config: Option, ) -> Result, BackendError> { match factory_type { FactoryType::Memory => { - Ok(crate::storage::state_backend::memory::MemoryStateStoreFactory::default_factory()) + Ok(crate::state_backend::memory::MemoryStateStoreFactory::default_factory()) } FactoryType::RocksDB => { let base_dir = base_dir.ok_or_else(|| { @@ -63,9 +63,8 @@ pub fn get_factory_for_task>( .join(format!("{}-{}", task_name, created_at)); let config = rocksdb_config.unwrap_or_default(); - let factory = crate::storage::state_backend::rocksdb::RocksDBStateStoreFactory::new( - db_path, config, - )?; + let factory = + crate::state_backend::rocksdb::RocksDBStateStoreFactory::new(db_path, config)?; Ok(Arc::new(factory)) } diff --git a/src/storage/state_backend/key_builder.rs b/src/wasm_runtime/src/state_backend/key_builder.rs similarity index 100% rename from src/storage/state_backend/key_builder.rs rename to src/wasm_runtime/src/state_backend/key_builder.rs diff --git a/src/storage/state_backend/memory/factory.rs b/src/wasm_runtime/src/state_backend/memory/factory.rs similarity index 86% rename from src/storage/state_backend/memory/factory.rs rename to src/wasm_runtime/src/state_backend/memory/factory.rs index b62bd444..0526ecbd 100644 --- a/src/storage/state_backend/memory/factory.rs +++ b/src/wasm_runtime/src/state_backend/memory/factory.rs @@ -11,8 +11,8 @@ // limitations under the License. use super::store::MemoryStateStore; -use crate::storage::state_backend::error::BackendError; -use crate::storage::state_backend::factory::StateStoreFactory; +use crate::state_backend::error::BackendError; +use crate::state_backend::factory::StateStoreFactory; use std::sync::{Arc, Mutex}; pub struct MemoryStateStoreFactory {} @@ -43,7 +43,7 @@ impl StateStoreFactory for MemoryStateStoreFactory { fn new_state_store( &self, _column_family: Option, - ) -> Result, BackendError> { + ) -> Result, BackendError> { Ok(Box::new(MemoryStateStore::new())) } } diff --git a/src/storage/state_backend/memory/mod.rs b/src/wasm_runtime/src/state_backend/memory/mod.rs similarity index 100% rename from src/storage/state_backend/memory/mod.rs rename to src/wasm_runtime/src/state_backend/memory/mod.rs diff --git a/src/storage/state_backend/memory/store.rs b/src/wasm_runtime/src/state_backend/memory/store.rs similarity index 95% rename from src/storage/state_backend/memory/store.rs rename to src/wasm_runtime/src/state_backend/memory/store.rs index e65e839d..5d07ad99 100644 --- a/src/storage/state_backend/memory/store.rs +++ b/src/wasm_runtime/src/state_backend/memory/store.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage::state_backend::error::BackendError; -use crate::storage::state_backend::store::{StateIterator, StateStore}; +use crate::state_backend::error::BackendError; +use crate::state_backend::store::{StateIterator, StateStore}; use std::collections::HashMap; use std::sync::{Arc, Mutex}; @@ -94,9 +94,8 @@ impl StateStore for MemoryStateStore { user_key: Vec, value: Vec, ) -> Result<(), BackendError> { - let key_bytes = crate::storage::state_backend::key_builder::build_key( - &key_group, &key, &namespace, &user_key, - ); + let key_bytes = + crate::state_backend::key_builder::build_key(&key_group, &key, &namespace, &user_key); let existing = self.get_state(key_bytes.clone())?; diff --git a/src/storage/state_backend/mod.rs b/src/wasm_runtime/src/state_backend/mod.rs similarity index 100% rename from src/storage/state_backend/mod.rs rename to src/wasm_runtime/src/state_backend/mod.rs diff --git a/src/storage/state_backend/rocksdb/factory.rs b/src/wasm_runtime/src/state_backend/rocksdb/factory.rs similarity index 94% rename from src/storage/state_backend/rocksdb/factory.rs rename to src/wasm_runtime/src/state_backend/rocksdb/factory.rs index 11554e13..74759b91 100644 --- a/src/storage/state_backend/rocksdb/factory.rs +++ b/src/wasm_runtime/src/state_backend/rocksdb/factory.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage::state_backend::error::BackendError; -use crate::storage::state_backend::factory::StateStoreFactory; +use crate::state_backend::error::BackendError; +use crate::state_backend::factory::StateStoreFactory; use rocksdb::{ColumnFamilyDescriptor, DB, Options}; use std::path::Path; use std::sync::{Arc, Mutex}; @@ -56,7 +56,7 @@ impl StateStoreFactory for RocksDBStateStoreFactory { fn new_state_store( &self, column_family: Option, - ) -> Result, BackendError> { + ) -> Result, BackendError> { self.new_state_store(column_family) } } @@ -137,7 +137,7 @@ impl RocksDBStateStoreFactory { pub fn new_state_store( &self, column_family: Option, - ) -> Result, BackendError> { + ) -> Result, BackendError> { if let Some(ref cf_name) = column_family && cf_name != "default" && self.db.cf_handle(cf_name).is_none() @@ -158,7 +158,7 @@ impl RocksDBStateStoreFactory { } } - crate::storage::state_backend::rocksdb::store::RocksDBStateStore::new_with_factory( + crate::state_backend::rocksdb::store::RocksDBStateStore::new_with_factory( self.db.clone(), column_family, ) diff --git a/src/storage/state_backend/rocksdb/mod.rs b/src/wasm_runtime/src/state_backend/rocksdb/mod.rs similarity index 100% rename from src/storage/state_backend/rocksdb/mod.rs rename to src/wasm_runtime/src/state_backend/rocksdb/mod.rs diff --git a/src/storage/state_backend/rocksdb/store.rs b/src/wasm_runtime/src/state_backend/rocksdb/store.rs similarity index 97% rename from src/storage/state_backend/rocksdb/store.rs rename to src/wasm_runtime/src/state_backend/rocksdb/store.rs index e8a2ad13..b7d98e4c 100644 --- a/src/storage/state_backend/rocksdb/store.rs +++ b/src/wasm_runtime/src/state_backend/rocksdb/store.rs @@ -10,9 +10,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage::state_backend::error::BackendError; -use crate::storage::state_backend::key_builder::{build_key, increment_key, is_all_0xff}; -use crate::storage::state_backend::store::{StateIterator, StateStore}; +use crate::state_backend::error::BackendError; +use crate::state_backend::key_builder::{build_key, increment_key, is_all_0xff}; +use crate::state_backend::store::{StateIterator, StateStore}; use rocksdb::{ BlockBasedOptions, Cache, ColumnFamilyDescriptor, DB, DBCompressionType, Direction, IteratorMode, Options, ReadOptions, WriteBatch, WriteOptions, diff --git a/src/storage/state_backend/server.rs b/src/wasm_runtime/src/state_backend/server.rs similarity index 92% rename from src/storage/state_backend/server.rs rename to src/wasm_runtime/src/state_backend/server.rs index efeceaa4..1fb24c38 100644 --- a/src/storage/state_backend/server.rs +++ b/src/wasm_runtime/src/state_backend/server.rs @@ -12,11 +12,9 @@ use crate::config::storage::{StateStorageConfig, StateStorageType}; use crate::config::{get_state_dir, get_state_dir_for_base}; -use crate::storage::state_backend::error::BackendError; -use crate::storage::state_backend::factory::{ - FactoryType, StateStoreFactory, get_factory_for_task, -}; -use crate::storage::state_backend::rocksdb::RocksDBConfig; +use crate::state_backend::error::BackendError; +use crate::state_backend::factory::{FactoryType, StateStoreFactory, get_factory_for_task}; +use crate::state_backend::rocksdb::RocksDBConfig; use std::fs; use std::path::PathBuf; use std::sync::Arc; diff --git a/src/storage/state_backend/store.rs b/src/wasm_runtime/src/state_backend/store.rs similarity index 88% rename from src/storage/state_backend/store.rs rename to src/wasm_runtime/src/state_backend/store.rs index aaa02988..cb897f6b 100644 --- a/src/storage/state_backend/store.rs +++ b/src/wasm_runtime/src/state_backend/store.rs @@ -10,7 +10,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage::state_backend::error::BackendError; +use crate::state_backend::error::BackendError; pub type StateIteratorItem = Result, Vec)>, BackendError>; @@ -107,9 +107,8 @@ pub trait StateStore: Send + Sync { user_key: Vec, value: Vec, ) -> Result<(), BackendError> { - let key_bytes = crate::storage::state_backend::key_builder::build_key( - &key_group, &key, &namespace, &user_key, - ); + let key_bytes = + crate::state_backend::key_builder::build_key(&key_group, &key, &namespace, &user_key); self.put_state(key_bytes, value) } @@ -132,9 +131,8 @@ pub trait StateStore: Send + Sync { namespace: Vec, user_key: Vec, ) -> Result>, BackendError> { - let key_bytes = crate::storage::state_backend::key_builder::build_key( - &key_group, &key, &namespace, &user_key, - ); + let key_bytes = + crate::state_backend::key_builder::build_key(&key_group, &key, &namespace, &user_key); self.get_state(key_bytes) } @@ -156,9 +154,8 @@ pub trait StateStore: Send + Sync { namespace: Vec, user_key: Vec, ) -> Result<(), BackendError> { - let key_bytes = crate::storage::state_backend::key_builder::build_key( - &key_group, &key, &namespace, &user_key, - ); + let key_bytes = + crate::state_backend::key_builder::build_key(&key_group, &key, &namespace, &user_key); self.delete_state(key_bytes) } @@ -199,12 +196,8 @@ pub trait StateStore: Send + Sync { key: Vec, namespace: Vec, ) -> Result { - let prefix_bytes = crate::storage::state_backend::key_builder::build_key( - &key_group, - &key, - &namespace, - &[], - ); + let prefix_bytes = + crate::state_backend::key_builder::build_key(&key_group, &key, &namespace, &[]); self.delete_prefix_bytes(prefix_bytes) } @@ -228,13 +221,13 @@ pub trait StateStore: Send + Sync { start_inclusive: Vec, end_exclusive: Vec, ) -> Result>, BackendError> { - let start_key = crate::storage::state_backend::key_builder::build_key( + let start_key = crate::state_backend::key_builder::build_key( &key_group, &key, &namespace, &start_inclusive, ); - let end_key = crate::storage::state_backend::key_builder::build_key( + let end_key = crate::state_backend::key_builder::build_key( &key_group, &key, &namespace, @@ -279,12 +272,8 @@ pub trait StateStore: Send + Sync { key: Vec, namespace: Vec, ) -> Result, BackendError> { - let prefix = crate::storage::state_backend::key_builder::build_key( - &key_group, - &key, - &namespace, - &[], - ); + let prefix = + crate::state_backend::key_builder::build_key(&key_group, &key, &namespace, &[]); self.scan(prefix) } } diff --git a/src/runtime/wasm/buffer_and_event/buffer_or_event.rs b/src/wasm_runtime/src/wasm/buffer_and_event/buffer_or_event.rs similarity index 100% rename from src/runtime/wasm/buffer_and_event/buffer_or_event.rs rename to src/wasm_runtime/src/wasm/buffer_and_event/buffer_or_event.rs diff --git a/src/runtime/wasm/buffer_and_event/mod.rs b/src/wasm_runtime/src/wasm/buffer_and_event/mod.rs similarity index 100% rename from src/runtime/wasm/buffer_and_event/mod.rs rename to src/wasm_runtime/src/wasm/buffer_and_event/mod.rs diff --git a/src/runtime/wasm/buffer_and_event/stream_element/mod.rs b/src/wasm_runtime/src/wasm/buffer_and_event/stream_element/mod.rs similarity index 100% rename from src/runtime/wasm/buffer_and_event/stream_element/mod.rs rename to src/wasm_runtime/src/wasm/buffer_and_event/stream_element/mod.rs diff --git a/src/runtime/wasm/buffer_and_event/stream_element/stream_element.rs b/src/wasm_runtime/src/wasm/buffer_and_event/stream_element/stream_element.rs similarity index 100% rename from src/runtime/wasm/buffer_and_event/stream_element/stream_element.rs rename to src/wasm_runtime/src/wasm/buffer_and_event/stream_element/stream_element.rs diff --git a/src/runtime/wasm/input/input_protocol.rs b/src/wasm_runtime/src/wasm/input/input_protocol.rs similarity index 95% rename from src/runtime/wasm/input/input_protocol.rs rename to src/wasm_runtime/src/wasm/input/input_protocol.rs index 50294201..782a7f4f 100644 --- a/src/runtime/wasm/input/input_protocol.rs +++ b/src/wasm_runtime/src/wasm/input/input_protocol.rs @@ -10,7 +10,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::wasm::buffer_and_event::BufferOrEvent; +use crate::wasm::buffer_and_event::BufferOrEvent; use std::time::Duration; pub trait InputProtocol: Send + Sync + 'static { diff --git a/src/runtime/wasm/input/input_provider.rs b/src/wasm_runtime/src/wasm/input/input_provider.rs similarity index 94% rename from src/runtime/wasm/input/input_provider.rs rename to src/wasm_runtime/src/wasm/input/input_provider.rs index 8eee649d..d7ed64b9 100644 --- a/src/runtime/wasm/input/input_provider.rs +++ b/src/wasm_runtime/src/wasm/input/input_provider.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::input::Input; -use crate::runtime::wasm::task::InputConfig; +use crate::input::Input; +use crate::wasm::task::InputConfig; pub struct InputProvider; @@ -66,8 +66,8 @@ impl InputProvider { extra, runtime: _, } => { - use crate::runtime::input::InputRunner; - use crate::runtime::input::protocol::kafka::{KafkaConfig, KafkaProtocol}; + use crate::input::InputRunner; + use crate::input::protocol::kafka::{KafkaConfig, KafkaProtocol}; let servers: Vec = bootstrap_servers .split(',') diff --git a/src/runtime/wasm/input/input_runner.rs b/src/wasm_runtime/src/wasm/input/input_runner.rs similarity index 96% rename from src/runtime/wasm/input/input_runner.rs rename to src/wasm_runtime/src/wasm/input/input_runner.rs index ece85e3d..24b96dae 100644 --- a/src/runtime/wasm/input/input_runner.rs +++ b/src/wasm_runtime/src/wasm/input/input_runner.rs @@ -10,13 +10,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::common::TaskCompletionFlag; -use crate::runtime::input::input_protocol::InputProtocol; -use crate::runtime::input::{Input, InputState}; -use crate::runtime::processor::function_error::FunctionErrorReport; -use crate::runtime::wasm::buffer_and_event::BufferOrEvent; -use crate::runtime::wasm::task::ControlMailBox; -use crate::runtime::wasm::task::InputRuntimeConfig; +use crate::common::TaskCompletionFlag; +use crate::input::input_protocol::InputProtocol; +use crate::input::{Input, InputState}; +use crate::processor::function_error::FunctionErrorReport; +use crate::wasm::buffer_and_event::BufferOrEvent; +use crate::wasm::task::ControlMailBox; +use crate::wasm::task::InputRuntimeConfig; use crossbeam_channel::{Receiver, Sender, bounded, unbounded}; use std::sync::{Arc, Mutex}; use std::thread; @@ -250,7 +250,7 @@ impl InputRunner

{ impl Input for InputRunner

{ fn init_with_context( &mut self, - init_context: &crate::runtime::wasm::taskexecutor::InitContext, + init_context: &crate::wasm::taskexecutor::InitContext, ) -> Result<(), Box> { if !matches!(*self.state.lock().unwrap(), InputState::Uninitialized) { return Ok(()); @@ -298,7 +298,7 @@ impl Input for InputRunner

{ }) .map_err(|e| Box::new(std::io::Error::other(e)) as Box)?; - use crate::runtime::processor::wasm::thread_pool::{ThreadGroup, ThreadGroupType}; + use crate::processor::wasm::thread_pool::{ThreadGroup, ThreadGroupType}; let mut group = ThreadGroup::new( ThreadGroupType::Input(self.group_id), format!("Input-g{}", self.group_id), diff --git a/src/runtime/wasm/input/interface.rs b/src/wasm_runtime/src/wasm/input/interface.rs similarity index 89% rename from src/runtime/wasm/input/interface.rs rename to src/wasm_runtime/src/wasm/input/interface.rs index 06da4923..cbfc61cc 100644 --- a/src/runtime/wasm/input/interface.rs +++ b/src/wasm_runtime/src/wasm/input/interface.rs @@ -10,10 +10,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::wasm::buffer_and_event::BufferOrEvent; -use crate::runtime::wasm::taskexecutor::InitContext; +use crate::wasm::buffer_and_event::BufferOrEvent; +use crate::wasm::taskexecutor::InitContext; -pub use crate::runtime::common::ComponentState as InputState; +pub use crate::common::ComponentState as InputState; pub trait Input: Send + Sync { fn init_with_context( diff --git a/src/runtime/wasm/input/mod.rs b/src/wasm_runtime/src/wasm/input/mod.rs similarity index 100% rename from src/runtime/wasm/input/mod.rs rename to src/wasm_runtime/src/wasm/input/mod.rs diff --git a/src/runtime/wasm/input/protocol/kafka/config.rs b/src/wasm_runtime/src/wasm/input/protocol/kafka/config.rs similarity index 100% rename from src/runtime/wasm/input/protocol/kafka/config.rs rename to src/wasm_runtime/src/wasm/input/protocol/kafka/config.rs diff --git a/src/runtime/wasm/input/protocol/kafka/kafka_protocol.rs b/src/wasm_runtime/src/wasm/input/protocol/kafka/kafka_protocol.rs similarity index 97% rename from src/runtime/wasm/input/protocol/kafka/kafka_protocol.rs rename to src/wasm_runtime/src/wasm/input/protocol/kafka/kafka_protocol.rs index 1fb487a6..928c9050 100644 --- a/src/runtime/wasm/input/protocol/kafka/kafka_protocol.rs +++ b/src/wasm_runtime/src/wasm/input/protocol/kafka/kafka_protocol.rs @@ -11,8 +11,8 @@ // limitations under the License. use super::config::KafkaConfig; -use crate::runtime::input::input_protocol::InputProtocol; -use crate::runtime::wasm::buffer_and_event::BufferOrEvent; +use crate::input::input_protocol::InputProtocol; +use crate::wasm::buffer_and_event::BufferOrEvent; use rdkafka::Message; use rdkafka::TopicPartitionList; use rdkafka::config::ClientConfig; diff --git a/src/runtime/wasm/input/protocol/kafka/mod.rs b/src/wasm_runtime/src/wasm/input/protocol/kafka/mod.rs similarity index 100% rename from src/runtime/wasm/input/protocol/kafka/mod.rs rename to src/wasm_runtime/src/wasm/input/protocol/kafka/mod.rs diff --git a/src/sql/connector/source/mod.rs b/src/wasm_runtime/src/wasm/input/protocol/mod.rs similarity index 100% rename from src/sql/connector/source/mod.rs rename to src/wasm_runtime/src/wasm/input/protocol/mod.rs diff --git a/src/runtime/wasm/mod.rs b/src/wasm_runtime/src/wasm/mod.rs similarity index 100% rename from src/runtime/wasm/mod.rs rename to src/wasm_runtime/src/wasm/mod.rs diff --git a/src/runtime/wasm/output/interface.rs b/src/wasm_runtime/src/wasm/output/interface.rs similarity index 93% rename from src/runtime/wasm/output/interface.rs rename to src/wasm_runtime/src/wasm/output/interface.rs index 21c3055d..e38217a1 100644 --- a/src/runtime/wasm/output/interface.rs +++ b/src/wasm_runtime/src/wasm/output/interface.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::wasm::buffer_and_event::BufferOrEvent; -use crate::runtime::wasm::taskexecutor::InitContext; +use crate::wasm::buffer_and_event::BufferOrEvent; +use crate::wasm::taskexecutor::InitContext; pub trait Output: Send + Sync { fn init_with_context( diff --git a/src/runtime/wasm/output/mod.rs b/src/wasm_runtime/src/wasm/output/mod.rs similarity index 100% rename from src/runtime/wasm/output/mod.rs rename to src/wasm_runtime/src/wasm/output/mod.rs diff --git a/src/runtime/wasm/output/output_protocol.rs b/src/wasm_runtime/src/wasm/output/output_protocol.rs similarity index 95% rename from src/runtime/wasm/output/output_protocol.rs rename to src/wasm_runtime/src/wasm/output/output_protocol.rs index 6140d3eb..8efc5f7d 100644 --- a/src/runtime/wasm/output/output_protocol.rs +++ b/src/wasm_runtime/src/wasm/output/output_protocol.rs @@ -10,7 +10,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::wasm::buffer_and_event::BufferOrEvent; +use crate::wasm::buffer_and_event::BufferOrEvent; pub trait OutputProtocol: Send + Sync + 'static { fn name(&self) -> String; diff --git a/src/runtime/wasm/output/output_provider.rs b/src/wasm_runtime/src/wasm/output/output_provider.rs similarity index 91% rename from src/runtime/wasm/output/output_provider.rs rename to src/wasm_runtime/src/wasm/output/output_provider.rs index 25ca8431..33b470d1 100644 --- a/src/runtime/wasm/output/output_provider.rs +++ b/src/wasm_runtime/src/wasm/output/output_provider.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::output::Output; -use crate::runtime::wasm::task::OutputConfig; +use crate::output::Output; +use crate::wasm::task::OutputConfig; pub struct OutputProvider; @@ -59,10 +59,8 @@ impl OutputProvider { extra, runtime: _, } => { - use crate::runtime::output::output_runner::OutputRunner; - use crate::runtime::output::protocol::kafka::{ - KafkaOutputProtocol, KafkaProducerConfig, - }; + use crate::output::output_runner::OutputRunner; + use crate::output::protocol::kafka::{KafkaOutputProtocol, KafkaProducerConfig}; let servers: Vec = bootstrap_servers .split(',') diff --git a/src/runtime/wasm/output/output_runner.rs b/src/wasm_runtime/src/wasm/output/output_runner.rs similarity index 96% rename from src/runtime/wasm/output/output_runner.rs rename to src/wasm_runtime/src/wasm/output/output_runner.rs index ca6d780c..b732693a 100644 --- a/src/runtime/wasm/output/output_runner.rs +++ b/src/wasm_runtime/src/wasm/output/output_runner.rs @@ -10,13 +10,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::common::{ComponentState, TaskCompletionFlag}; -use crate::runtime::output::Output; -use crate::runtime::output::output_protocol::OutputProtocol; -use crate::runtime::processor::function_error::FunctionErrorReport; -use crate::runtime::wasm::buffer_and_event::BufferOrEvent; -use crate::runtime::wasm::task::ControlMailBox; -use crate::runtime::wasm::task::OutputRuntimeConfig; +use crate::common::{ComponentState, TaskCompletionFlag}; +use crate::output::Output; +use crate::output::output_protocol::OutputProtocol; +use crate::processor::function_error::FunctionErrorReport; +use crate::wasm::buffer_and_event::BufferOrEvent; +use crate::wasm::task::ControlMailBox; +use crate::wasm::task::OutputRuntimeConfig; use crossbeam_channel::{Receiver, Sender, bounded, unbounded}; use std::sync::{Arc, Mutex}; use std::thread; @@ -288,7 +288,7 @@ impl OutputRunner

{ impl Output for OutputRunner

{ fn init_with_context( &mut self, - ctx: &crate::runtime::wasm::taskexecutor::InitContext, + ctx: &crate::wasm::taskexecutor::InitContext, ) -> Result<(), Box> { if !matches!(*self.state.lock().unwrap(), ComponentState::Uninitialized) { return Ok(()); @@ -333,7 +333,7 @@ impl Output for OutputRunner

{ }) .map_err(|e| Box::new(std::io::Error::other(e)) as Box)?; - use crate::runtime::processor::wasm::thread_pool::{ThreadGroup, ThreadGroupType}; + use crate::processor::wasm::thread_pool::{ThreadGroup, ThreadGroupType}; let mut group = ThreadGroup::new( ThreadGroupType::Output(self.output_id), format!("Output-{}", self.output_id), diff --git a/src/runtime/wasm/output/protocol/kafka/kafka_protocol.rs b/src/wasm_runtime/src/wasm/output/protocol/kafka/kafka_protocol.rs similarity index 96% rename from src/runtime/wasm/output/protocol/kafka/kafka_protocol.rs rename to src/wasm_runtime/src/wasm/output/protocol/kafka/kafka_protocol.rs index d9e6db4d..787804a2 100644 --- a/src/runtime/wasm/output/protocol/kafka/kafka_protocol.rs +++ b/src/wasm_runtime/src/wasm/output/protocol/kafka/kafka_protocol.rs @@ -11,8 +11,8 @@ // limitations under the License. use super::producer_config::KafkaProducerConfig; -use crate::runtime::output::output_protocol::OutputProtocol; -use crate::runtime::wasm::buffer_and_event::BufferOrEvent; +use crate::output::output_protocol::OutputProtocol; +use crate::wasm::buffer_and_event::BufferOrEvent; use rdkafka::producer::{BaseRecord, DefaultProducerContext, Producer, ThreadedProducer}; use std::sync::Mutex; use std::time::Duration; diff --git a/src/runtime/wasm/output/protocol/kafka/mod.rs b/src/wasm_runtime/src/wasm/output/protocol/kafka/mod.rs similarity index 100% rename from src/runtime/wasm/output/protocol/kafka/mod.rs rename to src/wasm_runtime/src/wasm/output/protocol/kafka/mod.rs diff --git a/src/runtime/wasm/output/protocol/kafka/producer_config.rs b/src/wasm_runtime/src/wasm/output/protocol/kafka/producer_config.rs similarity index 100% rename from src/runtime/wasm/output/protocol/kafka/producer_config.rs rename to src/wasm_runtime/src/wasm/output/protocol/kafka/producer_config.rs diff --git a/src/runtime/wasm/output/protocol/mod.rs b/src/wasm_runtime/src/wasm/output/protocol/mod.rs similarity index 100% rename from src/runtime/wasm/output/protocol/mod.rs rename to src/wasm_runtime/src/wasm/output/protocol/mod.rs diff --git a/src/runtime/wasm/processor/function_error.rs b/src/wasm_runtime/src/wasm/processor/function_error.rs similarity index 100% rename from src/runtime/wasm/processor/function_error.rs rename to src/wasm_runtime/src/wasm/processor/function_error.rs diff --git a/src/runtime/wasm/processor/mod.rs b/src/wasm_runtime/src/wasm/processor/mod.rs similarity index 100% rename from src/runtime/wasm/processor/mod.rs rename to src/wasm_runtime/src/wasm/processor/mod.rs diff --git a/src/runtime/wasm/processor/python/mod.rs b/src/wasm_runtime/src/wasm/processor/python/mod.rs similarity index 100% rename from src/runtime/wasm/processor/python/mod.rs rename to src/wasm_runtime/src/wasm/processor/python/mod.rs diff --git a/src/runtime/wasm/processor/python/python_host.rs b/src/wasm_runtime/src/wasm/processor/python/python_host.rs similarity index 100% rename from src/runtime/wasm/processor/python/python_host.rs rename to src/wasm_runtime/src/wasm/processor/python/python_host.rs diff --git a/src/runtime/wasm/processor/python/python_service.rs b/src/wasm_runtime/src/wasm/processor/python/python_service.rs similarity index 100% rename from src/runtime/wasm/processor/python/python_service.rs rename to src/wasm_runtime/src/wasm/processor/python/python_service.rs diff --git a/src/runtime/wasm/processor/wasm/input_strategy.rs b/src/wasm_runtime/src/wasm/processor/wasm/input_strategy.rs similarity index 100% rename from src/runtime/wasm/processor/wasm/input_strategy.rs rename to src/wasm_runtime/src/wasm/processor/wasm/input_strategy.rs diff --git a/src/runtime/wasm/processor/wasm/mod.rs b/src/wasm_runtime/src/wasm/processor/wasm/mod.rs similarity index 100% rename from src/runtime/wasm/processor/wasm/mod.rs rename to src/wasm_runtime/src/wasm/processor/wasm/mod.rs diff --git a/src/runtime/wasm/processor/wasm/thread_pool.rs b/src/wasm_runtime/src/wasm/processor/wasm/thread_pool.rs similarity index 99% rename from src/runtime/wasm/processor/wasm/thread_pool.rs rename to src/wasm_runtime/src/wasm/processor/wasm/thread_pool.rs index c78eef79..d625d86a 100644 --- a/src/runtime/wasm/processor/wasm/thread_pool.rs +++ b/src/wasm_runtime/src/wasm/processor/wasm/thread_pool.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::common::ComponentState; -use crate::runtime::processor::wasm::wasm_task::WasmTask; +use crate::common::ComponentState; +use crate::processor::wasm::wasm_task::WasmTask; use std::collections::HashMap; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; diff --git a/src/runtime/wasm/processor/wasm/wasm_cache.rs b/src/wasm_runtime/src/wasm/processor/wasm/wasm_cache.rs similarity index 100% rename from src/runtime/wasm/processor/wasm/wasm_cache.rs rename to src/wasm_runtime/src/wasm/processor/wasm/wasm_cache.rs diff --git a/src/runtime/wasm/processor/wasm/wasm_host.rs b/src/wasm_runtime/src/wasm/processor/wasm/wasm_host.rs similarity index 94% rename from src/runtime/wasm/processor/wasm/wasm_host.rs rename to src/wasm_runtime/src/wasm/processor/wasm/wasm_host.rs index 2bf7d4f0..348f3063 100644 --- a/src/runtime/wasm/processor/wasm/wasm_host.rs +++ b/src/wasm_runtime/src/wasm/processor/wasm/wasm_host.rs @@ -10,10 +10,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::output::Output; -use crate::runtime::processor::wasm::wasm_cache; -use crate::runtime::wasm::buffer_and_event::BufferOrEvent; -use crate::storage::state_backend::{StateStore, StateStoreFactory}; +use crate::output::Output; +use crate::processor::wasm::wasm_cache; +use crate::state_backend::{StateStore, StateStoreFactory}; +use crate::wasm::buffer_and_event::BufferOrEvent; use std::sync::{Arc, OnceLock}; use wasmtime::component::{Component, HasData, Linker, Resource, bindgen}; use wasmtime::{Config, Engine, Store}; @@ -67,7 +67,7 @@ fn get_global_engine(_wasm_size: usize) -> anyhow::Result> { bindgen!({ world: "processor", - path: "wit", + path: "../../wit", with: { "functionstream:core/kv.store": FunctionStreamStoreHandle, "functionstream:core/kv.iterator": FunctionStreamIteratorHandle, @@ -86,7 +86,7 @@ impl Drop for FunctionStreamStoreHandle { } pub struct FunctionStreamIteratorHandle { - pub state_iterator: Box, + pub state_iterator: Box, } pub struct HostState { @@ -205,7 +205,7 @@ impl HostStore for HostState { .get(&self_) .map_err(|e| Error::Other(format!("Failed to get store resource: {}", e)))?; - let real_key = crate::storage::state_backend::key_builder::build_key( + let real_key = crate::state_backend::key_builder::build_key( &key.key_group, &key.key, &key.namespace, @@ -228,7 +228,7 @@ impl HostStore for HostState { .get(&self_) .map_err(|e| Error::Other(format!("Failed to get store resource: {}", e)))?; - let real_key = crate::storage::state_backend::key_builder::build_key( + let real_key = crate::state_backend::key_builder::build_key( &key.key_group, &key.key, &key.namespace, @@ -251,7 +251,7 @@ impl HostStore for HostState { .get(&self_) .map_err(|e| Error::Other(format!("Failed to get store resource: {}", e)))?; - let real_key = crate::storage::state_backend::key_builder::build_key( + let real_key = crate::state_backend::key_builder::build_key( &key.key_group, &key.key, &key.namespace, @@ -284,7 +284,7 @@ impl HostStore for HostState { .get(&self_) .map_err(|e| Error::Other(format!("Failed to get store resource: {}", e)))?; - let prefix_key = crate::storage::state_backend::key_builder::build_key( + let prefix_key = crate::state_backend::key_builder::build_key( &key.key_group, &key.key, &key.namespace, @@ -323,13 +323,13 @@ impl HostStore for HostState { .get(&self_) .map_err(|e| Error::Other(format!("Failed to get store resource: {}", e)))?; - let start_key = crate::storage::state_backend::key_builder::build_key( + let start_key = crate::state_backend::key_builder::build_key( &key_group, &key, &namespace, &start_inclusive, ); - let end_key = crate::storage::state_backend::key_builder::build_key( + let end_key = crate::state_backend::key_builder::build_key( &key_group, &key, &namespace, @@ -449,7 +449,7 @@ pub fn create_wasm_host_with_component( engine: &Engine, component: &Component, outputs: Vec>, - init_context: &crate::runtime::wasm::taskexecutor::InitContext, + init_context: &crate::wasm::taskexecutor::InitContext, task_name: String, create_time: u64, ) -> anyhow::Result<(Processor, Store)> { @@ -495,7 +495,7 @@ pub fn create_wasm_host_with_component( pub fn create_wasm_host( wasm_bytes: &[u8], outputs: Vec>, - init_context: &crate::runtime::wasm::taskexecutor::InitContext, + init_context: &crate::wasm::taskexecutor::InitContext, task_name: String, create_time: u64, ) -> anyhow::Result<(Processor, Store)> { diff --git a/src/runtime/wasm/processor/wasm/wasm_processor.rs b/src/wasm_runtime/src/wasm/processor/wasm/wasm_processor.rs similarity index 99% rename from src/runtime/wasm/processor/wasm/wasm_processor.rs rename to src/wasm_runtime/src/wasm/processor/wasm/wasm_processor.rs index 52234bfe..741280bb 100644 --- a/src/runtime/wasm/processor/wasm/wasm_processor.rs +++ b/src/wasm_runtime/src/wasm/processor/wasm/wasm_processor.rs @@ -17,7 +17,7 @@ use super::wasm_host::{HostState, Processor}; use super::wasm_processor_trait::WasmProcessor; -use crate::runtime::output::Output; +use crate::output::Output; use std::cell::RefCell; use std::error::Error; use std::fmt; @@ -134,7 +134,7 @@ impl WasmProcessorImpl { impl WasmProcessor for WasmProcessorImpl { fn init_with_context( &mut self, - _init_context: &crate::runtime::wasm::taskexecutor::InitContext, + _init_context: &crate::wasm::taskexecutor::InitContext, ) -> Result<(), Box> { if self.initialized { log::warn!("WasmProcessor '{}' already initialized", self.name); @@ -405,7 +405,7 @@ impl WasmProcessor for WasmProcessorImpl { fn init_wasm_host( &mut self, outputs: Vec>, - init_context: &crate::runtime::wasm::taskexecutor::InitContext, + init_context: &crate::wasm::taskexecutor::InitContext, task_name: String, create_time: u64, ) -> Result<(), Box> { diff --git a/src/runtime/wasm/processor/wasm/wasm_processor_trait.rs b/src/wasm_runtime/src/wasm/processor/wasm/wasm_processor_trait.rs similarity index 97% rename from src/runtime/wasm/processor/wasm/wasm_processor_trait.rs rename to src/wasm_runtime/src/wasm/processor/wasm/wasm_processor_trait.rs index fb2c17fb..c699fe8b 100644 --- a/src/runtime/wasm/processor/wasm/wasm_processor_trait.rs +++ b/src/wasm_runtime/src/wasm/processor/wasm/wasm_processor_trait.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::output::Output; -use crate::runtime::wasm::taskexecutor::InitContext; +use crate::output::Output; +use crate::wasm::taskexecutor::InitContext; pub trait WasmProcessor: Send + Sync { fn process( diff --git a/src/runtime/wasm/processor/wasm/wasm_task.rs b/src/wasm_runtime/src/wasm/processor/wasm/wasm_task.rs similarity index 97% rename from src/runtime/wasm/processor/wasm/wasm_task.rs rename to src/wasm_runtime/src/wasm/processor/wasm/wasm_task.rs index 4330aaaf..ea0f5dc1 100644 --- a/src/runtime/wasm/processor/wasm/wasm_task.rs +++ b/src/wasm_runtime/src/wasm/processor/wasm/wasm_task.rs @@ -13,14 +13,14 @@ use super::input_strategy::{InputStrategy, RoundRobinStrategy, from_selector_name}; use super::thread_pool::ThreadGroup; use super::wasm_processor_trait::WasmProcessor; -use crate::runtime::common::{ComponentState, TaskCompletionFlag}; -use crate::runtime::input::Input; -use crate::runtime::output::Output; -use crate::runtime::processor::function_error::FunctionErrorReport; -use crate::runtime::wasm::buffer_and_event::BufferOrEvent; -use crate::runtime::wasm::task::ProcessorRuntimeConfig; -use crate::runtime::wasm::task::{ControlMailBox, TaskControlSignal, TaskLifecycle}; -use crate::storage::task::FunctionInfo; +use crate::common::{ComponentState, TaskCompletionFlag}; +use crate::input::Input; +use crate::output::Output; +use crate::processor::function_error::FunctionErrorReport; +use crate::task::FunctionInfo; +use crate::wasm::buffer_and_event::BufferOrEvent; +use crate::wasm::task::ProcessorRuntimeConfig; +use crate::wasm::task::{ControlMailBox, TaskControlSignal, TaskLifecycle}; use crossbeam_channel::{Receiver, after, select, unbounded}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc; @@ -120,7 +120,7 @@ impl WasmTask { pub fn init_with_context( &mut self, - init_context: &crate::runtime::wasm::taskexecutor::InitContext, + init_context: &crate::wasm::taskexecutor::InitContext, ) -> Result<(), Box> { let mut inputs = self.inputs.take().ok_or_else(|| { Box::new(std::io::Error::other("inputs already moved to thread")) @@ -235,7 +235,7 @@ impl WasmTask { ))) as Box })?; - use crate::runtime::processor::wasm::thread_pool::{ThreadGroup, ThreadGroupType}; + use crate::processor::wasm::thread_pool::{ThreadGroup, ThreadGroupType}; let mut main_runloop_group = ThreadGroup::new( ThreadGroupType::MainRunloop, format!("MainRunloop-{}", self.task_name), @@ -262,7 +262,7 @@ impl WasmTask { shared_state: Arc>, failure_cause: Arc>>, execution_state: Arc>, - _init_context: crate::runtime::wasm::taskexecutor::InitContext, + _init_context: crate::wasm::taskexecutor::InitContext, ) { let mut state = TaskState::Initialized; let mut last_idx: usize = 0; @@ -729,7 +729,7 @@ impl WasmTask { impl TaskLifecycle for WasmTask { fn init_with_context( &mut self, - init_context: &crate::runtime::wasm::taskexecutor::InitContext, + init_context: &crate::wasm::taskexecutor::InitContext, ) -> Result<(), Box> { ::init_with_context(self, init_context) } diff --git a/src/runtime/wasm/task/builder/mod.rs b/src/wasm_runtime/src/wasm/task/builder/mod.rs similarity index 100% rename from src/runtime/wasm/task/builder/mod.rs rename to src/wasm_runtime/src/wasm/task/builder/mod.rs diff --git a/src/runtime/wasm/task/builder/processor/mod.rs b/src/wasm_runtime/src/wasm/task/builder/processor/mod.rs similarity index 92% rename from src/runtime/wasm/task/builder/processor/mod.rs rename to src/wasm_runtime/src/wasm/task/builder/processor/mod.rs index c1306924..1ff39a7d 100644 --- a/src/runtime/wasm/task/builder/processor/mod.rs +++ b/src/wasm_runtime/src/wasm/task/builder/processor/mod.rs @@ -14,13 +14,13 @@ // // Specifically handles building logic for Processor type configuration -use crate::runtime::input::{Input, InputProvider}; -use crate::runtime::output::{Output, OutputProvider}; -use crate::runtime::processor::wasm::wasm_processor::WasmProcessorImpl; -use crate::runtime::processor::wasm::wasm_processor_trait::WasmProcessor; -use crate::runtime::processor::wasm::wasm_task::WasmTask; -use crate::runtime::wasm::task::yaml_keys::{TYPE, type_values}; -use crate::runtime::wasm::task::{InputConfig, OutputConfig, ProcessorConfig, WasmTaskConfig}; +use crate::input::{Input, InputProvider}; +use crate::output::{Output, OutputProvider}; +use crate::processor::wasm::wasm_processor::WasmProcessorImpl; +use crate::processor::wasm::wasm_processor_trait::WasmProcessor; +use crate::processor::wasm::wasm_task::WasmTask; +use crate::wasm::task::yaml_keys::{TYPE, type_values}; +use crate::wasm::task::{InputConfig, OutputConfig, ProcessorConfig, WasmTaskConfig}; use serde_yaml::Value; use std::sync::Arc; diff --git a/src/runtime/wasm/task/builder/python/mod.rs b/src/wasm_runtime/src/wasm/task/builder/python/mod.rs similarity index 89% rename from src/runtime/wasm/task/builder/python/mod.rs rename to src/wasm_runtime/src/wasm/task/builder/python/mod.rs index 1b31d2e5..8bc9aa2d 100644 --- a/src/runtime/wasm/task/builder/python/mod.rs +++ b/src/wasm_runtime/src/wasm/task/builder/python/mod.rs @@ -14,14 +14,14 @@ // // Specifically handles building logic for python runtime configuration -use crate::runtime::input::{Input, InputProvider}; -use crate::runtime::output::{Output, OutputProvider}; -use crate::runtime::processor::python::get_python_engine_and_component; -use crate::runtime::processor::wasm::wasm_processor::WasmProcessorImpl; -use crate::runtime::processor::wasm::wasm_processor_trait::WasmProcessor; -use crate::runtime::processor::wasm::wasm_task::WasmTask; -use crate::runtime::wasm::task::yaml_keys::{TYPE, type_values}; -use crate::runtime::wasm::task::{InputConfig, OutputConfig, ProcessorConfig, WasmTaskConfig}; +use crate::input::{Input, InputProvider}; +use crate::output::{Output, OutputProvider}; +use crate::processor::python::get_python_engine_and_component; +use crate::processor::wasm::wasm_processor::WasmProcessorImpl; +use crate::processor::wasm::wasm_processor_trait::WasmProcessor; +use crate::processor::wasm::wasm_task::WasmTask; +use crate::wasm::task::yaml_keys::{TYPE, type_values}; +use crate::wasm::task::{InputConfig, OutputConfig, ProcessorConfig, WasmTaskConfig}; use serde_yaml::Value; use std::sync::Arc; @@ -33,8 +33,7 @@ impl PythonBuilder { yaml_value: &Value, modules: &[(String, Vec)], create_time: u64, - ) -> Result, Box> - { + ) -> Result, Box> { let config_type = yaml_value .get(TYPE) .and_then(|v| v.as_str()) diff --git a/src/runtime/wasm/task/builder/sink/mod.rs b/src/wasm_runtime/src/wasm/task/builder/sink/mod.rs similarity index 95% rename from src/runtime/wasm/task/builder/sink/mod.rs rename to src/wasm_runtime/src/wasm/task/builder/sink/mod.rs index 65e8bc95..844b0aef 100644 --- a/src/runtime/wasm/task/builder/sink/mod.rs +++ b/src/wasm_runtime/src/wasm/task/builder/sink/mod.rs @@ -14,8 +14,8 @@ // // Specifically handles building logic for Sink type configuration (future support) -use crate::runtime::processor::wasm::wasm_task::WasmTask; -use crate::runtime::wasm::task::yaml_keys::{TYPE, type_values}; +use crate::processor::wasm::wasm_task::WasmTask; +use crate::wasm::task::yaml_keys::{TYPE, type_values}; use serde_yaml::Value; use std::sync::Arc; diff --git a/src/runtime/wasm/task/builder/source/mod.rs b/src/wasm_runtime/src/wasm/task/builder/source/mod.rs similarity index 95% rename from src/runtime/wasm/task/builder/source/mod.rs rename to src/wasm_runtime/src/wasm/task/builder/source/mod.rs index fc81bea9..938a5046 100644 --- a/src/runtime/wasm/task/builder/source/mod.rs +++ b/src/wasm_runtime/src/wasm/task/builder/source/mod.rs @@ -14,8 +14,8 @@ // // Specifically handles building logic for Source type configuration (future support) -use crate::runtime::processor::wasm::wasm_task::WasmTask; -use crate::runtime::wasm::task::yaml_keys::{TYPE, type_values}; +use crate::processor::wasm::wasm_task::WasmTask; +use crate::wasm::task::yaml_keys::{TYPE, type_values}; use serde_yaml::Value; use std::sync::Arc; diff --git a/src/runtime/wasm/task/builder/task_builder.rs b/src/wasm_runtime/src/wasm/task/builder/task_builder.rs similarity index 93% rename from src/runtime/wasm/task/builder/task_builder.rs rename to src/wasm_runtime/src/wasm/task/builder/task_builder.rs index 2246d6d8..9ca7236d 100644 --- a/src/runtime/wasm/task/builder/task_builder.rs +++ b/src/wasm_runtime/src/wasm/task/builder/task_builder.rs @@ -15,13 +15,13 @@ //! Provides unified factory methods to create TaskLifecycle instances from YAML config. //! Dispatches to specific builders (Processor, Source, Sink, Python) based on task type. -use crate::runtime::wasm::task::TaskLifecycle; -use crate::runtime::wasm::task::builder::processor::ProcessorBuilder; +use crate::wasm::task::TaskLifecycle; +use crate::wasm::task::builder::processor::ProcessorBuilder; #[cfg(feature = "python")] -use crate::runtime::wasm::task::builder::python::PythonBuilder; -use crate::runtime::wasm::task::builder::sink::SinkBuilder; -use crate::runtime::wasm::task::builder::source::SourceBuilder; -use crate::runtime::wasm::task::yaml_keys::{NAME, TYPE, type_values}; +use crate::wasm::task::builder::python::PythonBuilder; +use crate::wasm::task::builder::sink::SinkBuilder; +use crate::wasm::task::builder::source::SourceBuilder; +use crate::wasm::task::yaml_keys::{NAME, TYPE, type_values}; use serde_yaml::Value; use std::sync::Arc; @@ -169,7 +169,7 @@ impl TaskBuilder { /// Build and unwrap WASM task from Arc fn build_wasm_task( result: Result< - Arc, + Arc, Box, >, task_name: &str, diff --git a/src/runtime/wasm/task/control_mailbox.rs b/src/wasm_runtime/src/wasm/task/control_mailbox.rs similarity index 97% rename from src/runtime/wasm/task/control_mailbox.rs rename to src/wasm_runtime/src/wasm/task/control_mailbox.rs index 8aaf2de3..07397ac6 100644 --- a/src/runtime/wasm/task/control_mailbox.rs +++ b/src/wasm_runtime/src/wasm/task/control_mailbox.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::common::TaskCompletionFlag; -use crate::runtime::processor::function_error::FunctionErrorReport; +use crate::common::TaskCompletionFlag; +use crate::processor::function_error::FunctionErrorReport; use crossbeam_channel::Sender; #[derive(Clone)] diff --git a/src/runtime/wasm/task/lifecycle.rs b/src/wasm_runtime/src/wasm/task/lifecycle.rs similarity index 96% rename from src/runtime/wasm/task/lifecycle.rs rename to src/wasm_runtime/src/wasm/task/lifecycle.rs index ea00f7c2..cbe7955d 100644 --- a/src/runtime/wasm/task/lifecycle.rs +++ b/src/wasm_runtime/src/wasm/task/lifecycle.rs @@ -14,10 +14,10 @@ // // Defines the complete lifecycle management interface for Task, including initialization, start, stop, checkpoint and close -use crate::runtime::common::ComponentState; -use crate::runtime::wasm::task::control_mailbox::ControlMailBox; -use crate::runtime::wasm::taskexecutor::InitContext; -use crate::storage::task::FunctionInfo; +use crate::common::ComponentState; +use crate::task::FunctionInfo; +use crate::wasm::task::control_mailbox::ControlMailBox; +use crate::wasm::taskexecutor::InitContext; use std::sync::Arc; /// Task lifecycle management interface diff --git a/src/runtime/wasm/task/mod.rs b/src/wasm_runtime/src/wasm/task/mod.rs similarity index 100% rename from src/runtime/wasm/task/mod.rs rename to src/wasm_runtime/src/wasm/task/mod.rs diff --git a/src/runtime/wasm/task/processor_config.rs b/src/wasm_runtime/src/wasm/task/processor_config.rs similarity index 99% rename from src/runtime/wasm/task/processor_config.rs rename to src/wasm_runtime/src/wasm/task/processor_config.rs index a3069adc..8e642ed6 100644 --- a/src/runtime/wasm/task/processor_config.rs +++ b/src/wasm_runtime/src/wasm/task/processor_config.rs @@ -608,7 +608,7 @@ impl WasmTaskConfig { task_name: String, value: &Value, ) -> Result> { - use crate::runtime::wasm::task::yaml_keys::{INPUT_GROUPS, INPUTS, NAME, OUTPUTS}; + use crate::wasm::task::yaml_keys::{INPUT_GROUPS, INPUTS, NAME, OUTPUTS}; // 1. Get name from config (if exists), otherwise use the passed task_name let config_name = value diff --git a/src/runtime/wasm/task/yaml_keys.rs b/src/wasm_runtime/src/wasm/task/yaml_keys.rs similarity index 100% rename from src/runtime/wasm/task/yaml_keys.rs rename to src/wasm_runtime/src/wasm/task/yaml_keys.rs diff --git a/src/runtime/wasm/taskexecutor/init_context.rs b/src/wasm_runtime/src/wasm/taskexecutor/init_context.rs similarity index 90% rename from src/runtime/wasm/taskexecutor/init_context.rs rename to src/wasm_runtime/src/wasm/taskexecutor/init_context.rs index fca44a32..edd8837d 100644 --- a/src/runtime/wasm/taskexecutor/init_context.rs +++ b/src/wasm_runtime/src/wasm/taskexecutor/init_context.rs @@ -14,10 +14,10 @@ // // Provides various resources needed for task initialization, including state storage, task storage, thread pool, etc. -use crate::runtime::processor::wasm::thread_pool::{TaskThreadPool, ThreadGroup}; -use crate::runtime::wasm::task::ControlMailBox; -use crate::storage::state_backend::StateStorageServer; -use crate::storage::task::TaskStorage; +use crate::processor::wasm::thread_pool::{TaskThreadPool, ThreadGroup}; +use crate::state_backend::StateStorageServer; +use crate::task::TaskStorage; +use crate::wasm::task::ControlMailBox; use std::sync::{Arc, Mutex}; #[derive(Clone)] diff --git a/src/runtime/wasm/taskexecutor/mod.rs b/src/wasm_runtime/src/wasm/taskexecutor/mod.rs similarity index 100% rename from src/runtime/wasm/taskexecutor/mod.rs rename to src/wasm_runtime/src/wasm/taskexecutor/mod.rs diff --git a/src/runtime/wasm/taskexecutor/task_manager.rs b/src/wasm_runtime/src/wasm/taskexecutor/task_manager.rs similarity index 96% rename from src/runtime/wasm/taskexecutor/task_manager.rs rename to src/wasm_runtime/src/wasm/taskexecutor/task_manager.rs index 897e0a3d..d88c3df4 100644 --- a/src/runtime/wasm/taskexecutor/task_manager.rs +++ b/src/wasm_runtime/src/wasm/taskexecutor/task_manager.rs @@ -10,15 +10,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::common::ComponentState; use crate::config::GlobalConfig; -use crate::runtime::common::ComponentState; -use crate::runtime::processor::wasm::thread_pool::{GlobalTaskThreadPool, TaskThreadPool}; -use crate::runtime::wasm::task::{TaskBuilder, TaskLifecycle}; -use crate::runtime::wasm::taskexecutor::init_context::InitContext; -use crate::storage::state_backend::StateStorageServer; -use crate::storage::task::{ - FunctionInfo, StoredTaskInfo, TaskModuleBytes, TaskStorage, TaskStorageFactory, -}; +use crate::processor::wasm::thread_pool::{GlobalTaskThreadPool, TaskThreadPool}; +use crate::state_backend::StateStorageServer; +use crate::task::{FunctionInfo, StoredTaskInfo, TaskModuleBytes, TaskStorage, TaskStorageFactory}; +use crate::wasm::task::{TaskBuilder, TaskLifecycle}; +use crate::wasm::taskexecutor::init_context::InitContext; use anyhow::{Context, Result, anyhow}; use parking_lot::RwLock; From b2454bd04f475dc1e0a1a2419b5213f6afa22691 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Mon, 18 May 2026 23:13:03 +0800 Subject: [PATCH 02/16] update --- Cargo.lock | 89 +++++++++++++++++++ Cargo.toml | 3 + src/function-stream/Cargo.toml | 1 + src/function-stream/src/lib.rs | 8 +- src/function-stream/src/main.rs | 8 +- src/servicer/src/initializer.rs | 21 ++++- src/streaming_planner/src/schema/mod.rs | 2 +- src/streaming_runtime/Cargo.toml | 5 ++ src/streaming_runtime/src/lib.rs | 13 +-- src/streaming_runtime_core/Cargo.toml | 30 +++++++ .../src}/api/context.rs | 8 +- .../src}/api/mod.rs | 0 .../src}/api/operator.rs | 6 +- .../src}/api/source.rs | 2 +- .../src}/error.rs | 0 .../src}/execution/mod.rs | 0 .../src}/execution/operator_chain.rs | 8 +- .../src}/execution/pipeline.rs | 14 +-- .../src}/execution/source_driver.rs | 10 +-- .../src}/execution/tracker/barrier_aligner.rs | 0 .../src}/execution/tracker/mod.rs | 0 .../execution/tracker/watermark_tracker.rs | 2 +- .../src/lib.rs} | 10 +-- .../src}/network/endpoint.rs | 4 +- .../src}/network/environment.rs | 0 .../src}/network/mod.rs | 0 .../src}/protocol/control.rs | 0 .../src}/protocol/event.rs | 0 .../src}/protocol/mod.rs | 0 .../src}/state/error.rs | 0 .../src}/state/io_manager.rs | 0 .../src}/state/metrics.rs | 0 .../src}/state/mod.rs | 0 .../src}/state/operator_state.rs | 0 src/streaming_runtime_job_manager/Cargo.toml | 20 +++++ .../src}/job/edge_manager.rs | 2 +- .../src}/job/job_manager.rs | 28 +++--- .../src}/job/mod.rs | 5 +- .../src}/job/models.rs | 2 +- src/streaming_runtime_job_manager/src/lib.rs | 25 ++++++ .../src/stream_catalog.rs | 49 ++++++++++ src/streaming_runtime_operators/Cargo.toml | 47 ++++++++++ .../src}/factory/connector/delta.rs | 12 +-- .../src}/factory/connector/dispatchers.rs | 6 +- .../src}/factory/connector/filesystem.rs | 10 +-- .../src}/factory/connector/iceberg.rs | 12 +-- .../src}/factory/connector/kafka.rs | 14 +-- .../src}/factory/connector/lancedb.rs | 10 +-- .../src}/factory/connector/mod.rs | 0 .../src}/factory/connector/s3.rs | 10 +-- .../factory/connector/sink_props_codec.rs | 0 .../src}/factory/global/mod.rs | 0 .../src}/factory/global/session_registry.rs | 0 .../src}/factory/mod.rs | 0 .../src}/factory/operator_constructor.rs | 4 +- .../src}/factory/operator_factory.rs | 18 ++-- .../src}/format/config.rs | 0 .../src}/format/deserializer.rs | 0 .../src}/format/encoder.rs | 0 .../src}/format/json_encoder.rs | 0 .../src}/format/mod.rs | 0 .../src}/format/serializer.rs | 0 src/streaming_runtime_operators/src/lib.rs | 26 ++++++ .../grouping/incremental_aggregate.rs | 12 +-- .../src}/operators/grouping/mod.rs | 0 .../src}/operators/grouping/updating_cache.rs | 0 .../src}/operators/joins/join_instance.rs | 10 +-- .../operators/joins/join_with_expiration.rs | 10 +-- .../src}/operators/joins/mod.rs | 0 .../src}/operators/key_by.rs | 6 +- .../src}/operators/key_operator.rs | 10 +-- .../src}/operators/mod.rs | 0 .../src}/operators/projection.rs | 8 +- .../src}/operators/sink/delta/mod.rs | 8 +- .../src}/operators/sink/filesystem/mod.rs | 8 +- .../src}/operators/sink/iceberg/mod.rs | 8 +- .../src}/operators/sink/kafka/mod.rs | 8 +- .../src}/operators/sink/lancedb/mod.rs | 6 +- .../src}/operators/sink/mod.rs | 0 .../src}/operators/sink/s3/mod.rs | 6 +- .../src}/operators/source/kafka/mod.rs | 12 ++- .../src}/operators/source/mod.rs | 0 .../operators/stateless_physical_executor.rs | 2 +- .../src}/operators/value_execution.rs | 8 +- .../src}/operators/watermark/mod.rs | 0 .../watermark/watermark_generator.rs | 8 +- .../src}/operators/windows/mod.rs | 0 .../windows/session_aggregating_window.rs | 11 ++- .../windows/sliding_aggregating_window.rs | 10 +-- .../windows/tumbling_aggregating_window.rs | 10 +-- .../src}/operators/windows/window_function.rs | 10 +-- .../src/util/mod.rs | 0 .../src/util/physical_aggregate.rs | 0 93 files changed, 513 insertions(+), 202 deletions(-) create mode 100644 src/streaming_runtime_core/Cargo.toml rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/api/context.rs (97%) rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/api/mod.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/api/operator.rs (94%) rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/api/source.rs (98%) rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/error.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/execution/mod.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/execution/operator_chain.rs (98%) rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/execution/pipeline.rs (95%) rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/execution/source_driver.rs (96%) rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/execution/tracker/barrier_aligner.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/execution/tracker/mod.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/execution/tracker/watermark_tracker.rs (97%) rename src/{streaming_runtime/src/streaming/mod.rs => streaming_runtime_core/src/lib.rs} (81%) rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/network/endpoint.rs (95%) rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/network/environment.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/network/mod.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/protocol/control.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/protocol/event.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/protocol/mod.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/state/error.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/state/io_manager.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/state/metrics.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/state/mod.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_core/src}/state/operator_state.rs (100%) create mode 100644 src/streaming_runtime_job_manager/Cargo.toml rename src/{streaming_runtime/src/streaming => streaming_runtime_job_manager/src}/job/edge_manager.rs (98%) rename src/{streaming_runtime/src/streaming => streaming_runtime_job_manager/src}/job/job_manager.rs (97%) rename src/{streaming_runtime/src/streaming => streaming_runtime_job_manager/src}/job/mod.rs (75%) rename src/{streaming_runtime/src/streaming => streaming_runtime_job_manager/src}/job/models.rs (98%) create mode 100644 src/streaming_runtime_job_manager/src/lib.rs create mode 100644 src/streaming_runtime_job_manager/src/stream_catalog.rs create mode 100644 src/streaming_runtime_operators/Cargo.toml rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/factory/connector/delta.rs (88%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/factory/connector/dispatchers.rs (93%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/factory/connector/filesystem.rs (91%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/factory/connector/iceberg.rs (87%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/factory/connector/kafka.rs (95%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/factory/connector/lancedb.rs (93%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/factory/connector/mod.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/factory/connector/s3.rs (90%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/factory/connector/sink_props_codec.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/factory/global/mod.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/factory/global/session_registry.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/factory/mod.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/factory/operator_constructor.rs (88%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/factory/operator_factory.rs (94%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/format/config.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/format/deserializer.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/format/encoder.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/format/json_encoder.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/format/mod.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/format/serializer.rs (100%) create mode 100644 src/streaming_runtime_operators/src/lib.rs rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/grouping/incremental_aggregate.rs (99%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/grouping/mod.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/grouping/updating_cache.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/joins/join_instance.rs (98%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/joins/join_with_expiration.rs (98%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/joins/mod.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/key_by.rs (97%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/key_operator.rs (94%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/mod.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/projection.rs (95%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/sink/delta/mod.rs (97%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/sink/filesystem/mod.rs (97%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/sink/iceberg/mod.rs (97%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/sink/kafka/mod.rs (98%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/sink/lancedb/mod.rs (97%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/sink/mod.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/sink/s3/mod.rs (98%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/source/kafka/mod.rs (98%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/source/mod.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/stateless_physical_executor.rs (98%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/value_execution.rs (91%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/watermark/mod.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/watermark/watermark_generator.rs (97%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/windows/mod.rs (100%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/windows/session_aggregating_window.rs (99%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/windows/sliding_aggregating_window.rs (99%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/windows/tumbling_aggregating_window.rs (98%) rename src/{streaming_runtime/src/streaming => streaming_runtime_operators/src}/operators/windows/window_function.rs (97%) rename src/{streaming_runtime => streaming_runtime_operators}/src/util/mod.rs (100%) rename src/{streaming_runtime => streaming_runtime_operators}/src/util/physical_aggregate.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index a1176666..fbb2e117 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3804,6 +3804,7 @@ dependencies = [ "function-stream-logger", "function-stream-runtime-common", "function-stream-streaming-planner", + "function-stream-streaming-runtime", "futures", "governor", "itertools 0.14.0", @@ -3967,6 +3968,94 @@ dependencies = [ [[package]] name = "function-stream-streaming-runtime" version = "0.6.0" +dependencies = [ + "function-stream-streaming-runtime-core", + "function-stream-streaming-runtime-job-manager", + "function-stream-streaming-runtime-operators", +] + +[[package]] +name = "function-stream-streaming-runtime-core" +version = "0.6.0" +dependencies = [ + "anyhow", + "arrow 55.2.0", + "arrow-array 55.2.0", + "arrow-schema 55.2.0", + "async-trait", + "crossbeam-channel", + "function-stream-runtime-common", + "function-stream-streaming-planner", + "parking_lot", + "parquet", + "protocol", + "serde", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tracing", + "uuid", +] + +[[package]] +name = "function-stream-streaming-runtime-job-manager" +version = "0.6.0" +dependencies = [ + "anyhow", + "function-stream-config", + "function-stream-runtime-common", + "function-stream-streaming-planner", + "function-stream-streaming-runtime-core", + "function-stream-streaming-runtime-operators", + "protocol", + "tokio", + "tokio-stream", + "tracing", +] + +[[package]] +name = "function-stream-streaming-runtime-operators" +version = "0.6.0" +dependencies = [ + "ahash", + "anyhow", + "apache-avro", + "arrow 55.2.0", + "arrow-array 55.2.0", + "arrow-array 57.3.0", + "arrow-ipc 55.2.0", + "arrow-ipc 57.3.0", + "arrow-json 55.2.0", + "arrow-schema 55.2.0", + "async-trait", + "base64", + "bincode", + "bytes", + "datafusion 48.0.1", + "datafusion-common 48.0.1", + "datafusion-execution 48.0.1", + "datafusion-expr 48.0.1", + "datafusion-physical-expr 48.0.1", + "datafusion-proto", + "function-stream-config", + "function-stream-runtime-common", + "function-stream-streaming-planner", + "function-stream-streaming-runtime-core", + "futures", + "governor", + "itertools 0.14.0", + "lance", + "object_store", + "parquet", + "prost 0.13.5", + "protocol", + "rdkafka", + "serde", + "serde_json", + "tokio", + "tracing", +] [[package]] name = "function-stream-wasm-runtime" diff --git a/Cargo.toml b/Cargo.toml index 324c882b..eb03560f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,9 @@ members = [ "src/runtime_common", "src/servicer", "src/sqlparser", + "src/streaming_runtime_core", + "src/streaming_runtime_operators", + "src/streaming_runtime_job_manager", "src/streaming_runtime", "src/streaming_planner", "src/wasm_runtime", diff --git a/src/function-stream/Cargo.toml b/src/function-stream/Cargo.toml index baf9d5ea..184686f7 100644 --- a/src/function-stream/Cargo.toml +++ b/src/function-stream/Cargo.toml @@ -30,6 +30,7 @@ protocol = { path = "../../protocol" } function-stream-config = { path = "../config" } function-stream-logger = { path = "../logger" } function-stream-runtime-common = { path = "../runtime_common" } +function-stream-streaming-runtime = { path = "../streaming_runtime" } function-stream-streaming-planner = { path = "../streaming_planner" } prost = "0.13" rdkafka = { version = "0.38", features = ["cmake-build", "ssl", "gssapi", "curl"] } diff --git a/src/function-stream/src/lib.rs b/src/function-stream/src/lib.rs index a3fc9234..1731c4cc 100644 --- a/src/function-stream/src/lib.rs +++ b/src/function-stream/src/lib.rs @@ -24,12 +24,8 @@ pub mod coordinator; pub use function_stream_logger as logging; pub use function_stream_runtime_common::{common, memory}; - -#[path = "../../streaming_runtime/src/streaming/mod.rs"] -pub mod streaming; - -#[path = "../../streaming_runtime/src/util/mod.rs"] -pub mod util; +pub use function_stream_streaming_runtime as streaming; +pub use function_stream_streaming_runtime::util; #[path = "../../wasm_runtime/src/wasm/mod.rs"] pub mod wasm; diff --git a/src/function-stream/src/main.rs b/src/function-stream/src/main.rs index b7e1093f..665be071 100644 --- a/src/function-stream/src/main.rs +++ b/src/function-stream/src/main.rs @@ -18,17 +18,13 @@ mod coordinator; pub use function_stream_logger as logging; pub use function_stream_runtime_common::{common, memory}; +pub use function_stream_streaming_runtime as streaming; +pub use function_stream_streaming_runtime::util; use std::sync::Arc; use anyhow::{Context, Result}; -#[path = "../../streaming_runtime/src/streaming/mod.rs"] -mod streaming; - -#[path = "../../streaming_runtime/src/util/mod.rs"] -mod util; - #[path = "../../wasm_runtime/src/wasm/mod.rs"] mod wasm; diff --git a/src/servicer/src/initializer.rs b/src/servicer/src/initializer.rs index 78b16c73..8f95cc1a 100644 --- a/src/servicer/src/initializer.rs +++ b/src/servicer/src/initializer.rs @@ -170,9 +170,26 @@ fn initialize_memory_service(config: &GlobalConfig) -> Result<()> { fn initialize_job_manager(config: &GlobalConfig) -> Result<()> { use crate::streaming::factory::OperatorFactory; use crate::streaming::factory::Registry; - use crate::streaming::job::{JobManager, StateConfig}; + use crate::streaming::job::{ + CheckpointCatalog, JobManager, StateConfig, install_global_checkpoint_catalog, + }; use std::sync::Arc; + struct StreamCatalogCheckpointStore; + + impl CheckpointCatalog for StreamCatalogCheckpointStore { + fn commit_job_checkpoint( + &self, + job_id: &str, + epoch: u64, + source_infos: Vec, + ) -> Result<()> { + let catalog = crate::stream_catalog::CatalogManager::global() + .context("Checkpoint commit requires StreamCatalog")?; + Ok(catalog.commit_job_checkpoint(job_id, epoch, source_infos)?) + } + } + let per_operator_memory_bytes = config .streaming .operator_state_store_memory_bytes @@ -181,6 +198,8 @@ fn initialize_job_manager(config: &GlobalConfig) -> Result<()> { let registry = Arc::new(Registry::new()); let factory = Arc::new(OperatorFactory::new(registry)); + install_global_checkpoint_catalog(Arc::new(StreamCatalogCheckpointStore)) + .context("JobManager checkpoint catalog bridge install failed")?; let state_base_dir = std::env::temp_dir().join("function-stream").join("state"); let state_config = StateConfig { diff --git a/src/streaming_planner/src/schema/mod.rs b/src/streaming_planner/src/schema/mod.rs index 75546636..d383a205 100644 --- a/src/streaming_planner/src/schema/mod.rs +++ b/src/streaming_planner/src/schema/mod.rs @@ -24,7 +24,7 @@ pub mod utils; pub use catalog::{ExternalTable, LookupTable, SinkTable, SourceTable}; pub use column_descriptor::ColumnDescriptor; pub use introspection::{ - catalog_table_row_detail, schema_columns_one_line, show_create_catalog_table, DdlCompiler, + DdlCompiler, catalog_table_row_detail, schema_columns_one_line, show_create_catalog_table, try_compile_connector_create_table, }; pub use schema_provider::{ObjectName, StreamPlanningContext, StreamSchemaProvider, StreamTable}; diff --git a/src/streaming_runtime/Cargo.toml b/src/streaming_runtime/Cargo.toml index 026b50e1..061c5850 100644 --- a/src/streaming_runtime/Cargo.toml +++ b/src/streaming_runtime/Cargo.toml @@ -6,3 +6,8 @@ edition.workspace = true [lib] name = "function_stream_streaming_runtime" path = "src/lib.rs" + +[dependencies] +function-stream-streaming-runtime-core = { path = "../streaming_runtime_core" } +function-stream-streaming-runtime-job-manager = { path = "../streaming_runtime_job_manager" } +function-stream-streaming-runtime-operators = { path = "../streaming_runtime_operators" } diff --git a/src/streaming_runtime/src/lib.rs b/src/streaming_runtime/src/lib.rs index aeb056d5..801b25e1 100644 --- a/src/streaming_runtime/src/lib.rs +++ b/src/streaming_runtime/src/lib.rs @@ -10,11 +10,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Streaming execution runtime. -//! -//! The streaming engine and shared runtime helpers (`streaming/`, `util/`) are -//! implemented under [`src/streaming`] and [`src/util`] in this package. They are -//! currently **compiled as part of the `function-stream` crate** via `#[path]` in -//! `src/lib.rs` / `src/main.rs`, sharing the root `crate::sql` name (re-exported streaming planner crate). +//! Facade crate for the split streaming runtime libraries. + +pub use function_stream_streaming_runtime_core::{ + StreamOutput, api, error, execution, network, protocol, state, +}; +pub use function_stream_streaming_runtime_job_manager::{job, stream_catalog}; +pub use function_stream_streaming_runtime_operators::{factory, format, operators, util}; pub const CRATE_NAME: &str = "function-stream-streaming-runtime"; diff --git a/src/streaming_runtime_core/Cargo.toml b/src/streaming_runtime_core/Cargo.toml new file mode 100644 index 00000000..16b76104 --- /dev/null +++ b/src/streaming_runtime_core/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "function-stream-streaming-runtime-core" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_streaming_runtime_core" +path = "src/lib.rs" + +[dependencies] +anyhow = "1.0" +async-trait = "0.1" +arrow = { version = "55", default-features = false } +arrow-array = "55" +arrow-schema = { version = "55", features = ["serde"] } +crossbeam-channel = "0.5" +function-stream-runtime-common = { path = "../runtime_common" } +function-stream-streaming-planner = { path = "../streaming_planner" } +parking_lot = "0.12" +parquet = "55" +protocol = { path = "../../protocol" } +serde = { version = "1.0", features = ["derive"] } +thiserror = "2" +tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "sync", "time", "net", "signal"] } +tokio-stream = "0.1.18" +tracing = "0.1" +uuid = { version = "1.0", features = ["v7"] } + +[dev-dependencies] +tempfile = "3.27.0" diff --git a/src/streaming_runtime/src/streaming/api/context.rs b/src/streaming_runtime_core/src/api/context.rs similarity index 97% rename from src/streaming_runtime/src/streaming/api/context.rs rename to src/streaming_runtime_core/src/api/context.rs index a29d7ac2..48d7f337 100644 --- a/src/streaming_runtime/src/streaming/api/context.rs +++ b/src/streaming_runtime_core/src/api/context.rs @@ -20,10 +20,10 @@ use protocol::storage::SourceCheckpointInfo; use tokio::sync::mpsc; use crate::memory::{MemoryBlock, MemoryPool, get_array_memory_size}; -use crate::streaming::network::endpoint::PhysicalSender; -use crate::streaming::protocol::control::JobMasterEvent; -use crate::streaming::protocol::event::{StreamEvent, TrackedEvent}; -use crate::streaming::state::IoManager; +use crate::network::endpoint::PhysicalSender; +use crate::protocol::control::JobMasterEvent; +use crate::protocol::event::{StreamEvent, TrackedEvent}; +use crate::state::IoManager; #[derive(Debug, Clone)] pub struct TaskContextConfig { diff --git a/src/streaming_runtime/src/streaming/api/mod.rs b/src/streaming_runtime_core/src/api/mod.rs similarity index 100% rename from src/streaming_runtime/src/streaming/api/mod.rs rename to src/streaming_runtime_core/src/api/mod.rs diff --git a/src/streaming_runtime/src/streaming/api/operator.rs b/src/streaming_runtime_core/src/api/operator.rs similarity index 94% rename from src/streaming_runtime/src/streaming/api/operator.rs rename to src/streaming_runtime_core/src/api/operator.rs index 56cefcee..c58c41e3 100644 --- a/src/streaming_runtime/src/streaming/api/operator.rs +++ b/src/streaming_runtime_core/src/api/operator.rs @@ -11,9 +11,9 @@ // limitations under the License. use crate::sql::common::{CheckpointBarrier, Watermark}; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::source::SourceOperator; -use crate::streaming::protocol::event::StreamOutput; +use crate::api::context::TaskContext; +use crate::api::source::SourceOperator; +use crate::protocol::event::StreamOutput; use arrow_array::RecordBatch; use async_trait::async_trait; diff --git a/src/streaming_runtime/src/streaming/api/source.rs b/src/streaming_runtime_core/src/api/source.rs similarity index 98% rename from src/streaming_runtime/src/streaming/api/source.rs rename to src/streaming_runtime_core/src/api/source.rs index 4bed912f..3c2445e4 100644 --- a/src/streaming_runtime/src/streaming/api/source.rs +++ b/src/streaming_runtime_core/src/api/source.rs @@ -11,7 +11,7 @@ // limitations under the License. use crate::sql::common::{CheckpointBarrier, Watermark}; -use crate::streaming::api::context::TaskContext; +use crate::api::context::TaskContext; use arrow_array::RecordBatch; use async_trait::async_trait; use protocol::storage::{ diff --git a/src/streaming_runtime/src/streaming/error.rs b/src/streaming_runtime_core/src/error.rs similarity index 100% rename from src/streaming_runtime/src/streaming/error.rs rename to src/streaming_runtime_core/src/error.rs diff --git a/src/streaming_runtime/src/streaming/execution/mod.rs b/src/streaming_runtime_core/src/execution/mod.rs similarity index 100% rename from src/streaming_runtime/src/streaming/execution/mod.rs rename to src/streaming_runtime_core/src/execution/mod.rs diff --git a/src/streaming_runtime/src/streaming/execution/operator_chain.rs b/src/streaming_runtime_core/src/execution/operator_chain.rs similarity index 98% rename from src/streaming_runtime/src/streaming/execution/operator_chain.rs rename to src/streaming_runtime_core/src/execution/operator_chain.rs index 38584c5d..f540cc6b 100644 --- a/src/streaming_runtime/src/streaming/execution/operator_chain.rs +++ b/src/streaming_runtime_core/src/execution/operator_chain.rs @@ -14,10 +14,10 @@ use anyhow::anyhow; use async_trait::async_trait; use crate::sql::common::CheckpointBarrier; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::operator::{Collector, Operator}; -use crate::streaming::error::RunError; -use crate::streaming::protocol::{ +use crate::api::context::TaskContext; +use crate::api::operator::{Collector, Operator}; +use crate::error::RunError; +use crate::protocol::{ control::{ControlCommand, StopMode}, event::{StreamEvent, StreamOutput, TrackedEvent}, }; diff --git a/src/streaming_runtime/src/streaming/execution/pipeline.rs b/src/streaming_runtime_core/src/execution/pipeline.rs similarity index 95% rename from src/streaming_runtime/src/streaming/execution/pipeline.rs rename to src/streaming_runtime_core/src/execution/pipeline.rs index 6b6b1878..57e67ca9 100644 --- a/src/streaming_runtime/src/streaming/execution/pipeline.rs +++ b/src/streaming_runtime_core/src/execution/pipeline.rs @@ -15,16 +15,16 @@ use tokio_stream::{StreamExt, StreamMap}; use tracing::{Instrument, info, info_span}; use crate::sql::common::Watermark; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::operator::Operator; -use crate::streaming::error::RunError; -use crate::streaming::execution::operator_chain::{ChainBuilder, OperatorDrive}; -use crate::streaming::execution::tracker::{ +use crate::api::context::TaskContext; +use crate::api::operator::Operator; +use crate::error::RunError; +use crate::execution::operator_chain::{ChainBuilder, OperatorDrive}; +use crate::execution::tracker::{ barrier_aligner::{AlignmentStatus, BarrierAligner}, watermark_tracker::WatermarkTracker, }; -use crate::streaming::network::endpoint::BoxedEventStream; -use crate::streaming::protocol::{ +use crate::network::endpoint::BoxedEventStream; +use crate::protocol::{ control::ControlCommand, event::{StreamEvent, TrackedEvent}, }; diff --git a/src/streaming_runtime/src/streaming/execution/source_driver.rs b/src/streaming_runtime_core/src/execution/source_driver.rs similarity index 96% rename from src/streaming_runtime/src/streaming/execution/source_driver.rs rename to src/streaming_runtime_core/src/execution/source_driver.rs index 72cc160e..315aa6dc 100644 --- a/src/streaming_runtime/src/streaming/execution/source_driver.rs +++ b/src/streaming_runtime_core/src/execution/source_driver.rs @@ -15,11 +15,11 @@ use tokio::time::{Instant, sleep}; use tracing::{Instrument, info, info_span, warn}; use crate::sql::common::CheckpointBarrier; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::source::{SourceCheckpointReport, SourceEvent, SourceOperator}; -use crate::streaming::error::RunError; -use crate::streaming::execution::OperatorDrive; -use crate::streaming::protocol::{ +use crate::api::context::TaskContext; +use crate::api::source::{SourceCheckpointReport, SourceEvent, SourceOperator}; +use crate::error::RunError; +use crate::execution::OperatorDrive; +use crate::protocol::{ control::ControlCommand, event::{StreamEvent, TrackedEvent}, }; diff --git a/src/streaming_runtime/src/streaming/execution/tracker/barrier_aligner.rs b/src/streaming_runtime_core/src/execution/tracker/barrier_aligner.rs similarity index 100% rename from src/streaming_runtime/src/streaming/execution/tracker/barrier_aligner.rs rename to src/streaming_runtime_core/src/execution/tracker/barrier_aligner.rs diff --git a/src/streaming_runtime/src/streaming/execution/tracker/mod.rs b/src/streaming_runtime_core/src/execution/tracker/mod.rs similarity index 100% rename from src/streaming_runtime/src/streaming/execution/tracker/mod.rs rename to src/streaming_runtime_core/src/execution/tracker/mod.rs diff --git a/src/streaming_runtime/src/streaming/execution/tracker/watermark_tracker.rs b/src/streaming_runtime_core/src/execution/tracker/watermark_tracker.rs similarity index 97% rename from src/streaming_runtime/src/streaming/execution/tracker/watermark_tracker.rs rename to src/streaming_runtime_core/src/execution/tracker/watermark_tracker.rs index ec44726d..17de1ec4 100644 --- a/src/streaming_runtime/src/streaming/execution/tracker/watermark_tracker.rs +++ b/src/streaming_runtime_core/src/execution/tracker/watermark_tracker.rs @@ -11,7 +11,7 @@ // limitations under the License. use crate::sql::common::Watermark; -use crate::streaming::protocol::event::{merge_watermarks, watermark_strictly_advances}; +use crate::protocol::event::{merge_watermarks, watermark_strictly_advances}; #[derive(Debug)] pub struct WatermarkTracker { diff --git a/src/streaming_runtime/src/streaming/mod.rs b/src/streaming_runtime_core/src/lib.rs similarity index 81% rename from src/streaming_runtime/src/streaming/mod.rs rename to src/streaming_runtime_core/src/lib.rs index b092c85d..89f0b07b 100644 --- a/src/streaming_runtime/src/streaming/mod.rs +++ b/src/streaming_runtime_core/src/lib.rs @@ -1,6 +1,5 @@ // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. -// // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 @@ -11,16 +10,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Streaming actor runtime (vendored from Arroyo `arroyo-actor-runtime`). +//! Core streaming runtime building blocks. + +pub use function_stream_runtime_common::{common, memory}; +pub use function_stream_streaming_planner as sql; pub mod api; pub mod error; pub mod execution; -pub mod factory; -pub mod format; -pub mod job; pub mod network; -pub mod operators; pub mod protocol; pub mod state; diff --git a/src/streaming_runtime/src/streaming/network/endpoint.rs b/src/streaming_runtime_core/src/network/endpoint.rs similarity index 95% rename from src/streaming_runtime/src/streaming/network/endpoint.rs rename to src/streaming_runtime_core/src/network/endpoint.rs index 12550977..10958228 100644 --- a/src/streaming_runtime/src/streaming/network/endpoint.rs +++ b/src/streaming_runtime_core/src/network/endpoint.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::streaming::protocol::event::StreamEvent; -use crate::streaming::protocol::event::TrackedEvent; +use crate::protocol::event::StreamEvent; +use crate::protocol::event::TrackedEvent; use anyhow::{Result, anyhow}; use std::pin::Pin; use tokio::sync::mpsc; diff --git a/src/streaming_runtime/src/streaming/network/environment.rs b/src/streaming_runtime_core/src/network/environment.rs similarity index 100% rename from src/streaming_runtime/src/streaming/network/environment.rs rename to src/streaming_runtime_core/src/network/environment.rs diff --git a/src/streaming_runtime/src/streaming/network/mod.rs b/src/streaming_runtime_core/src/network/mod.rs similarity index 100% rename from src/streaming_runtime/src/streaming/network/mod.rs rename to src/streaming_runtime_core/src/network/mod.rs diff --git a/src/streaming_runtime/src/streaming/protocol/control.rs b/src/streaming_runtime_core/src/protocol/control.rs similarity index 100% rename from src/streaming_runtime/src/streaming/protocol/control.rs rename to src/streaming_runtime_core/src/protocol/control.rs diff --git a/src/streaming_runtime/src/streaming/protocol/event.rs b/src/streaming_runtime_core/src/protocol/event.rs similarity index 100% rename from src/streaming_runtime/src/streaming/protocol/event.rs rename to src/streaming_runtime_core/src/protocol/event.rs diff --git a/src/streaming_runtime/src/streaming/protocol/mod.rs b/src/streaming_runtime_core/src/protocol/mod.rs similarity index 100% rename from src/streaming_runtime/src/streaming/protocol/mod.rs rename to src/streaming_runtime_core/src/protocol/mod.rs diff --git a/src/streaming_runtime/src/streaming/state/error.rs b/src/streaming_runtime_core/src/state/error.rs similarity index 100% rename from src/streaming_runtime/src/streaming/state/error.rs rename to src/streaming_runtime_core/src/state/error.rs diff --git a/src/streaming_runtime/src/streaming/state/io_manager.rs b/src/streaming_runtime_core/src/state/io_manager.rs similarity index 100% rename from src/streaming_runtime/src/streaming/state/io_manager.rs rename to src/streaming_runtime_core/src/state/io_manager.rs diff --git a/src/streaming_runtime/src/streaming/state/metrics.rs b/src/streaming_runtime_core/src/state/metrics.rs similarity index 100% rename from src/streaming_runtime/src/streaming/state/metrics.rs rename to src/streaming_runtime_core/src/state/metrics.rs diff --git a/src/streaming_runtime/src/streaming/state/mod.rs b/src/streaming_runtime_core/src/state/mod.rs similarity index 100% rename from src/streaming_runtime/src/streaming/state/mod.rs rename to src/streaming_runtime_core/src/state/mod.rs diff --git a/src/streaming_runtime/src/streaming/state/operator_state.rs b/src/streaming_runtime_core/src/state/operator_state.rs similarity index 100% rename from src/streaming_runtime/src/streaming/state/operator_state.rs rename to src/streaming_runtime_core/src/state/operator_state.rs diff --git a/src/streaming_runtime_job_manager/Cargo.toml b/src/streaming_runtime_job_manager/Cargo.toml new file mode 100644 index 00000000..af80046f --- /dev/null +++ b/src/streaming_runtime_job_manager/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "function-stream-streaming-runtime-job-manager" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_streaming_runtime_job_manager" +path = "src/lib.rs" + +[dependencies] +anyhow = "1.0" +function-stream-config = { path = "../config" } +function-stream-runtime-common = { path = "../runtime_common" } +function-stream-streaming-planner = { path = "../streaming_planner" } +function-stream-streaming-runtime-core = { path = "../streaming_runtime_core" } +function-stream-streaming-runtime-operators = { path = "../streaming_runtime_operators" } +protocol = { path = "../../protocol" } +tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "sync", "time", "net", "signal"] } +tokio-stream = "0.1.18" +tracing = "0.1" diff --git a/src/streaming_runtime/src/streaming/job/edge_manager.rs b/src/streaming_runtime_job_manager/src/job/edge_manager.rs similarity index 98% rename from src/streaming_runtime/src/streaming/job/edge_manager.rs rename to src/streaming_runtime_job_manager/src/job/edge_manager.rs index 32bb3722..0cf7e551 100644 --- a/src/streaming_runtime/src/streaming/job/edge_manager.rs +++ b/src/streaming_runtime_job_manager/src/job/edge_manager.rs @@ -16,7 +16,7 @@ use anyhow::{Result, anyhow}; use tokio::sync::mpsc; use tracing::{debug, info, warn}; -use crate::streaming::protocol::event::TrackedEvent; +use crate::core::protocol::event::TrackedEvent; use protocol::function_stream_graph::{FsEdge, FsNode}; const DEFAULT_CHANNEL_CAPACITY: usize = 2048; diff --git a/src/streaming_runtime/src/streaming/job/job_manager.rs b/src/streaming_runtime_job_manager/src/job/job_manager.rs similarity index 97% rename from src/streaming_runtime/src/streaming/job/job_manager.rs rename to src/streaming_runtime_job_manager/src/job/job_manager.rs index c4c0a826..0f708ebf 100644 --- a/src/streaming_runtime/src/streaming/job/job_manager.rs +++ b/src/streaming_runtime_job_manager/src/job/job_manager.rs @@ -33,19 +33,19 @@ use crate::config::{ use crate::memory::global_memory_pool; use crate::sql::logical_node::logical::OperatorName; use crate::stream_catalog::CatalogManager; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::operator::{ConstructedOperator, Operator}; -use crate::streaming::api::source::SourceOperator; -use crate::streaming::execution::{ChainBuilder, Pipeline, SourceDriver}; -use crate::streaming::factory::OperatorFactory; -use crate::streaming::job::edge_manager::EdgeManager; -use crate::streaming::job::models::{ +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{ConstructedOperator, Operator}; +use crate::core::api::source::SourceOperator; +use crate::core::execution::{ChainBuilder, Pipeline, SourceDriver}; +use crate::factory::OperatorFactory; +use crate::job::edge_manager::EdgeManager; +use crate::job::models::{ PhysicalExecutionGraph, PhysicalPipeline, PipelineStatus, StreamingJobRollupStatus, }; -use crate::streaming::network::endpoint::{BoxedEventStream, PhysicalSender}; -use crate::streaming::protocol::control::{ControlCommand, JobMasterEvent, StopMode}; -use crate::streaming::protocol::event::CheckpointBarrier; -use crate::streaming::state::{IoManager, IoPool, NoopMetricsCollector}; +use crate::core::network::endpoint::{BoxedEventStream, PhysicalSender}; +use crate::core::protocol::control::{ControlCommand, JobMasterEvent, StopMode}; +use crate::core::protocol::event::CheckpointBarrier; +use crate::core::state::{IoManager, IoPool, NoopMetricsCollector}; #[derive(Debug, Clone)] pub struct StreamingJobSummary { @@ -80,7 +80,7 @@ pub struct StateConfig { pub pipeline_parallelism: u32, pub job_manager_control_plane_threads: u32, pub job_manager_data_plane_threads: u32, - /// Total bytes shared by all [`crate::streaming::state::OperatorStateStore`] (global pool). + /// Total bytes shared by all [`crate::core::state::OperatorStateStore`] (global pool). pub per_operator_memory_bytes: u64, } @@ -103,7 +103,7 @@ impl Default for StateConfig { static GLOBAL_JOB_MANAGER: OnceLock> = OnceLock::new(); -/// Operators that create an [`crate::streaming::state::OperatorStateStore`] at runtime. +/// Operators that create an [`crate::core::state::OperatorStateStore`] at runtime. fn pipeline_state_store_operator_count(operators: &[ChainedOperator]) -> usize { operators .iter() @@ -159,7 +159,7 @@ struct CheckpointCoordinatorConfig { } impl PipelineRunner { - async fn run(self) -> Result<(), crate::streaming::error::RunError> { + async fn run(self) -> Result<(), crate::core::error::RunError> { match self { PipelineRunner::Source(driver) => driver.run().await, PipelineRunner::Standard(pipeline) => pipeline.run().await, diff --git a/src/streaming_runtime/src/streaming/job/mod.rs b/src/streaming_runtime_job_manager/src/job/mod.rs similarity index 75% rename from src/streaming_runtime/src/streaming/job/mod.rs rename to src/streaming_runtime_job_manager/src/job/mod.rs index 59d5c61f..eaf1cc72 100644 --- a/src/streaming_runtime/src/streaming/job/mod.rs +++ b/src/streaming_runtime_job_manager/src/job/mod.rs @@ -14,4 +14,7 @@ pub mod edge_manager; pub mod job_manager; pub mod models; -pub use job_manager::{JobManager, StateConfig, StreamingJobSummary}; +pub use crate::stream_catalog::{CheckpointCatalog, install_global_checkpoint_catalog}; +pub use job_manager::{ + JobManager, PipelineDetail, StateConfig, StreamingJobDetail, StreamingJobSummary, +}; diff --git a/src/streaming_runtime/src/streaming/job/models.rs b/src/streaming_runtime_job_manager/src/job/models.rs similarity index 98% rename from src/streaming_runtime/src/streaming/job/models.rs rename to src/streaming_runtime_job_manager/src/job/models.rs index a6d226bd..3f8ce8c4 100644 --- a/src/streaming_runtime/src/streaming/job/models.rs +++ b/src/streaming_runtime_job_manager/src/job/models.rs @@ -19,7 +19,7 @@ use protocol::function_stream_graph::FsProgram; use tokio::sync::mpsc; use tokio::task::JoinHandle; -use crate::streaming::protocol::control::ControlCommand; +use crate::core::protocol::control::ControlCommand; #[derive(Debug, Clone, PartialEq)] pub enum PipelineStatus { diff --git a/src/streaming_runtime_job_manager/src/lib.rs b/src/streaming_runtime_job_manager/src/lib.rs new file mode 100644 index 00000000..400b07c2 --- /dev/null +++ b/src/streaming_runtime_job_manager/src/lib.rs @@ -0,0 +1,25 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! JobManager crate for streaming runtime orchestration. + +pub use function_stream_config as config; +pub use function_stream_runtime_common::{common, memory}; +pub use function_stream_streaming_planner as sql; +pub use function_stream_streaming_runtime_core as core; +pub use function_stream_streaming_runtime_core::{ + StreamOutput, api, error, execution, network, protocol, state, +}; +pub use function_stream_streaming_runtime_operators::{factory, format, operators, util}; + +pub mod job; +pub mod stream_catalog; diff --git a/src/streaming_runtime_job_manager/src/stream_catalog.rs b/src/streaming_runtime_job_manager/src/stream_catalog.rs new file mode 100644 index 00000000..2dcab934 --- /dev/null +++ b/src/streaming_runtime_job_manager/src/stream_catalog.rs @@ -0,0 +1,49 @@ +use std::sync::{Arc, OnceLock}; + +use anyhow::{Result, anyhow}; +use protocol::storage::SourceCheckpointInfo; + +pub trait CheckpointCatalog: Send + Sync { + fn commit_job_checkpoint( + &self, + job_id: &str, + epoch: u64, + source_infos: Vec, + ) -> Result<()>; +} + +static GLOBAL_CHECKPOINT_CATALOG: OnceLock> = OnceLock::new(); + +pub fn install_global_checkpoint_catalog(catalog: Arc) -> Result<()> { + GLOBAL_CHECKPOINT_CATALOG + .set(catalog) + .map_err(|_| anyhow!("CheckpointCatalog singleton already initialized")) +} + +#[derive(Clone)] +pub struct CatalogHandle { + inner: Arc, +} + +pub struct CatalogManager; + +impl CatalogManager { + pub fn try_global() -> Option { + GLOBAL_CHECKPOINT_CATALOG + .get() + .cloned() + .map(|inner| CatalogHandle { inner }) + } +} + +impl CatalogHandle { + pub fn commit_job_checkpoint( + &self, + job_id: &str, + epoch: u64, + source_infos: Vec, + ) -> Result<()> { + self.inner + .commit_job_checkpoint(job_id, epoch, source_infos) + } +} diff --git a/src/streaming_runtime_operators/Cargo.toml b/src/streaming_runtime_operators/Cargo.toml new file mode 100644 index 00000000..d6949084 --- /dev/null +++ b/src/streaming_runtime_operators/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "function-stream-streaming-runtime-operators" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_streaming_runtime_operators" +path = "src/lib.rs" + +[dependencies] +ahash = "0.8" +anyhow = "1.0" +apache-avro = "0.21" +arrow = { version = "55", default-features = false } +arrow-array = "55" +arrow-array-lance = { package = "arrow-array", version = "57.3.0" } +arrow-ipc = "55" +arrow-ipc-lance = { package = "arrow-ipc", version = "57.3.0" } +arrow-json = { version = "55.2.0" } +arrow-schema = { version = "55", features = ["serde"] } +async-trait = "0.1" +base64 = "0.22" +bincode = { version = "2", features = ["serde"] } +bytes = "1" +datafusion = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-common = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-execution = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-expr = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-physical-expr = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-proto = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +function-stream-config = { path = "../config" } +function-stream-runtime-common = { path = "../runtime_common" } +function-stream-streaming-planner = { path = "../streaming_planner" } +function-stream-streaming-runtime-core = { path = "../streaming_runtime_core" } +futures = "0.3" +governor = "0.8.0" +itertools = "0.14" +lance = { version = "4.0.0", default-features = false, features = ["aws"] } +object_store = { version = "0.12.5", features = ["aws"] } +parquet = "55" +prost = "0.13" +protocol = { path = "../../protocol" } +rdkafka = { version = "0.38", features = ["cmake-build", "ssl", "gssapi", "curl"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "sync", "time", "net", "signal", "fs", "io-util"] } +tracing = "0.1" diff --git a/src/streaming_runtime/src/streaming/factory/connector/delta.rs b/src/streaming_runtime_operators/src/factory/connector/delta.rs similarity index 88% rename from src/streaming_runtime/src/streaming/factory/connector/delta.rs rename to src/streaming_runtime_operators/src/factory/connector/delta.rs index 0c684e94..688bca53 100644 --- a/src/streaming_runtime/src/streaming/factory/connector/delta.rs +++ b/src/streaming_runtime_operators/src/factory/connector/delta.rs @@ -20,14 +20,14 @@ use protocol::function_stream_graph::connector_op::Config; use crate::sql::common::constants::connection_format_value; use crate::sql::common::with_option_keys as opt; -use crate::streaming::api::operator::ConstructedOperator; -use crate::streaming::factory::connector::sink_props_codec::{ +use crate::core::api::operator::ConstructedOperator; +use crate::factory::connector::sink_props_codec::{ apply_common_sink_fields, normalized_props, parse_sink_memory_bytes, }; -use crate::streaming::factory::global::Registry; -use crate::streaming::factory::operator_constructor::OperatorConstructor; -use crate::streaming::operators::sink::delta::{DeltaFormat, DeltaSinkOperator}; -use crate::streaming::operators::sink::filesystem::compression_from_str; +use crate::factory::global::Registry; +use crate::factory::operator_constructor::OperatorConstructor; +use crate::operators::sink::delta::{DeltaFormat, DeltaSinkOperator}; +use crate::operators::sink::filesystem::compression_from_str; pub struct DeltaSinkDispatcher; diff --git a/src/streaming_runtime/src/streaming/factory/connector/dispatchers.rs b/src/streaming_runtime_operators/src/factory/connector/dispatchers.rs similarity index 93% rename from src/streaming_runtime/src/streaming/factory/connector/dispatchers.rs rename to src/streaming_runtime_operators/src/factory/connector/dispatchers.rs index bcd540a4..7b345dd2 100644 --- a/src/streaming_runtime/src/streaming/factory/connector/dispatchers.rs +++ b/src/streaming_runtime_operators/src/factory/connector/dispatchers.rs @@ -17,9 +17,9 @@ use prost::Message; use protocol::function_stream_graph::ConnectorOp; use crate::sql::common::constants::connector_type; -use crate::streaming::api::operator::ConstructedOperator; -use crate::streaming::factory::global::Registry; -use crate::streaming::factory::operator_constructor::OperatorConstructor; +use crate::core::api::operator::ConstructedOperator; +use crate::factory::global::Registry; +use crate::factory::operator_constructor::OperatorConstructor; use super::{ DeltaSinkDispatcher, FilesystemSinkDispatcher, IcebergSinkDispatcher, LanceDbSinkDispatcher, diff --git a/src/streaming_runtime/src/streaming/factory/connector/filesystem.rs b/src/streaming_runtime_operators/src/factory/connector/filesystem.rs similarity index 91% rename from src/streaming_runtime/src/streaming/factory/connector/filesystem.rs rename to src/streaming_runtime_operators/src/factory/connector/filesystem.rs index b680590c..bc7f9da1 100644 --- a/src/streaming_runtime/src/streaming/factory/connector/filesystem.rs +++ b/src/streaming_runtime_operators/src/factory/connector/filesystem.rs @@ -20,13 +20,13 @@ use protocol::function_stream_graph::connector_op::Config; use crate::sql::common::constants::connection_format_value; use crate::sql::common::with_option_keys as opt; -use crate::streaming::api::operator::ConstructedOperator; -use crate::streaming::factory::connector::sink_props_codec::{ +use crate::core::api::operator::ConstructedOperator; +use crate::factory::connector::sink_props_codec::{ apply_common_sink_fields, normalized_props, parse_sink_memory_bytes, }; -use crate::streaming::factory::global::Registry; -use crate::streaming::factory::operator_constructor::OperatorConstructor; -use crate::streaming::operators::sink::filesystem::{ +use crate::factory::global::Registry; +use crate::factory::operator_constructor::OperatorConstructor; +use crate::operators::sink::filesystem::{ FilesystemFormat, FilesystemSinkOperator, compression_from_str, }; diff --git a/src/streaming_runtime/src/streaming/factory/connector/iceberg.rs b/src/streaming_runtime_operators/src/factory/connector/iceberg.rs similarity index 87% rename from src/streaming_runtime/src/streaming/factory/connector/iceberg.rs rename to src/streaming_runtime_operators/src/factory/connector/iceberg.rs index a6285cbf..c3b3ad81 100644 --- a/src/streaming_runtime/src/streaming/factory/connector/iceberg.rs +++ b/src/streaming_runtime_operators/src/factory/connector/iceberg.rs @@ -20,14 +20,14 @@ use protocol::function_stream_graph::connector_op::Config; use crate::sql::common::constants::connection_format_value; use crate::sql::common::with_option_keys as opt; -use crate::streaming::api::operator::ConstructedOperator; -use crate::streaming::factory::connector::sink_props_codec::{ +use crate::core::api::operator::ConstructedOperator; +use crate::factory::connector::sink_props_codec::{ apply_common_sink_fields, normalized_props, parse_sink_memory_bytes, }; -use crate::streaming::factory::global::Registry; -use crate::streaming::factory::operator_constructor::OperatorConstructor; -use crate::streaming::operators::sink::filesystem::compression_from_str; -use crate::streaming::operators::sink::iceberg::{IcebergFormat, IcebergSinkOperator}; +use crate::factory::global::Registry; +use crate::factory::operator_constructor::OperatorConstructor; +use crate::operators::sink::filesystem::compression_from_str; +use crate::operators::sink::iceberg::{IcebergFormat, IcebergSinkOperator}; pub struct IcebergSinkDispatcher; diff --git a/src/streaming_runtime/src/streaming/factory/connector/kafka.rs b/src/streaming_runtime_operators/src/factory/connector/kafka.rs similarity index 95% rename from src/streaming_runtime/src/streaming/factory/connector/kafka.rs rename to src/streaming_runtime_operators/src/factory/connector/kafka.rs index ad77af06..7dbffdc0 100644 --- a/src/streaming_runtime/src/streaming/factory/connector/kafka.rs +++ b/src/streaming_runtime_operators/src/factory/connector/kafka.rs @@ -25,16 +25,16 @@ use protocol::function_stream_graph::{ use tracing::info; use crate::sql::common::FsSchema; -use crate::streaming::api::operator::ConstructedOperator; -use crate::streaming::api::source::SourceOffset; -use crate::streaming::factory::global::Registry; -use crate::streaming::factory::operator_constructor::OperatorConstructor; -use crate::streaming::format::{ +use crate::core::api::operator::ConstructedOperator; +use crate::core::api::source::SourceOffset; +use crate::factory::global::Registry; +use crate::factory::operator_constructor::OperatorConstructor; +use crate::format::{ BadDataPolicy as RtBadDataPolicy, DataSerializer, DecimalEncoding as RtDecimalEncoding, Format as RuntimeFormat, JsonFormat as RuntimeJsonFormat, TimestampFormat as RtTimestampFormat, }; -use crate::streaming::operators::sink::kafka::{ConsistencyMode, KafkaSinkOperator}; -use crate::streaming::operators::source::kafka::{BufferedDeserializer, KafkaSourceOperator}; +use crate::operators::sink::kafka::{ConsistencyMode, KafkaSinkOperator}; +use crate::operators::source::kafka::{BufferedDeserializer, KafkaSourceOperator}; const DEFAULT_SOURCE_BATCH_SIZE: usize = 1024; diff --git a/src/streaming_runtime/src/streaming/factory/connector/lancedb.rs b/src/streaming_runtime_operators/src/factory/connector/lancedb.rs similarity index 93% rename from src/streaming_runtime/src/streaming/factory/connector/lancedb.rs rename to src/streaming_runtime_operators/src/factory/connector/lancedb.rs index 3deb055d..32ad49ef 100644 --- a/src/streaming_runtime/src/streaming/factory/connector/lancedb.rs +++ b/src/streaming_runtime_operators/src/factory/connector/lancedb.rs @@ -20,13 +20,13 @@ use protocol::function_stream_graph::connector_op::Config; use crate::sql::common::constants::connection_format_value; use crate::sql::common::with_option_keys as opt; -use crate::streaming::api::operator::ConstructedOperator; -use crate::streaming::factory::connector::sink_props_codec::{ +use crate::core::api::operator::ConstructedOperator; +use crate::factory::connector::sink_props_codec::{ apply_common_sink_fields, normalized_props, }; -use crate::streaming::factory::global::Registry; -use crate::streaming::factory::operator_constructor::OperatorConstructor; -use crate::streaming::operators::sink::lancedb::LanceDbSinkOperator; +use crate::factory::global::Registry; +use crate::factory::operator_constructor::OperatorConstructor; +use crate::operators::sink::lancedb::LanceDbSinkOperator; pub struct LanceDbSinkDispatcher; diff --git a/src/streaming_runtime/src/streaming/factory/connector/mod.rs b/src/streaming_runtime_operators/src/factory/connector/mod.rs similarity index 100% rename from src/streaming_runtime/src/streaming/factory/connector/mod.rs rename to src/streaming_runtime_operators/src/factory/connector/mod.rs diff --git a/src/streaming_runtime/src/streaming/factory/connector/s3.rs b/src/streaming_runtime_operators/src/factory/connector/s3.rs similarity index 90% rename from src/streaming_runtime/src/streaming/factory/connector/s3.rs rename to src/streaming_runtime_operators/src/factory/connector/s3.rs index 462b25d9..a6782bb8 100644 --- a/src/streaming_runtime/src/streaming/factory/connector/s3.rs +++ b/src/streaming_runtime_operators/src/factory/connector/s3.rs @@ -20,13 +20,13 @@ use protocol::function_stream_graph::connector_op::Config; use crate::sql::common::constants::connection_format_value; use crate::sql::common::with_option_keys as opt; -use crate::streaming::api::operator::ConstructedOperator; -use crate::streaming::factory::connector::sink_props_codec::{ +use crate::core::api::operator::ConstructedOperator; +use crate::factory::connector::sink_props_codec::{ apply_common_sink_fields, normalized_props, }; -use crate::streaming::factory::global::Registry; -use crate::streaming::factory::operator_constructor::OperatorConstructor; -use crate::streaming::operators::sink::s3::{S3Format, S3SinkOperator, compression_from_str}; +use crate::factory::global::Registry; +use crate::factory::operator_constructor::OperatorConstructor; +use crate::operators::sink::s3::{S3Format, S3SinkOperator, compression_from_str}; pub struct S3SinkDispatcher; diff --git a/src/streaming_runtime/src/streaming/factory/connector/sink_props_codec.rs b/src/streaming_runtime_operators/src/factory/connector/sink_props_codec.rs similarity index 100% rename from src/streaming_runtime/src/streaming/factory/connector/sink_props_codec.rs rename to src/streaming_runtime_operators/src/factory/connector/sink_props_codec.rs diff --git a/src/streaming_runtime/src/streaming/factory/global/mod.rs b/src/streaming_runtime_operators/src/factory/global/mod.rs similarity index 100% rename from src/streaming_runtime/src/streaming/factory/global/mod.rs rename to src/streaming_runtime_operators/src/factory/global/mod.rs diff --git a/src/streaming_runtime/src/streaming/factory/global/session_registry.rs b/src/streaming_runtime_operators/src/factory/global/session_registry.rs similarity index 100% rename from src/streaming_runtime/src/streaming/factory/global/session_registry.rs rename to src/streaming_runtime_operators/src/factory/global/session_registry.rs diff --git a/src/streaming_runtime/src/streaming/factory/mod.rs b/src/streaming_runtime_operators/src/factory/mod.rs similarity index 100% rename from src/streaming_runtime/src/streaming/factory/mod.rs rename to src/streaming_runtime_operators/src/factory/mod.rs diff --git a/src/streaming_runtime/src/streaming/factory/operator_constructor.rs b/src/streaming_runtime_operators/src/factory/operator_constructor.rs similarity index 88% rename from src/streaming_runtime/src/streaming/factory/operator_constructor.rs rename to src/streaming_runtime_operators/src/factory/operator_constructor.rs index 2cd96e6d..54788b7d 100644 --- a/src/streaming_runtime/src/streaming/factory/operator_constructor.rs +++ b/src/streaming_runtime_operators/src/factory/operator_constructor.rs @@ -13,8 +13,8 @@ use anyhow::Result; use std::sync::Arc; -use crate::streaming::api::operator::ConstructedOperator; -use crate::streaming::factory::global::Registry; +use crate::core::api::operator::ConstructedOperator; +use crate::factory::global::Registry; /// Builds a [`ConstructedOperator`] from serialized configuration and a [`Registry`]. pub trait OperatorConstructor: Send + Sync { diff --git a/src/streaming_runtime/src/streaming/factory/operator_factory.rs b/src/streaming_runtime_operators/src/factory/operator_factory.rs similarity index 94% rename from src/streaming_runtime/src/streaming/factory/operator_factory.rs rename to src/streaming_runtime_operators/src/factory/operator_factory.rs index 10b9d3bf..e695dd51 100644 --- a/src/streaming_runtime/src/streaming/factory/operator_factory.rs +++ b/src/streaming_runtime_operators/src/factory/operator_factory.rs @@ -11,23 +11,23 @@ // limitations under the License. use super::operator_constructor::OperatorConstructor; -use crate::streaming::api::operator::ConstructedOperator; -use crate::streaming::factory::connector::{ConnectorSinkDispatcher, ConnectorSourceDispatcher}; -use crate::streaming::factory::global::Registry; -use crate::streaming::operators::grouping::IncrementalAggregatingConstructor; -use crate::streaming::operators::joins::{InstantJoinConstructor, JoinWithExpirationConstructor}; +use crate::core::api::operator::ConstructedOperator; +use crate::factory::connector::{ConnectorSinkDispatcher, ConnectorSourceDispatcher}; +use crate::factory::global::Registry; +use crate::operators::grouping::IncrementalAggregatingConstructor; +use crate::operators::joins::{InstantJoinConstructor, JoinWithExpirationConstructor}; use anyhow::{Result, anyhow}; use prost::Message; use protocol::function_stream_graph::ProjectionOperator as ProjectionOperatorProto; use std::collections::HashMap; use std::sync::Arc; -use crate::streaming::operators::watermark::WatermarkGeneratorConstructor; -use crate::streaming::operators::windows::{ +use crate::operators::watermark::WatermarkGeneratorConstructor; +use crate::operators::windows::{ SessionAggregatingWindowConstructor, SlidingAggregatingWindowConstructor, TumblingAggregateWindowConstructor, WindowFunctionConstructor, }; -use crate::streaming::operators::{ +use crate::operators::{ KeyExecutionOperator, ProjectionOperator, StatelessPhysicalExecutor, ValueExecutionOperator, }; use protocol::function_stream_graph::{ @@ -121,7 +121,7 @@ impl OperatorFactory { ); self.register_named(OperatorName::ConnectorSink, Box::new(ConnectorSinkBridge)); - crate::streaming::factory::register_kafka_connector_plugins(self); + crate::factory::register_kafka_connector_plugins(self); } } diff --git a/src/streaming_runtime/src/streaming/format/config.rs b/src/streaming_runtime_operators/src/format/config.rs similarity index 100% rename from src/streaming_runtime/src/streaming/format/config.rs rename to src/streaming_runtime_operators/src/format/config.rs diff --git a/src/streaming_runtime/src/streaming/format/deserializer.rs b/src/streaming_runtime_operators/src/format/deserializer.rs similarity index 100% rename from src/streaming_runtime/src/streaming/format/deserializer.rs rename to src/streaming_runtime_operators/src/format/deserializer.rs diff --git a/src/streaming_runtime/src/streaming/format/encoder.rs b/src/streaming_runtime_operators/src/format/encoder.rs similarity index 100% rename from src/streaming_runtime/src/streaming/format/encoder.rs rename to src/streaming_runtime_operators/src/format/encoder.rs diff --git a/src/streaming_runtime/src/streaming/format/json_encoder.rs b/src/streaming_runtime_operators/src/format/json_encoder.rs similarity index 100% rename from src/streaming_runtime/src/streaming/format/json_encoder.rs rename to src/streaming_runtime_operators/src/format/json_encoder.rs diff --git a/src/streaming_runtime/src/streaming/format/mod.rs b/src/streaming_runtime_operators/src/format/mod.rs similarity index 100% rename from src/streaming_runtime/src/streaming/format/mod.rs rename to src/streaming_runtime_operators/src/format/mod.rs diff --git a/src/streaming_runtime/src/streaming/format/serializer.rs b/src/streaming_runtime_operators/src/format/serializer.rs similarity index 100% rename from src/streaming_runtime/src/streaming/format/serializer.rs rename to src/streaming_runtime_operators/src/format/serializer.rs diff --git a/src/streaming_runtime_operators/src/lib.rs b/src/streaming_runtime_operators/src/lib.rs new file mode 100644 index 00000000..d2c11639 --- /dev/null +++ b/src/streaming_runtime_operators/src/lib.rs @@ -0,0 +1,26 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Streaming operators, formats, and operator factory. + +pub use function_stream_config as config; +pub use function_stream_runtime_common::{common, memory}; +pub use function_stream_streaming_planner as sql; +pub use function_stream_streaming_runtime_core as core; +pub use function_stream_streaming_runtime_core::{ + StreamOutput, api, error, execution, network, protocol, state, +}; + +pub mod factory; +pub mod format; +pub mod operators; +pub mod util; diff --git a/src/streaming_runtime/src/streaming/operators/grouping/incremental_aggregate.rs b/src/streaming_runtime_operators/src/operators/grouping/incremental_aggregate.rs similarity index 99% rename from src/streaming_runtime/src/streaming/operators/grouping/incremental_aggregate.rs rename to src/streaming_runtime_operators/src/operators/grouping/incremental_aggregate.rs index 55ec2006..e2b455d0 100644 --- a/src/streaming_runtime/src/streaming/operators/grouping/incremental_aggregate.rs +++ b/src/streaming_runtime_operators/src/operators/grouping/incremental_aggregate.rs @@ -45,12 +45,12 @@ use crate::sql::common::{ CheckpointBarrier, FsSchema, TIMESTAMP_FIELD, UPDATING_META_FIELD, Watermark, to_nanos, }; use crate::sql::physical::updating_meta_fields; -use crate::streaming::StreamOutput; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::operator::{Collector, Operator}; -use crate::streaming::factory::Registry; -use crate::streaming::operators::{Key, UpdatingCache}; -use crate::streaming::state::OperatorStateStore; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::factory::Registry; +use crate::operators::{Key, UpdatingCache}; +use crate::core::state::OperatorStateStore; use crate::util::decode_aggregate; #[derive(Debug, Copy, Clone)] diff --git a/src/streaming_runtime/src/streaming/operators/grouping/mod.rs b/src/streaming_runtime_operators/src/operators/grouping/mod.rs similarity index 100% rename from src/streaming_runtime/src/streaming/operators/grouping/mod.rs rename to src/streaming_runtime_operators/src/operators/grouping/mod.rs diff --git a/src/streaming_runtime/src/streaming/operators/grouping/updating_cache.rs b/src/streaming_runtime_operators/src/operators/grouping/updating_cache.rs similarity index 100% rename from src/streaming_runtime/src/streaming/operators/grouping/updating_cache.rs rename to src/streaming_runtime_operators/src/operators/grouping/updating_cache.rs diff --git a/src/streaming_runtime/src/streaming/operators/joins/join_instance.rs b/src/streaming_runtime_operators/src/operators/joins/join_instance.rs similarity index 98% rename from src/streaming_runtime/src/streaming/operators/joins/join_instance.rs rename to src/streaming_runtime_operators/src/operators/joins/join_instance.rs index 232c2985..7287154a 100644 --- a/src/streaming_runtime/src/streaming/operators/joins/join_instance.rs +++ b/src/streaming_runtime_operators/src/operators/joins/join_instance.rs @@ -27,11 +27,11 @@ use tracing::{info, warn}; use crate::sql::common::{CheckpointBarrier, FsSchema, FsSchemaRef, Watermark}; use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; -use crate::streaming::StreamOutput; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::operator::{Collector, Operator}; -use crate::streaming::factory::Registry; -use crate::streaming::state::OperatorStateStore; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::factory::Registry; +use crate::core::state::OperatorStateStore; use async_trait::async_trait; use protocol::function_stream_graph::JoinOperator; diff --git a/src/streaming_runtime/src/streaming/operators/joins/join_with_expiration.rs b/src/streaming_runtime_operators/src/operators/joins/join_with_expiration.rs similarity index 98% rename from src/streaming_runtime/src/streaming/operators/joins/join_with_expiration.rs rename to src/streaming_runtime_operators/src/operators/joins/join_with_expiration.rs index 139e88cd..c335ef32 100644 --- a/src/streaming_runtime/src/streaming/operators/joins/join_with_expiration.rs +++ b/src/streaming_runtime_operators/src/operators/joins/join_with_expiration.rs @@ -26,11 +26,11 @@ use tracing::{info, warn}; use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark}; use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; -use crate::streaming::StreamOutput; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::operator::{Collector, Operator}; -use crate::streaming::factory::Registry; -use crate::streaming::state::OperatorStateStore; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::factory::Registry; +use crate::core::state::OperatorStateStore; use async_trait::async_trait; use protocol::function_stream_graph::JoinOperator; diff --git a/src/streaming_runtime/src/streaming/operators/joins/mod.rs b/src/streaming_runtime_operators/src/operators/joins/mod.rs similarity index 100% rename from src/streaming_runtime/src/streaming/operators/joins/mod.rs rename to src/streaming_runtime_operators/src/operators/joins/mod.rs diff --git a/src/streaming_runtime/src/streaming/operators/key_by.rs b/src/streaming_runtime_operators/src/operators/key_by.rs similarity index 97% rename from src/streaming_runtime/src/streaming/operators/key_by.rs rename to src/streaming_runtime_operators/src/operators/key_by.rs index f1e2831d..8c74514c 100644 --- a/src/streaming_runtime/src/streaming/operators/key_by.rs +++ b/src/streaming_runtime_operators/src/operators/key_by.rs @@ -20,9 +20,9 @@ use datafusion_physical_expr::expressions::Column; use std::sync::Arc; use crate::sql::common::{CheckpointBarrier, Watermark}; -use crate::streaming::StreamOutput; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::operator::{Collector, Operator}; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; use protocol::function_stream_graph::KeyPlanOperator; diff --git a/src/streaming_runtime/src/streaming/operators/key_operator.rs b/src/streaming_runtime_operators/src/operators/key_operator.rs similarity index 94% rename from src/streaming_runtime/src/streaming/operators/key_operator.rs rename to src/streaming_runtime_operators/src/operators/key_operator.rs index a8064d2e..1a7d0dd0 100644 --- a/src/streaming_runtime/src/streaming/operators/key_operator.rs +++ b/src/streaming_runtime_operators/src/operators/key_operator.rs @@ -13,13 +13,13 @@ //! Key-by over the physical plan output: key column(s) are **values** projected by the plan //! (e.g. `_key_user_id`); **shuffle / `StreamOutput::Keyed` uses `u64` hashes** computed by //! [`datafusion_common::hash_utils::create_hashes`] on those columns — same mechanism as -//! [`crate::streaming::operators::key_by::KeyByOperator`]. +//! [`crate::operators::key_by::KeyByOperator`]. use crate::sql::common::{CheckpointBarrier, Watermark}; -use crate::streaming::StreamOutput; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::operator::{Collector, Operator}; -use crate::streaming::operators::StatelessPhysicalExecutor; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::operators::StatelessPhysicalExecutor; use ahash::RandomState; use anyhow::{Result, anyhow}; use arrow::compute::{sort_to_indices, take}; diff --git a/src/streaming_runtime/src/streaming/operators/mod.rs b/src/streaming_runtime_operators/src/operators/mod.rs similarity index 100% rename from src/streaming_runtime/src/streaming/operators/mod.rs rename to src/streaming_runtime_operators/src/operators/mod.rs diff --git a/src/streaming_runtime/src/streaming/operators/projection.rs b/src/streaming_runtime_operators/src/operators/projection.rs similarity index 95% rename from src/streaming_runtime/src/streaming/operators/projection.rs rename to src/streaming_runtime_operators/src/operators/projection.rs index 54882547..566dd936 100644 --- a/src/streaming_runtime/src/streaming/operators/projection.rs +++ b/src/streaming_runtime_operators/src/operators/projection.rs @@ -24,10 +24,10 @@ use protocol::function_stream_graph::ProjectionOperator as ProjectionOperatorPro use crate::sql::common::{CheckpointBarrier, FsSchema, FsSchemaRef, Watermark}; use crate::sql::logical_node::logical::OperatorName; -use crate::streaming::StreamOutput; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::operator::{Collector, Operator}; -use crate::streaming::factory::global::Registry; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::factory::global::Registry; pub struct ProjectionOperator { name: String, diff --git a/src/streaming_runtime/src/streaming/operators/sink/delta/mod.rs b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs similarity index 97% rename from src/streaming_runtime/src/streaming/operators/sink/delta/mod.rs rename to src/streaming_runtime_operators/src/operators/sink/delta/mod.rs index b2d16c60..b828c129 100644 --- a/src/streaming_runtime/src/streaming/operators/sink/delta/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs @@ -30,10 +30,10 @@ use crate::memory::{MemoryBlock, try_global_memory_pool}; use crate::sql::common::constants::factory_operator_name; use crate::sql::common::with_option_keys as opt; use crate::sql::common::{CheckpointBarrier, Watermark}; -use crate::streaming::StreamOutput; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::operator::{Collector, Operator}; -use crate::streaming::format::encoder::FormatEncoder; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::format::encoder::FormatEncoder; /// Flush early when buffered batches exceed this size. const DEFAULT_MAX_BUFFER_BYTES: usize = 64 * 1024 * 1024; diff --git a/src/streaming_runtime/src/streaming/operators/sink/filesystem/mod.rs b/src/streaming_runtime_operators/src/operators/sink/filesystem/mod.rs similarity index 97% rename from src/streaming_runtime/src/streaming/operators/sink/filesystem/mod.rs rename to src/streaming_runtime_operators/src/operators/sink/filesystem/mod.rs index 3461c4a4..642724af 100644 --- a/src/streaming_runtime/src/streaming/operators/sink/filesystem/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/filesystem/mod.rs @@ -23,10 +23,10 @@ use tracing::{debug, info, warn}; use crate::memory::{MemoryBlock, try_global_memory_pool}; use crate::sql::common::constants::factory_operator_name; use crate::sql::common::{CheckpointBarrier, Watermark}; -use crate::streaming::StreamOutput; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::operator::{Collector, Operator}; -use crate::streaming::format::encoder::FormatEncoder; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::format::encoder::FormatEncoder; const DEFAULT_MAX_BUFFER_BYTES: usize = 64 * 1024 * 1024; diff --git a/src/streaming_runtime/src/streaming/operators/sink/iceberg/mod.rs b/src/streaming_runtime_operators/src/operators/sink/iceberg/mod.rs similarity index 97% rename from src/streaming_runtime/src/streaming/operators/sink/iceberg/mod.rs rename to src/streaming_runtime_operators/src/operators/sink/iceberg/mod.rs index 51715aa2..044ebe4b 100644 --- a/src/streaming_runtime/src/streaming/operators/sink/iceberg/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/iceberg/mod.rs @@ -30,10 +30,10 @@ use crate::memory::{MemoryBlock, try_global_memory_pool}; use crate::sql::common::constants::factory_operator_name; use crate::sql::common::with_option_keys as opt; use crate::sql::common::{CheckpointBarrier, Watermark}; -use crate::streaming::StreamOutput; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::operator::{Collector, Operator}; -use crate::streaming::format::encoder::FormatEncoder; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::format::encoder::FormatEncoder; const DEFAULT_MAX_BUFFER_BYTES: usize = 64 * 1024 * 1024; diff --git a/src/streaming_runtime/src/streaming/operators/sink/kafka/mod.rs b/src/streaming_runtime_operators/src/operators/sink/kafka/mod.rs similarity index 98% rename from src/streaming_runtime/src/streaming/operators/sink/kafka/mod.rs rename to src/streaming_runtime_operators/src/operators/sink/kafka/mod.rs index 1266ebdd..5ab646a4 100644 --- a/src/streaming_runtime/src/streaming/operators/sink/kafka/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/kafka/mod.rs @@ -38,10 +38,10 @@ use tracing::{info, warn}; use crate::sql::common::constants::factory_operator_name; use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark}; -use crate::streaming::StreamOutput; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::operator::{Collector, Operator}; -use crate::streaming::format::DataSerializer; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::format::DataSerializer; // ============================================================================ // ============================================================================ diff --git a/src/streaming_runtime/src/streaming/operators/sink/lancedb/mod.rs b/src/streaming_runtime_operators/src/operators/sink/lancedb/mod.rs similarity index 97% rename from src/streaming_runtime/src/streaming/operators/sink/lancedb/mod.rs rename to src/streaming_runtime_operators/src/operators/sink/lancedb/mod.rs index e69a59ee..a30f9dd9 100644 --- a/src/streaming_runtime/src/streaming/operators/sink/lancedb/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/lancedb/mod.rs @@ -25,9 +25,9 @@ use tracing::{info, warn}; use crate::sql::common::constants::factory_operator_name; use crate::sql::common::{CheckpointBarrier, Watermark}; -use crate::streaming::StreamOutput; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::operator::{Collector, Operator}; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; pub struct LanceDbSinkOperator { table_name: String, diff --git a/src/streaming_runtime/src/streaming/operators/sink/mod.rs b/src/streaming_runtime_operators/src/operators/sink/mod.rs similarity index 100% rename from src/streaming_runtime/src/streaming/operators/sink/mod.rs rename to src/streaming_runtime_operators/src/operators/sink/mod.rs diff --git a/src/streaming_runtime/src/streaming/operators/sink/s3/mod.rs b/src/streaming_runtime_operators/src/operators/sink/s3/mod.rs similarity index 98% rename from src/streaming_runtime/src/streaming/operators/sink/s3/mod.rs rename to src/streaming_runtime_operators/src/operators/sink/s3/mod.rs index f374b95c..61aa72d8 100644 --- a/src/streaming_runtime/src/streaming/operators/sink/s3/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/s3/mod.rs @@ -29,9 +29,9 @@ use tracing::{info, warn}; use crate::sql::common::constants::factory_operator_name; use crate::sql::common::with_option_keys as opt; use crate::sql::common::{CheckpointBarrier, Watermark}; -use crate::streaming::StreamOutput; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::operator::{Collector, Operator}; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; #[derive(Debug, Clone, Copy)] pub enum S3Format { diff --git a/src/streaming_runtime/src/streaming/operators/source/kafka/mod.rs b/src/streaming_runtime_operators/src/operators/source/kafka/mod.rs similarity index 98% rename from src/streaming_runtime/src/streaming/operators/source/kafka/mod.rs rename to src/streaming_runtime_operators/src/operators/source/kafka/mod.rs index 7f5e09d4..ebcf6363 100644 --- a/src/streaming_runtime/src/streaming/operators/source/kafka/mod.rs +++ b/src/streaming_runtime_operators/src/operators/source/kafka/mod.rs @@ -33,11 +33,11 @@ use tracing::{debug, error, info, warn}; use crate::sql::common::fs_schema::FieldValueType; use crate::sql::common::{CheckpointBarrier, MetadataField}; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::source::{ +use crate::core::api::context::TaskContext; +use crate::core::api::source::{ SourceCheckpointReport, SourceEvent, SourceOffset, SourceOperator, }; -use crate::streaming::format::{BadDataPolicy, DataDeserializer, Format}; +use crate::format::{BadDataPolicy, DataDeserializer, Format}; pub trait BatchDeserializer: Send + 'static { fn deserialize_slice( @@ -116,7 +116,11 @@ impl BatchDeserializer for BufferedDeserializer { } } -impl SourceOffset { +trait SourceOffsetExt { + fn rdkafka_offset(self) -> Offset; +} + +impl SourceOffsetExt for SourceOffset { fn rdkafka_offset(self) -> Offset { match self { SourceOffset::Earliest => Offset::Beginning, diff --git a/src/streaming_runtime/src/streaming/operators/source/mod.rs b/src/streaming_runtime_operators/src/operators/source/mod.rs similarity index 100% rename from src/streaming_runtime/src/streaming/operators/source/mod.rs rename to src/streaming_runtime_operators/src/operators/source/mod.rs diff --git a/src/streaming_runtime/src/streaming/operators/stateless_physical_executor.rs b/src/streaming_runtime_operators/src/operators/stateless_physical_executor.rs similarity index 98% rename from src/streaming_runtime/src/streaming/operators/stateless_physical_executor.rs rename to src/streaming_runtime_operators/src/operators/stateless_physical_executor.rs index 560ae8c5..1107ae68 100644 --- a/src/streaming_runtime/src/streaming/operators/stateless_physical_executor.rs +++ b/src/streaming_runtime_operators/src/operators/stateless_physical_executor.rs @@ -25,7 +25,7 @@ use futures::StreamExt; use prost::Message; use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; -use crate::streaming::factory::Registry; +use crate::factory::Registry; pub struct StatelessPhysicalExecutor { batch: Arc>>, diff --git a/src/streaming_runtime/src/streaming/operators/value_execution.rs b/src/streaming_runtime_operators/src/operators/value_execution.rs similarity index 91% rename from src/streaming_runtime/src/streaming/operators/value_execution.rs rename to src/streaming_runtime_operators/src/operators/value_execution.rs index d971668f..79a948dd 100644 --- a/src/streaming_runtime/src/streaming/operators/value_execution.rs +++ b/src/streaming_runtime_operators/src/operators/value_execution.rs @@ -16,10 +16,10 @@ use async_trait::async_trait; use futures::StreamExt; use crate::sql::common::{CheckpointBarrier, Watermark}; -use crate::streaming::StreamOutput; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::operator::{Collector, Operator}; -use crate::streaming::operators::StatelessPhysicalExecutor; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::operators::StatelessPhysicalExecutor; pub struct ValueExecutionOperator { name: String, diff --git a/src/streaming_runtime/src/streaming/operators/watermark/mod.rs b/src/streaming_runtime_operators/src/operators/watermark/mod.rs similarity index 100% rename from src/streaming_runtime/src/streaming/operators/watermark/mod.rs rename to src/streaming_runtime_operators/src/operators/watermark/mod.rs diff --git a/src/streaming_runtime/src/streaming/operators/watermark/watermark_generator.rs b/src/streaming_runtime_operators/src/operators/watermark/watermark_generator.rs similarity index 97% rename from src/streaming_runtime/src/streaming/operators/watermark/watermark_generator.rs rename to src/streaming_runtime_operators/src/operators/watermark/watermark_generator.rs index ea70b41b..d2720367 100644 --- a/src/streaming_runtime/src/streaming/operators/watermark/watermark_generator.rs +++ b/src/streaming_runtime_operators/src/operators/watermark/watermark_generator.rs @@ -26,10 +26,10 @@ use std::time::{Duration, SystemTime}; use tracing::debug; use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark, from_nanos, to_millis}; -use crate::streaming::StreamOutput; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::operator::{Collector, Operator}; -use crate::streaming::factory::Registry; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::factory::Registry; use async_trait::async_trait; use protocol::function_stream_graph::ExpressionWatermarkConfig; diff --git a/src/streaming_runtime/src/streaming/operators/windows/mod.rs b/src/streaming_runtime_operators/src/operators/windows/mod.rs similarity index 100% rename from src/streaming_runtime/src/streaming/operators/windows/mod.rs rename to src/streaming_runtime_operators/src/operators/windows/mod.rs diff --git a/src/streaming_runtime/src/streaming/operators/windows/session_aggregating_window.rs b/src/streaming_runtime_operators/src/operators/windows/session_aggregating_window.rs similarity index 99% rename from src/streaming_runtime/src/streaming/operators/windows/session_aggregating_window.rs rename to src/streaming_runtime_operators/src/operators/windows/session_aggregating_window.rs index b42fcc13..9da970ad 100644 --- a/src/streaming_runtime/src/streaming/operators/windows/session_aggregating_window.rs +++ b/src/streaming_runtime_operators/src/operators/windows/session_aggregating_window.rs @@ -42,11 +42,11 @@ use crate::sql::common::{ }; use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; use crate::sql::schema::utils::window_arrow_struct; -use crate::streaming::StreamOutput; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::operator::{Collector, Operator}; -use crate::streaming::factory::Registry; -use crate::streaming::state::OperatorStateStore; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::factory::Registry; +use crate::core::state::OperatorStateStore; use async_trait::async_trait; use protocol::function_stream_graph::SessionWindowAggregateOperator; // ============================================================================ @@ -711,7 +711,6 @@ impl SessionWindowOperator { .context("failed to create session window output batch") } - #[allow(dead_code)] fn earliest_batch_time(&self) -> Option { self.pq_start_times .first_key_value() diff --git a/src/streaming_runtime/src/streaming/operators/windows/sliding_aggregating_window.rs b/src/streaming_runtime_operators/src/operators/windows/sliding_aggregating_window.rs similarity index 99% rename from src/streaming_runtime/src/streaming/operators/windows/sliding_aggregating_window.rs rename to src/streaming_runtime_operators/src/operators/windows/sliding_aggregating_window.rs index b9c57835..371024a5 100644 --- a/src/streaming_runtime/src/streaming/operators/windows/sliding_aggregating_window.rs +++ b/src/streaming_runtime_operators/src/operators/windows/sliding_aggregating_window.rs @@ -35,11 +35,11 @@ use tracing::info; use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark, from_nanos, to_nanos}; use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; -use crate::streaming::StreamOutput; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::operator::{Collector, Operator}; -use crate::streaming::factory::Registry; -use crate::streaming::state::OperatorStateStore; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::factory::Registry; +use crate::core::state::OperatorStateStore; use async_trait::async_trait; use protocol::function_stream_graph::SlidingWindowAggregateOperator; // ============================================================================ diff --git a/src/streaming_runtime/src/streaming/operators/windows/tumbling_aggregating_window.rs b/src/streaming_runtime_operators/src/operators/windows/tumbling_aggregating_window.rs similarity index 98% rename from src/streaming_runtime/src/streaming/operators/windows/tumbling_aggregating_window.rs rename to src/streaming_runtime_operators/src/operators/windows/tumbling_aggregating_window.rs index 7d45a684..1d3a76b6 100644 --- a/src/streaming_runtime/src/streaming/operators/windows/tumbling_aggregating_window.rs +++ b/src/streaming_runtime_operators/src/operators/windows/tumbling_aggregating_window.rs @@ -38,11 +38,11 @@ use crate::sql::common::time_utils::print_time; use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark, from_nanos, to_nanos}; use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; use crate::sql::schema::utils::add_timestamp_field_arrow; -use crate::streaming::StreamOutput; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::operator::{Collector, Operator}; -use crate::streaming::factory::Registry; -use crate::streaming::state::OperatorStateStore; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::factory::Registry; +use crate::core::state::OperatorStateStore; use async_trait::async_trait; use protocol::function_stream_graph::TumblingWindowAggregateOperator; diff --git a/src/streaming_runtime/src/streaming/operators/windows/window_function.rs b/src/streaming_runtime_operators/src/operators/windows/window_function.rs similarity index 97% rename from src/streaming_runtime/src/streaming/operators/windows/window_function.rs rename to src/streaming_runtime_operators/src/operators/windows/window_function.rs index 83f14551..df4f5a67 100644 --- a/src/streaming_runtime/src/streaming/operators/windows/window_function.rs +++ b/src/streaming_runtime_operators/src/operators/windows/window_function.rs @@ -31,11 +31,11 @@ use crate::sql::common::{ CheckpointBarrier, FsSchema, FsSchemaRef, Watermark, from_nanos, to_nanos, }; use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; -use crate::streaming::StreamOutput; -use crate::streaming::api::context::TaskContext; -use crate::streaming::api::operator::{Collector, Operator}; -use crate::streaming::factory::Registry; -use crate::streaming::state::OperatorStateStore; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::factory::Registry; +use crate::core::state::OperatorStateStore; use async_trait::async_trait; // ============================================================================ diff --git a/src/streaming_runtime/src/util/mod.rs b/src/streaming_runtime_operators/src/util/mod.rs similarity index 100% rename from src/streaming_runtime/src/util/mod.rs rename to src/streaming_runtime_operators/src/util/mod.rs diff --git a/src/streaming_runtime/src/util/physical_aggregate.rs b/src/streaming_runtime_operators/src/util/physical_aggregate.rs similarity index 100% rename from src/streaming_runtime/src/util/physical_aggregate.rs rename to src/streaming_runtime_operators/src/util/physical_aggregate.rs From cee4f80caa23a4236b1cd0a2f98b6601c2ab98cd Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 19 May 2026 23:18:56 +0800 Subject: [PATCH 03/16] update --- Cargo.lock | 939 +++++++++++++++++- .../src/api/operator.rs | 2 +- src/streaming_runtime_core/src/api/source.rs | 2 +- .../src/execution/operator_chain.rs | 2 +- .../src/execution/pipeline.rs | 2 +- .../src/execution/source_driver.rs | 2 +- .../execution/tracker/watermark_tracker.rs | 2 +- .../src/job/job_manager.rs | 14 +- src/streaming_runtime_operators/Cargo.toml | 3 + .../src/factory/connector/delta.rs | 14 +- .../src/factory/connector/dispatchers.rs | 2 +- .../src/factory/connector/filesystem.rs | 4 +- .../src/factory/connector/iceberg.rs | 4 +- .../src/factory/connector/kafka.rs | 2 +- .../src/factory/connector/lancedb.rs | 8 +- .../src/factory/connector/s3.rs | 8 +- .../grouping/incremental_aggregate.rs | 10 +- .../src/operators/joins/join_instance.rs | 6 +- .../operators/joins/join_with_expiration.rs | 6 +- .../src/operators/key_by.rs | 2 +- .../src/operators/key_operator.rs | 2 +- .../src/operators/projection.rs | 4 +- .../src/operators/sink/delta/delta_commit.rs | 271 +++++ .../src/operators/sink/delta/mod.rs | 275 +++-- .../src/operators/sink/filesystem/mod.rs | 6 +- .../src/operators/sink/iceberg/mod.rs | 8 +- .../src/operators/sink/kafka/mod.rs | 4 +- .../src/operators/sink/lancedb/mod.rs | 4 +- .../src/operators/sink/s3/mod.rs | 6 +- .../src/operators/source/kafka/mod.rs | 8 +- .../operators/stateless_physical_executor.rs | 2 +- .../src/operators/value_execution.rs | 2 +- .../watermark/watermark_generator.rs | 2 +- .../windows/session_aggregating_window.rs | 10 +- .../windows/sliding_aggregating_window.rs | 6 +- .../windows/tumbling_aggregating_window.rs | 10 +- .../src/operators/windows/window_function.rs | 10 +- 37 files changed, 1440 insertions(+), 224 deletions(-) create mode 100644 src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs diff --git a/Cargo.lock b/Cargo.lock index fbb2e117..47828130 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -234,6 +234,27 @@ dependencies = [ "arrow-string 57.3.0", ] +[[package]] +name = "arrow" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" +dependencies = [ + "arrow-arith 58.3.0", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-cast 58.3.0", + "arrow-csv 58.3.0", + "arrow-data 58.3.0", + "arrow-ipc 58.3.0", + "arrow-json 58.3.0", + "arrow-ord 58.3.0", + "arrow-row 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", + "arrow-string 58.3.0", +] + [[package]] name = "arrow-arith" version = "55.2.0" @@ -262,6 +283,20 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-arith" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" +dependencies = [ + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "chrono", + "num-traits", +] + [[package]] name = "arrow-array" version = "52.2.0" @@ -314,6 +349,25 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-array" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" +dependencies = [ + "ahash", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + [[package]] name = "arrow-buffer" version = "52.2.0" @@ -348,6 +402,18 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-buffer" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + [[package]] name = "arrow-cast" version = "52.2.0" @@ -411,6 +477,27 @@ dependencies = [ "ryu", ] +[[package]] +name = "arrow-cast" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" +dependencies = [ + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-ord 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", + "atoi", + "base64", + "chrono", + "half", + "lexical-core 1.0.6", + "num-traits", + "ryu", +] + [[package]] name = "arrow-csv" version = "55.2.0" @@ -441,6 +528,21 @@ dependencies = [ "regex", ] +[[package]] +name = "arrow-csv" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" +dependencies = [ + "arrow-array 58.3.0", + "arrow-cast 58.3.0", + "arrow-schema 58.3.0", + "chrono", + "csv", + "csv-core", + "regex", +] + [[package]] name = "arrow-data" version = "52.2.0" @@ -478,6 +580,19 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-data" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" +dependencies = [ + "arrow-buffer 58.3.0", + "arrow-schema 58.3.0", + "half", + "num-integer", + "num-traits", +] + [[package]] name = "arrow-ipc" version = "52.2.0" @@ -522,6 +637,20 @@ dependencies = [ "zstd", ] +[[package]] +name = "arrow-ipc" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" +dependencies = [ + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", + "flatbuffers 25.12.19", +] + [[package]] name = "arrow-json" version = "55.2.0" @@ -568,6 +697,31 @@ dependencies = [ "simdutf8", ] +[[package]] +name = "arrow-json" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" +dependencies = [ + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-cast 58.3.0", + "arrow-ord 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", + "chrono", + "half", + "indexmap 2.13.1", + "itoa", + "lexical-core 1.0.6", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + [[package]] name = "arrow-ord" version = "55.2.0" @@ -594,6 +748,19 @@ dependencies = [ "arrow-select 57.3.0", ] +[[package]] +name = "arrow-ord" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" +dependencies = [ + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", +] + [[package]] name = "arrow-row" version = "55.2.0" @@ -620,6 +787,19 @@ dependencies = [ "half", ] +[[package]] +name = "arrow-row" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" +dependencies = [ + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "half", +] + [[package]] name = "arrow-schema" version = "52.2.0" @@ -647,6 +827,17 @@ dependencies = [ "serde_json", ] +[[package]] +name = "arrow-schema" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" +dependencies = [ + "bitflags 2.11.0", + "serde", + "serde_core", +] + [[package]] name = "arrow-select" version = "52.2.0" @@ -689,6 +880,20 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-select" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" +dependencies = [ + "ahash", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "num-traits", +] + [[package]] name = "arrow-string" version = "55.2.0" @@ -723,6 +928,23 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "arrow-string" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" +dependencies = [ + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + [[package]] name = "async-channel" version = "2.5.0" @@ -1430,7 +1652,7 @@ version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" dependencies = [ - "darling", + "darling 0.23.0", "ident_case", "prettyplease", "proc-macro2", @@ -1469,6 +1691,50 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "buoyant_kernel" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d3ca37afa82755db7b4fd51a4eab9e53eeb0aa1898fae15d373bd6df4bdf0f8" +dependencies = [ + "arrow 58.3.0", + "buoyant_kernel_derive", + "bytes", + "chrono", + "crc", + "futures", + "indexmap 2.13.1", + "itertools 0.14.0", + "object_store 0.13.2", + "parquet 58.3.0", + "percent-encoding", + "rand 0.9.2", + "reqwest 0.13.3", + "roaring", + "rustc_version", + "serde", + "serde_json", + "strum 0.27.2", + "thiserror 2.0.18", + "tokio", + "tracing", + "tracing-subscriber", + "url", + "uuid", + "z85", +] + +[[package]] +name = "buoyant_kernel_derive" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3448e05bba811d98c73a466843abd8c16d0416dc083f92b66847693fcb0714f4" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "bytemuck" version = "1.25.0" @@ -1639,6 +1905,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.44" @@ -1747,6 +2024,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "comfy-table" version = "7.2.2" @@ -1799,6 +2086,25 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +[[package]] +name = "convert_case" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db05ffb6856bf0ecdf6367558a76a0e8a77b1713044eb92845c692100ed50190" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -1983,6 +2289,21 @@ version = "0.128.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "903adeaf4938e60209a97b53a2e4326cd2d356aab9764a1934630204bae381c9" +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32c" version = "0.6.8" @@ -2114,6 +2435,23 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctor" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83cf0d42651b16c6dfe68685716d18480d18a9c39c62d76e8cf3eb6ed5d8bcbf" +dependencies = [ + "ctor-proc-macro", + "dtor", + "link-section", +] + +[[package]] +name = "ctor-proc-macro" +version = "0.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a949c44fcacbbbb7ada007dc7acb34603dd97cd47de5d054f2b6493ecebb483" + [[package]] name = "curl-sys" version = "0.4.87+curl-8.19.0" @@ -2129,14 +2467,38 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + [[package]] name = "darling" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.23.0", + "darling_macro 0.23.0", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", ] [[package]] @@ -2152,13 +2514,24 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.117", +] + [[package]] name = "darling_macro" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core", + "darling_core 0.23.0", "quote", "syn 2.0.117", ] @@ -2216,9 +2589,9 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.12.5", "parking_lot", - "parquet", + "parquet 55.2.0", "rand 0.9.2", "regex", "sqlparser 0.55.0", @@ -2268,7 +2641,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.12.5", "parking_lot", "rand 0.9.2", "regex", @@ -2299,7 +2672,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.12.5", "parking_lot", "tokio", ] @@ -2324,7 +2697,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.12.5", "parking_lot", "tokio", ] @@ -2347,7 +2720,7 @@ dependencies = [ "datafusion-session 48.0.1", "futures", "log", - "object_store", + "object_store 0.12.5", "tokio", ] @@ -2371,7 +2744,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.12.5", ] [[package]] @@ -2388,8 +2761,8 @@ dependencies = [ "indexmap 2.13.1", "libc", "log", - "object_store", - "parquet", + "object_store 0.12.5", + "parquet 55.2.0", "paste", "recursive", "sqlparser 0.55.0", @@ -2412,7 +2785,7 @@ dependencies = [ "indexmap 2.13.1", "libc", "log", - "object_store", + "object_store 0.12.5", "paste", "sqlparser 0.59.0", "tokio", @@ -2464,8 +2837,8 @@ dependencies = [ "glob", "itertools 0.14.0", "log", - "object_store", - "parquet", + "object_store 0.12.5", + "parquet 55.2.0", "rand 0.9.2", "tempfile", "tokio", @@ -2498,7 +2871,7 @@ dependencies = [ "glob", "itertools 0.14.0", "log", - "object_store", + "object_store 0.12.5", "rand 0.9.2", "tokio", "url", @@ -2524,7 +2897,7 @@ dependencies = [ "datafusion-session 52.5.0", "futures", "itertools 0.14.0", - "object_store", + "object_store 0.12.5", "tokio", ] @@ -2547,7 +2920,7 @@ dependencies = [ "datafusion-physical-plan 48.0.1", "datafusion-session 48.0.1", "futures", - "object_store", + "object_store 0.12.5", "regex", "tokio", ] @@ -2570,7 +2943,7 @@ dependencies = [ "datafusion-physical-plan 52.5.0", "datafusion-session 52.5.0", "futures", - "object_store", + "object_store 0.12.5", "regex", "tokio", ] @@ -2594,7 +2967,7 @@ dependencies = [ "datafusion-physical-plan 48.0.1", "datafusion-session 48.0.1", "futures", - "object_store", + "object_store 0.12.5", "serde_json", "tokio", ] @@ -2617,7 +2990,7 @@ dependencies = [ "datafusion-physical-plan 52.5.0", "datafusion-session 52.5.0", "futures", - "object_store", + "object_store 0.12.5", "tokio", ] @@ -2644,9 +3017,9 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.12.5", "parking_lot", - "parquet", + "parquet 55.2.0", "rand 0.9.2", "tokio", ] @@ -2673,7 +3046,7 @@ dependencies = [ "datafusion-expr 48.0.1", "futures", "log", - "object_store", + "object_store 0.12.5", "parking_lot", "rand 0.9.2", "tempfile", @@ -2694,7 +3067,7 @@ dependencies = [ "datafusion-expr 52.5.0", "futures", "log", - "object_store", + "object_store 0.12.5", "parking_lot", "rand 0.9.2", "tempfile", @@ -3275,7 +3648,7 @@ dependencies = [ "datafusion-common 48.0.1", "datafusion-expr 48.0.1", "datafusion-proto-common", - "object_store", + "object_store 0.12.5", "prost 0.13.5", ] @@ -3324,7 +3697,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.12.5", "parking_lot", "tokio", ] @@ -3405,6 +3778,80 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "deltalake" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33449d2bcc676a8a0088ed644c3f7c5bf094ae4b115c1730735ba30e8a1866bb" +dependencies = [ + "buoyant_kernel", + "ctor", + "deltalake-core", +] + +[[package]] +name = "deltalake-core" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8e67c5de0c7cc492a22c6e5bd7f37cfb59c471dd135617058eac609ddcfb878" +dependencies = [ + "arrow 58.3.0", + "arrow-arith 58.3.0", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-cast 58.3.0", + "arrow-ipc 58.3.0", + "arrow-json 58.3.0", + "arrow-ord 58.3.0", + "arrow-row 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", + "async-trait", + "buoyant_kernel", + "bytes", + "cfg-if", + "chrono", + "dashmap", + "deltalake-derive", + "dirs", + "either", + "futures", + "indexmap 2.13.1", + "itertools 0.14.0", + "num_cpus", + "object_store 0.13.2", + "parking_lot", + "parquet 58.3.0", + "percent-encoding", + "percent-encoding-rfc3986", + "pin-project-lite", + "rand 0.10.1", + "regex", + "serde", + "serde_json", + "sqlparser 0.61.0", + "strum 0.27.2", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "uuid", + "validator", +] + +[[package]] +name = "deltalake-derive" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5a646e2b05608db1e5f91e75b6af841d92294ea0864789bb01f3f818e5f7b25" +dependencies = [ + "convert_case", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "deranged" version = "0.5.8" @@ -3504,6 +3951,21 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" +[[package]] +name = "dtor" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edf234dd1594d6dd434a8fb8cada51ddbbc593e40e4a01556a0b31c62da2775b" +dependencies = [ + "dtor-proc-macro", +] + +[[package]] +name = "dtor-proc-macro" +version = "0.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2647271c92754afcb174e758003cfd1cbf1e43e5a7853d7b1813e63e19e39a73" + [[package]] name = "duct" version = "0.13.7" @@ -3812,9 +4274,9 @@ dependencies = [ "log", "lru", "num_cpus", - "object_store", + "object_store 0.12.5", "parking_lot", - "parquet", + "parquet 55.2.0", "petgraph 0.7.1", "proctitle", "prost 0.13.5", @@ -3987,7 +4449,7 @@ dependencies = [ "function-stream-runtime-common", "function-stream-streaming-planner", "parking_lot", - "parquet", + "parquet 55.2.0", "protocol", "serde", "tempfile", @@ -4038,6 +4500,7 @@ dependencies = [ "datafusion-expr 48.0.1", "datafusion-physical-expr 48.0.1", "datafusion-proto", + "deltalake", "function-stream-config", "function-stream-runtime-common", "function-stream-streaming-planner", @@ -4046,15 +4509,17 @@ dependencies = [ "governor", "itertools 0.14.0", "lance", - "object_store", - "parquet", + "object_store 0.12.5", + "parquet 55.2.0", "prost 0.13.5", "protocol", "rdkafka", "serde", "serde_json", + "thiserror 2.0.18", "tokio", "tracing", + "url", ] [[package]] @@ -4236,6 +4701,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", ] @@ -4362,6 +4828,12 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "heck" version = "0.5.0" @@ -4548,9 +5020,11 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2 0.6.3", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", ] [[package]] @@ -4878,12 +5352,61 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076" [[package]] -name = "jiff-tzdb-platform" -version = "0.1.3" +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ - "jiff-tzdb", + "quote", + "syn 2.0.117", ] [[package]] @@ -4975,7 +5498,7 @@ dependencies = [ "lance-table", "log", "moka", - "object_store", + "object_store 0.12.5", "permutation", "pin-project", "prost 0.14.3", @@ -5052,7 +5575,7 @@ dependencies = [ "mock_instant", "moka", "num_cpus", - "object_store", + "object_store 0.12.5", "pin-project", "prost 0.14.3", "rand 0.9.2", @@ -5183,7 +5706,7 @@ dependencies = [ "lance-io", "log", "num-traits", - "object_store", + "object_store 0.12.5", "prost 0.14.3", "prost-build 0.14.3", "prost-types 0.14.3", @@ -5237,7 +5760,7 @@ dependencies = [ "log", "ndarray", "num-traits", - "object_store", + "object_store 0.12.5", "prost 0.14.3", "prost-build 0.14.3", "prost-types 0.14.3", @@ -5286,7 +5809,7 @@ dependencies = [ "lance-core", "lance-namespace", "log", - "object_store", + "object_store 0.12.5", "object_store_opendal", "opendal", "path_abs", @@ -5340,7 +5863,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee2e48de899e2931afb67fcddd0a08e439bf5d8b6ea2a2ed9cb8f4df669bd5cc" dependencies = [ - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "serde_repr", @@ -5369,7 +5892,7 @@ dependencies = [ "lance-file", "lance-io", "log", - "object_store", + "object_store 0.12.5", "prost 0.14.3", "prost-build 0.14.3", "prost-types 0.14.3", @@ -5596,6 +6119,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "link-section" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b685d66585d646efe09fec763d796c291049c8b6bf84e04954bffc8748341f0d" + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -5700,6 +6229,15 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +dependencies = [ + "twox-hash", +] + [[package]] name = "lzma-sys" version = "0.1.20" @@ -6101,8 +6639,48 @@ dependencies = [ "percent-encoding", "quick-xml 0.38.4", "rand 0.9.2", - "reqwest", + "reqwest 0.12.28", + "ring", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "object_store" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +dependencies = [ + "async-trait", + "base64", + "bytes", + "chrono", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body-util", + "httparse", + "humantime", + "hyper", + "itertools 0.14.0", + "md-5", + "parking_lot", + "percent-encoding", + "quick-xml 0.39.4", + "rand 0.10.1", + "reqwest 0.12.28", "ring", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", @@ -6125,7 +6703,7 @@ dependencies = [ "bytes", "chrono", "futures", - "object_store", + "object_store 0.12.5", "opendal", "pin-project", "tokio", @@ -6170,7 +6748,7 @@ dependencies = [ "percent-encoding", "quick-xml 0.38.4", "reqsign", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "tokio", @@ -6309,7 +6887,43 @@ dependencies = [ "lz4_flex 0.11.6", "num", "num-bigint", - "object_store", + "object_store 0.12.5", + "paste", + "seq-macro", + "simdutf8", + "snap", + "thrift", + "tokio", + "twox-hash", + "zstd", +] + +[[package]] +name = "parquet" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" +dependencies = [ + "ahash", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-ipc 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", + "base64", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.17.1", + "lz4_flex 0.13.1", + "num-bigint", + "num-integer", + "num-traits", + "object_store 0.13.2", "paste", "seq-macro", "simdutf8", @@ -6350,6 +6964,12 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "percent-encoding-rfc3986" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3637c05577168127568a64e9dc5a6887da720efef07b3d9472d45f63ab191166" + [[package]] name = "permutation" version = "0.4.1" @@ -6514,6 +7134,28 @@ dependencies = [ "toml_edit", ] +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -6723,6 +7365,16 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.39.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quinn" version = "0.11.9" @@ -6749,6 +7401,7 @@ version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.3.4", "lru-slab", @@ -6836,6 +7489,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -6874,6 +7538,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rand_distr" version = "0.4.3" @@ -7121,7 +7791,7 @@ dependencies = [ "percent-encoding", "quick-xml 0.37.5", "rand 0.8.5", - "reqwest", + "reqwest 0.12.28", "rust-ini", "serde", "serde_json", @@ -7176,6 +7846,44 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "reqwest" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "h2", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower 0.5.3", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "ring" version = "0.17.14" @@ -7330,6 +8038,33 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.11" @@ -7425,7 +8160,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags 2.11.0", - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -7690,6 +8425,16 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -7812,6 +8557,16 @@ dependencies = [ "sqlparser_derive 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "sqlparser" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" +dependencies = [ + "log", + "recursive", +] + [[package]] name = "sqlparser_derive" version = "0.3.0" @@ -7890,6 +8645,9 @@ name = "strum" version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros 0.27.2", +] [[package]] name = "strum_macros" @@ -7964,6 +8722,27 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.11.0", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "system-interface" version = "0.27.3" @@ -8309,6 +9088,7 @@ dependencies = [ "bytes", "libc", "mio", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2 0.6.3", @@ -8541,6 +9321,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -8693,6 +9474,7 @@ dependencies = [ "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] @@ -8727,10 +9509,41 @@ checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" dependencies = [ "getrandom 0.4.2", "js-sys", + "rand 0.10.1", "serde_core", "wasm-bindgen", ] +[[package]] +name = "validator" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0b4a29d8709210980a09379f27ee31549b73292c87ab9899beee1c0d3be6303" +dependencies = [ + "idna", + "once_cell", + "regex", + "serde", + "serde_derive", + "serde_json", + "url", + "validator_derive", +] + +[[package]] +name = "validator_derive" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bac855a2ce6f843beb229757e6e570a42e837bcb15e5f449dd48d5747d41bf77" +dependencies = [ + "darling 0.20.11", + "once_cell", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "valuable" version = "0.1.1" @@ -9337,6 +10150,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "1.0.7" @@ -9478,6 +10300,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -9863,6 +10696,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "z85" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6e61e59a957b7ccee15d2049f86e8bfd6f66968fcd88f018950662d9b86e675" + [[package]] name = "zerocopy" version = "0.8.48" diff --git a/src/streaming_runtime_core/src/api/operator.rs b/src/streaming_runtime_core/src/api/operator.rs index c58c41e3..89e7d9f1 100644 --- a/src/streaming_runtime_core/src/api/operator.rs +++ b/src/streaming_runtime_core/src/api/operator.rs @@ -10,10 +10,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sql::common::{CheckpointBarrier, Watermark}; use crate::api::context::TaskContext; use crate::api::source::SourceOperator; use crate::protocol::event::StreamOutput; +use crate::sql::common::{CheckpointBarrier, Watermark}; use arrow_array::RecordBatch; use async_trait::async_trait; diff --git a/src/streaming_runtime_core/src/api/source.rs b/src/streaming_runtime_core/src/api/source.rs index 3c2445e4..b79dc8c3 100644 --- a/src/streaming_runtime_core/src/api/source.rs +++ b/src/streaming_runtime_core/src/api/source.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sql::common::{CheckpointBarrier, Watermark}; use crate::api::context::TaskContext; +use crate::sql::common::{CheckpointBarrier, Watermark}; use arrow_array::RecordBatch; use async_trait::async_trait; use protocol::storage::{ diff --git a/src/streaming_runtime_core/src/execution/operator_chain.rs b/src/streaming_runtime_core/src/execution/operator_chain.rs index f540cc6b..3de774d6 100644 --- a/src/streaming_runtime_core/src/execution/operator_chain.rs +++ b/src/streaming_runtime_core/src/execution/operator_chain.rs @@ -13,7 +13,6 @@ use anyhow::anyhow; use async_trait::async_trait; -use crate::sql::common::CheckpointBarrier; use crate::api::context::TaskContext; use crate::api::operator::{Collector, Operator}; use crate::error::RunError; @@ -21,6 +20,7 @@ use crate::protocol::{ control::{ControlCommand, StopMode}, event::{StreamEvent, StreamOutput, TrackedEvent}, }; +use crate::sql::common::CheckpointBarrier; // ============================================================================ // Core Traits diff --git a/src/streaming_runtime_core/src/execution/pipeline.rs b/src/streaming_runtime_core/src/execution/pipeline.rs index 57e67ca9..69cbe6af 100644 --- a/src/streaming_runtime_core/src/execution/pipeline.rs +++ b/src/streaming_runtime_core/src/execution/pipeline.rs @@ -14,7 +14,6 @@ use tokio::sync::mpsc::UnboundedReceiver; use tokio_stream::{StreamExt, StreamMap}; use tracing::{Instrument, info, info_span}; -use crate::sql::common::Watermark; use crate::api::context::TaskContext; use crate::api::operator::Operator; use crate::error::RunError; @@ -28,6 +27,7 @@ use crate::protocol::{ control::ControlCommand, event::{StreamEvent, TrackedEvent}, }; +use crate::sql::common::Watermark; pub struct Pipeline { chain_head: Box, diff --git a/src/streaming_runtime_core/src/execution/source_driver.rs b/src/streaming_runtime_core/src/execution/source_driver.rs index 315aa6dc..6d015f44 100644 --- a/src/streaming_runtime_core/src/execution/source_driver.rs +++ b/src/streaming_runtime_core/src/execution/source_driver.rs @@ -14,7 +14,6 @@ use tokio::sync::mpsc::UnboundedReceiver; use tokio::time::{Instant, sleep}; use tracing::{Instrument, info, info_span, warn}; -use crate::sql::common::CheckpointBarrier; use crate::api::context::TaskContext; use crate::api::source::{SourceCheckpointReport, SourceEvent, SourceOperator}; use crate::error::RunError; @@ -23,6 +22,7 @@ use crate::protocol::{ control::ControlCommand, event::{StreamEvent, TrackedEvent}, }; +use crate::sql::common::CheckpointBarrier; pub struct SourceDriver { operator: Box, diff --git a/src/streaming_runtime_core/src/execution/tracker/watermark_tracker.rs b/src/streaming_runtime_core/src/execution/tracker/watermark_tracker.rs index 17de1ec4..3234e130 100644 --- a/src/streaming_runtime_core/src/execution/tracker/watermark_tracker.rs +++ b/src/streaming_runtime_core/src/execution/tracker/watermark_tracker.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sql::common::Watermark; use crate::protocol::event::{merge_watermarks, watermark_strictly_advances}; +use crate::sql::common::Watermark; #[derive(Debug)] pub struct WatermarkTracker { diff --git a/src/streaming_runtime_job_manager/src/job/job_manager.rs b/src/streaming_runtime_job_manager/src/job/job_manager.rs index 0f708ebf..f6e94118 100644 --- a/src/streaming_runtime_job_manager/src/job/job_manager.rs +++ b/src/streaming_runtime_job_manager/src/job/job_manager.rs @@ -30,22 +30,22 @@ use crate::config::{ DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_OPERATOR_STATE_STORE_MEMORY_BYTES, DEFAULT_PIPELINE_PARALLELISM, }; -use crate::memory::global_memory_pool; -use crate::sql::logical_node::logical::OperatorName; -use crate::stream_catalog::CatalogManager; use crate::core::api::context::TaskContext; use crate::core::api::operator::{ConstructedOperator, Operator}; use crate::core::api::source::SourceOperator; use crate::core::execution::{ChainBuilder, Pipeline, SourceDriver}; +use crate::core::network::endpoint::{BoxedEventStream, PhysicalSender}; +use crate::core::protocol::control::{ControlCommand, JobMasterEvent, StopMode}; +use crate::core::protocol::event::CheckpointBarrier; +use crate::core::state::{IoManager, IoPool, NoopMetricsCollector}; use crate::factory::OperatorFactory; use crate::job::edge_manager::EdgeManager; use crate::job::models::{ PhysicalExecutionGraph, PhysicalPipeline, PipelineStatus, StreamingJobRollupStatus, }; -use crate::core::network::endpoint::{BoxedEventStream, PhysicalSender}; -use crate::core::protocol::control::{ControlCommand, JobMasterEvent, StopMode}; -use crate::core::protocol::event::CheckpointBarrier; -use crate::core::state::{IoManager, IoPool, NoopMetricsCollector}; +use crate::memory::global_memory_pool; +use crate::sql::logical_node::logical::OperatorName; +use crate::stream_catalog::CatalogManager; #[derive(Debug, Clone)] pub struct StreamingJobSummary { diff --git a/src/streaming_runtime_operators/Cargo.toml b/src/streaming_runtime_operators/Cargo.toml index d6949084..d20f491c 100644 --- a/src/streaming_runtime_operators/Cargo.toml +++ b/src/streaming_runtime_operators/Cargo.toml @@ -28,6 +28,7 @@ datafusion-execution = { git = "https://github.com/FunctionStream/datafusion", b datafusion-expr = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } datafusion-physical-expr = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } datafusion-proto = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +deltalake = { version = "0.32.3", default-features = false, features = ["rustls"] } function-stream-config = { path = "../config" } function-stream-runtime-common = { path = "../runtime_common" } function-stream-streaming-planner = { path = "../streaming_planner" } @@ -43,5 +44,7 @@ protocol = { path = "../../protocol" } rdkafka = { version = "0.38", features = ["cmake-build", "ssl", "gssapi", "curl"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +thiserror = "2" tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "sync", "time", "net", "signal", "fs", "io-util"] } tracing = "0.1" +url = "2" diff --git a/src/streaming_runtime_operators/src/factory/connector/delta.rs b/src/streaming_runtime_operators/src/factory/connector/delta.rs index 688bca53..1b5aab0a 100644 --- a/src/streaming_runtime_operators/src/factory/connector/delta.rs +++ b/src/streaming_runtime_operators/src/factory/connector/delta.rs @@ -18,8 +18,6 @@ use prost::Message; use protocol::function_stream_graph::ConnectorOp; use protocol::function_stream_graph::connector_op::Config; -use crate::sql::common::constants::connection_format_value; -use crate::sql::common::with_option_keys as opt; use crate::core::api::operator::ConstructedOperator; use crate::factory::connector::sink_props_codec::{ apply_common_sink_fields, normalized_props, parse_sink_memory_bytes, @@ -28,6 +26,9 @@ use crate::factory::global::Registry; use crate::factory::operator_constructor::OperatorConstructor; use crate::operators::sink::delta::{DeltaFormat, DeltaSinkOperator}; use crate::operators::sink::filesystem::compression_from_str; +use crate::sql::common::FsSchema; +use crate::sql::common::constants::connection_format_value; +use crate::sql::common::with_option_keys as opt; pub struct DeltaSinkDispatcher; @@ -40,6 +41,14 @@ impl OperatorConstructor for DeltaSinkDispatcher { }; let props = normalized_props(props); + let catalog_schema = op + .fs_schema + .as_ref() + .map(|fs| FsSchema::try_from(fs.clone())) + .transpose() + .map_err(|e| anyhow::anyhow!("invalid fs_schema for delta sink: {e}"))? + .map(|fs| fs.schema); + let format = props .get(opt::FORMAT) .map(String::as_str) @@ -69,6 +78,7 @@ impl OperatorConstructor for DeltaSinkDispatcher { compression, sink_memory_bytes, props, + catalog_schema, )?, ))) } diff --git a/src/streaming_runtime_operators/src/factory/connector/dispatchers.rs b/src/streaming_runtime_operators/src/factory/connector/dispatchers.rs index 7b345dd2..ca9f98aa 100644 --- a/src/streaming_runtime_operators/src/factory/connector/dispatchers.rs +++ b/src/streaming_runtime_operators/src/factory/connector/dispatchers.rs @@ -16,10 +16,10 @@ use anyhow::{Context, Result, bail}; use prost::Message; use protocol::function_stream_graph::ConnectorOp; -use crate::sql::common::constants::connector_type; use crate::core::api::operator::ConstructedOperator; use crate::factory::global::Registry; use crate::factory::operator_constructor::OperatorConstructor; +use crate::sql::common::constants::connector_type; use super::{ DeltaSinkDispatcher, FilesystemSinkDispatcher, IcebergSinkDispatcher, LanceDbSinkDispatcher, diff --git a/src/streaming_runtime_operators/src/factory/connector/filesystem.rs b/src/streaming_runtime_operators/src/factory/connector/filesystem.rs index bc7f9da1..39696a7a 100644 --- a/src/streaming_runtime_operators/src/factory/connector/filesystem.rs +++ b/src/streaming_runtime_operators/src/factory/connector/filesystem.rs @@ -18,8 +18,6 @@ use prost::Message; use protocol::function_stream_graph::ConnectorOp; use protocol::function_stream_graph::connector_op::Config; -use crate::sql::common::constants::connection_format_value; -use crate::sql::common::with_option_keys as opt; use crate::core::api::operator::ConstructedOperator; use crate::factory::connector::sink_props_codec::{ apply_common_sink_fields, normalized_props, parse_sink_memory_bytes, @@ -29,6 +27,8 @@ use crate::factory::operator_constructor::OperatorConstructor; use crate::operators::sink::filesystem::{ FilesystemFormat, FilesystemSinkOperator, compression_from_str, }; +use crate::sql::common::constants::connection_format_value; +use crate::sql::common::with_option_keys as opt; pub struct FilesystemSinkDispatcher; diff --git a/src/streaming_runtime_operators/src/factory/connector/iceberg.rs b/src/streaming_runtime_operators/src/factory/connector/iceberg.rs index c3b3ad81..dc8f2cb6 100644 --- a/src/streaming_runtime_operators/src/factory/connector/iceberg.rs +++ b/src/streaming_runtime_operators/src/factory/connector/iceberg.rs @@ -18,8 +18,6 @@ use prost::Message; use protocol::function_stream_graph::ConnectorOp; use protocol::function_stream_graph::connector_op::Config; -use crate::sql::common::constants::connection_format_value; -use crate::sql::common::with_option_keys as opt; use crate::core::api::operator::ConstructedOperator; use crate::factory::connector::sink_props_codec::{ apply_common_sink_fields, normalized_props, parse_sink_memory_bytes, @@ -28,6 +26,8 @@ use crate::factory::global::Registry; use crate::factory::operator_constructor::OperatorConstructor; use crate::operators::sink::filesystem::compression_from_str; use crate::operators::sink::iceberg::{IcebergFormat, IcebergSinkOperator}; +use crate::sql::common::constants::connection_format_value; +use crate::sql::common::with_option_keys as opt; pub struct IcebergSinkDispatcher; diff --git a/src/streaming_runtime_operators/src/factory/connector/kafka.rs b/src/streaming_runtime_operators/src/factory/connector/kafka.rs index 7dbffdc0..4e967f6d 100644 --- a/src/streaming_runtime_operators/src/factory/connector/kafka.rs +++ b/src/streaming_runtime_operators/src/factory/connector/kafka.rs @@ -24,7 +24,6 @@ use protocol::function_stream_graph::{ }; use tracing::info; -use crate::sql::common::FsSchema; use crate::core::api::operator::ConstructedOperator; use crate::core::api::source::SourceOffset; use crate::factory::global::Registry; @@ -35,6 +34,7 @@ use crate::format::{ }; use crate::operators::sink::kafka::{ConsistencyMode, KafkaSinkOperator}; use crate::operators::source::kafka::{BufferedDeserializer, KafkaSourceOperator}; +use crate::sql::common::FsSchema; const DEFAULT_SOURCE_BATCH_SIZE: usize = 1024; diff --git a/src/streaming_runtime_operators/src/factory/connector/lancedb.rs b/src/streaming_runtime_operators/src/factory/connector/lancedb.rs index 32ad49ef..3df7ae36 100644 --- a/src/streaming_runtime_operators/src/factory/connector/lancedb.rs +++ b/src/streaming_runtime_operators/src/factory/connector/lancedb.rs @@ -18,15 +18,13 @@ use prost::Message; use protocol::function_stream_graph::ConnectorOp; use protocol::function_stream_graph::connector_op::Config; -use crate::sql::common::constants::connection_format_value; -use crate::sql::common::with_option_keys as opt; use crate::core::api::operator::ConstructedOperator; -use crate::factory::connector::sink_props_codec::{ - apply_common_sink_fields, normalized_props, -}; +use crate::factory::connector::sink_props_codec::{apply_common_sink_fields, normalized_props}; use crate::factory::global::Registry; use crate::factory::operator_constructor::OperatorConstructor; use crate::operators::sink::lancedb::LanceDbSinkOperator; +use crate::sql::common::constants::connection_format_value; +use crate::sql::common::with_option_keys as opt; pub struct LanceDbSinkDispatcher; diff --git a/src/streaming_runtime_operators/src/factory/connector/s3.rs b/src/streaming_runtime_operators/src/factory/connector/s3.rs index a6782bb8..0947394a 100644 --- a/src/streaming_runtime_operators/src/factory/connector/s3.rs +++ b/src/streaming_runtime_operators/src/factory/connector/s3.rs @@ -18,15 +18,13 @@ use prost::Message; use protocol::function_stream_graph::ConnectorOp; use protocol::function_stream_graph::connector_op::Config; -use crate::sql::common::constants::connection_format_value; -use crate::sql::common::with_option_keys as opt; use crate::core::api::operator::ConstructedOperator; -use crate::factory::connector::sink_props_codec::{ - apply_common_sink_fields, normalized_props, -}; +use crate::factory::connector::sink_props_codec::{apply_common_sink_fields, normalized_props}; use crate::factory::global::Registry; use crate::factory::operator_constructor::OperatorConstructor; use crate::operators::sink::s3::{S3Format, S3SinkOperator, compression_from_str}; +use crate::sql::common::constants::connection_format_value; +use crate::sql::common::with_option_keys as opt; pub struct S3SinkDispatcher; diff --git a/src/streaming_runtime_operators/src/operators/grouping/incremental_aggregate.rs b/src/streaming_runtime_operators/src/operators/grouping/incremental_aggregate.rs index e2b455d0..c1b34b71 100644 --- a/src/streaming_runtime_operators/src/operators/grouping/incremental_aggregate.rs +++ b/src/streaming_runtime_operators/src/operators/grouping/incremental_aggregate.rs @@ -41,16 +41,16 @@ use std::{collections::HashMap, mem, sync::Arc}; use tracing::{debug, info, warn}; // ========================================================================= // ========================================================================= -use crate::sql::common::{ - CheckpointBarrier, FsSchema, TIMESTAMP_FIELD, UPDATING_META_FIELD, Watermark, to_nanos, -}; -use crate::sql::physical::updating_meta_fields; use crate::core::StreamOutput; use crate::core::api::context::TaskContext; use crate::core::api::operator::{Collector, Operator}; +use crate::core::state::OperatorStateStore; use crate::factory::Registry; use crate::operators::{Key, UpdatingCache}; -use crate::core::state::OperatorStateStore; +use crate::sql::common::{ + CheckpointBarrier, FsSchema, TIMESTAMP_FIELD, UPDATING_META_FIELD, Watermark, to_nanos, +}; +use crate::sql::physical::updating_meta_fields; use crate::util::decode_aggregate; #[derive(Debug, Copy, Clone)] diff --git a/src/streaming_runtime_operators/src/operators/joins/join_instance.rs b/src/streaming_runtime_operators/src/operators/joins/join_instance.rs index 7287154a..3c9c55fc 100644 --- a/src/streaming_runtime_operators/src/operators/joins/join_instance.rs +++ b/src/streaming_runtime_operators/src/operators/joins/join_instance.rs @@ -25,13 +25,13 @@ use std::sync::{Arc, RwLock}; use std::time::UNIX_EPOCH; use tracing::{info, warn}; -use crate::sql::common::{CheckpointBarrier, FsSchema, FsSchemaRef, Watermark}; -use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; use crate::core::StreamOutput; use crate::core::api::context::TaskContext; use crate::core::api::operator::{Collector, Operator}; -use crate::factory::Registry; use crate::core::state::OperatorStateStore; +use crate::factory::Registry; +use crate::sql::common::{CheckpointBarrier, FsSchema, FsSchemaRef, Watermark}; +use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; use async_trait::async_trait; use protocol::function_stream_graph::JoinOperator; diff --git a/src/streaming_runtime_operators/src/operators/joins/join_with_expiration.rs b/src/streaming_runtime_operators/src/operators/joins/join_with_expiration.rs index c335ef32..f0e5eec5 100644 --- a/src/streaming_runtime_operators/src/operators/joins/join_with_expiration.rs +++ b/src/streaming_runtime_operators/src/operators/joins/join_with_expiration.rs @@ -24,13 +24,13 @@ use std::sync::{Arc, RwLock}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tracing::{info, warn}; -use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark}; -use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; use crate::core::StreamOutput; use crate::core::api::context::TaskContext; use crate::core::api::operator::{Collector, Operator}; -use crate::factory::Registry; use crate::core::state::OperatorStateStore; +use crate::factory::Registry; +use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark}; +use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; use async_trait::async_trait; use protocol::function_stream_graph::JoinOperator; diff --git a/src/streaming_runtime_operators/src/operators/key_by.rs b/src/streaming_runtime_operators/src/operators/key_by.rs index 8c74514c..63ac06cd 100644 --- a/src/streaming_runtime_operators/src/operators/key_by.rs +++ b/src/streaming_runtime_operators/src/operators/key_by.rs @@ -19,10 +19,10 @@ use datafusion_common::hash_utils::create_hashes; use datafusion_physical_expr::expressions::Column; use std::sync::Arc; -use crate::sql::common::{CheckpointBarrier, Watermark}; use crate::core::StreamOutput; use crate::core::api::context::TaskContext; use crate::core::api::operator::{Collector, Operator}; +use crate::sql::common::{CheckpointBarrier, Watermark}; use protocol::function_stream_graph::KeyPlanOperator; diff --git a/src/streaming_runtime_operators/src/operators/key_operator.rs b/src/streaming_runtime_operators/src/operators/key_operator.rs index 1a7d0dd0..a485bf9e 100644 --- a/src/streaming_runtime_operators/src/operators/key_operator.rs +++ b/src/streaming_runtime_operators/src/operators/key_operator.rs @@ -15,11 +15,11 @@ //! [`datafusion_common::hash_utils::create_hashes`] on those columns — same mechanism as //! [`crate::operators::key_by::KeyByOperator`]. -use crate::sql::common::{CheckpointBarrier, Watermark}; use crate::core::StreamOutput; use crate::core::api::context::TaskContext; use crate::core::api::operator::{Collector, Operator}; use crate::operators::StatelessPhysicalExecutor; +use crate::sql::common::{CheckpointBarrier, Watermark}; use ahash::RandomState; use anyhow::{Result, anyhow}; use arrow::compute::{sort_to_indices, take}; diff --git a/src/streaming_runtime_operators/src/operators/projection.rs b/src/streaming_runtime_operators/src/operators/projection.rs index 566dd936..b2aa0af3 100644 --- a/src/streaming_runtime_operators/src/operators/projection.rs +++ b/src/streaming_runtime_operators/src/operators/projection.rs @@ -22,12 +22,12 @@ use std::sync::Arc; use protocol::function_stream_graph::ProjectionOperator as ProjectionOperatorProto; -use crate::sql::common::{CheckpointBarrier, FsSchema, FsSchemaRef, Watermark}; -use crate::sql::logical_node::logical::OperatorName; use crate::core::StreamOutput; use crate::core::api::context::TaskContext; use crate::core::api::operator::{Collector, Operator}; use crate::factory::global::Registry; +use crate::sql::common::{CheckpointBarrier, FsSchema, FsSchemaRef, Watermark}; +use crate::sql::logical_node::logical::OperatorName; pub struct ProjectionOperator { name: String, diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs b/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs new file mode 100644 index 00000000..5c5574fd --- /dev/null +++ b/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs @@ -0,0 +1,271 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +use arrow_array::RecordBatch; +use arrow_ipc::writer::StreamWriter; +use arrow_schema::Schema as ArrowSchema; +use deltalake::errors::DeltaTableError; +use deltalake::kernel::engine::arrow_conversion::TryIntoKernel as _; +use deltalake::kernel::transaction::CommitBuilder; +use deltalake::kernel::{Action, Add, StructField, StructType}; +use deltalake::protocol::{DeltaOperation, SaveMode}; +use deltalake::{DeltaTable, open_table_with_storage_options}; +use tracing::info; +use url::Url; + +use super::DeltaSinkError; + +pub struct UncommittedDataFile { + pub path: String, + pub size_bytes: u64, + pub record_count: u64, +} + +pub struct DeltaTableCommitter { + table_uri: Url, + storage_options: HashMap, + uncommitted: Vec, + table: Option, + /// Precomputed from catalog `fs_schema` at startup, or lazily from the first batch. + delta_columns: Option>, +} + +impl DeltaTableCommitter { + pub fn try_new( + table_uri: Url, + storage_options: HashMap, + catalog_schema: Option>, + ) -> Result { + let delta_columns = catalog_schema + .as_deref() + .map(arrow_schema_to_delta_columns) + .transpose()?; + + Ok(Self { + table_uri, + storage_options, + uncommitted: Vec::new(), + table: None, + delta_columns, + }) + } + + /// Fallback when catalog schema is absent: derive columns from the first flushed batch. + pub fn update_schema(&mut self, schema: Arc) -> Result<(), DeltaSinkError> { + if self.delta_columns.is_none() { + self.delta_columns = Some(arrow_schema_to_delta_columns(&schema)?); + } + Ok(()) + } + + pub fn register_uncommitted(&mut self, file: UncommittedDataFile) { + self.uncommitted.push(file); + } + + pub fn has_uncommitted(&self) -> bool { + !self.uncommitted.is_empty() + } + + async fn ensure_table(&mut self) -> Result<(), DeltaSinkError> { + if self.table.is_some() { + return Ok(()); + } + + let open_result = if self.storage_options.is_empty() { + deltalake::open_table(self.table_uri.clone()).await + } else { + open_table_with_storage_options(self.table_uri.clone(), self.storage_options.clone()) + .await + }; + + match open_result { + Ok(table) => { + self.table = Some(table); + return Ok(()); + } + Err(DeltaTableError::NotATable(_)) => {} + Err(e) => { + return Err(DeltaSinkError::CommitterFailed(e.to_string())); + } + } + + let columns = self.delta_columns.as_ref().ok_or_else(|| { + DeltaSinkError::Config( + "no catalog schema and no batch schema available before commit".into(), + ) + })?; + + let table = if self.storage_options.is_empty() { + DeltaTable::try_from_url(self.table_uri.clone()) + .await + .map_err(|e| DeltaSinkError::CommitterFailed(e.to_string()))? + } else { + DeltaTable::try_from_url_with_storage_options( + self.table_uri.clone(), + self.storage_options.clone(), + ) + .await + .map_err(|e| DeltaSinkError::CommitterFailed(e.to_string()))? + }; + + let table = table + .create() + .with_columns(columns.clone()) + .await + .map_err(|e| DeltaSinkError::CommitterFailed(e.to_string()))?; + + self.table = Some(table); + Ok(()) + } + + pub async fn commit_checkpoint(&mut self, epoch: u64) -> Result<(), DeltaSinkError> { + if self.uncommitted.is_empty() { + return Ok(()); + } + + let files = std::mem::take(&mut self.uncommitted); + self.ensure_table().await?; + + let table = self + .table + .as_mut() + .ok_or_else(|| DeltaSinkError::CommitterFailed("delta table not loaded".into()))?; + + let now_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as i64; + + let adds: Vec = files + .iter() + .map(|f| { + Action::Add(Add { + path: f.path.clone(), + size: f.size_bytes as i64, + modification_time: now_ms, + data_change: true, + partition_values: HashMap::new(), + stats: simple_stats_json(f.record_count), + ..Default::default() + }) + }) + .collect(); + + let snapshot = table + .snapshot() + .map_err(|e| DeltaSinkError::CommitterFailed(e.to_string()))?; + + let operation = DeltaOperation::Write { + mode: SaveMode::Append, + partition_by: None, + predicate: None, + }; + + let finalized = CommitBuilder::default() + .with_actions(adds) + .build(Some(snapshot), table.log_store().clone(), operation) + .await + .map_err(|e| DeltaSinkError::CommitterFailed(e.to_string()))?; + + let version = finalized.version(); + table.state = Some(finalized.snapshot()); + table + .load() + .await + .map_err(|e| DeltaSinkError::CommitterFailed(e.to_string()))?; + + info!( + epoch, + version, + file_count = files.len(), + "successfully committed delta checkpoint (2PC completed)" + ); + Ok(()) + } +} + +/// Bridge arrow 55 (runtime) schema to deltalake kernel schema via IPC. +fn arrow_schema_to_delta_columns(schema: &ArrowSchema) -> Result, DeltaSinkError> { + let empty = RecordBatch::new_empty(Arc::new(schema.clone())); + let mut buf = Vec::new(); + { + let mut writer = StreamWriter::try_new(&mut buf, &empty.schema()).map_err(|e| { + DeltaSinkError::CommitterFailed(format!("ipc schema encode failed: {e}")) + })?; + writer.write(&empty).map_err(|e| { + DeltaSinkError::CommitterFailed(format!("ipc schema encode failed: {e}")) + })?; + writer.finish().map_err(|e| { + DeltaSinkError::CommitterFailed(format!("ipc schema encode failed: {e}")) + })?; + } + + let cursor = std::io::Cursor::new(buf); + let reader = deltalake::arrow::ipc::reader::StreamReader::try_new(cursor, None) + .map_err(|e| DeltaSinkError::CommitterFailed(format!("ipc schema decode failed: {e}")))?; + let delta_schema = reader.schema(); + let struct_type: StructType = delta_schema + .try_into_kernel() + .map_err(|e| DeltaSinkError::CommitterFailed(e.to_string()))?; + Ok(struct_type.fields().cloned().collect()) +} + +fn simple_stats_json(record_count: u64) -> Option { + Some(format!(r#"{{"numRecords":{record_count}}}"#)) +} + +pub fn build_delta_storage_options(options: &HashMap) -> HashMap { + use crate::sql::common::with_option_keys as opt; + + let mut storage = HashMap::new(); + if let Some(v) = options.get(opt::S3_ACCESS_KEY_ID) { + storage.insert("AWS_ACCESS_KEY_ID".to_string(), v.clone()); + } + if let Some(v) = options.get(opt::S3_SECRET_ACCESS_KEY) { + storage.insert("AWS_SECRET_ACCESS_KEY".to_string(), v.clone()); + } + if let Some(v) = options.get(opt::S3_REGION) { + storage.insert("AWS_REGION".to_string(), v.clone()); + } + if let Some(v) = options.get(opt::S3_ENDPOINT) { + storage.insert("AWS_ENDPOINT_URL".to_string(), v.clone()); + if !v.starts_with("https://") { + storage.insert("AWS_ALLOW_HTTP".to_string(), "true".to_string()); + } + } + if let Some(v) = options.get(opt::S3_SESSION_TOKEN) { + storage.insert("AWS_SESSION_TOKEN".to_string(), v.clone()); + } + storage +} + +pub fn resolve_delta_table_uri(path: &str, bucket: Option<&str>) -> Result { + if path.contains("://") { + return Url::parse(path).map_err(|e| DeltaSinkError::Config(e.to_string())); + } + if let Some(bucket) = bucket { + let trimmed = path.trim_matches('/'); + let uri = if trimmed.is_empty() { + format!("s3://{bucket}") + } else { + format!("s3://{bucket}/{trimmed}") + }; + return Url::parse(&uri).map_err(|e| DeltaSinkError::Config(e.to_string())); + } + let canonical = std::fs::canonicalize(path).map_err(DeltaSinkError::Io)?; + Url::from_directory_path(canonical) + .map_err(|_| DeltaSinkError::Config(format!("invalid local delta table path: {path}"))) +} diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs index b828c129..10b6d2fd 100644 --- a/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs @@ -10,34 +10,57 @@ // See the License for the specific language governing permissions and // limitations under the License. +mod delta_commit; + use std::collections::HashMap; -use std::fs::create_dir_all; use std::path::PathBuf; use std::sync::Arc; -use anyhow::{Context, Result}; use arrow_array::RecordBatch; +use arrow_schema::Schema as ArrowSchema; use async_trait::async_trait; use bytes::Bytes; +use delta_commit::{ + DeltaTableCommitter, UncommittedDataFile, build_delta_storage_options, resolve_delta_table_uri, +}; use object_store::aws::AmazonS3Builder; use object_store::path::Path as ObjectStorePath; use object_store::{ObjectStore, PutPayload}; use parquet::basic::Compression; use tokio::io::AsyncWriteExt; -use tracing::{debug, info, warn}; +use tracing::{debug, info, instrument, warn}; +use url::Url; -use crate::memory::{MemoryBlock, try_global_memory_pool}; -use crate::sql::common::constants::factory_operator_name; -use crate::sql::common::with_option_keys as opt; -use crate::sql::common::{CheckpointBarrier, Watermark}; use crate::core::StreamOutput; use crate::core::api::context::TaskContext; use crate::core::api::operator::{Collector, Operator}; use crate::format::encoder::FormatEncoder; +use crate::memory::{MemoryBlock, try_global_memory_pool}; +use crate::sql::common::constants::factory_operator_name; +use crate::sql::common::with_option_keys as opt; +use crate::sql::common::{CheckpointBarrier, Watermark}; -/// Flush early when buffered batches exceed this size. const DEFAULT_MAX_BUFFER_BYTES: usize = 64 * 1024 * 1024; +/// Strongly typed error domain for the Delta sink. +#[derive(thiserror::Error, Debug)] +pub enum DeltaSinkError { + #[error("local filesystem I/O error: {0}")] + Io(#[from] std::io::Error), + + #[error("object store error: {0}")] + ObjectStore(#[from] object_store::Error), + + #[error("serialization task panicked: {0}")] + SerializationPanic(String), + + #[error("delta committer failed: {0}")] + CommitterFailed(String), + + #[error("configuration error: {0}")] + Config(String), +} + enum DeltaDestination { Local(PathBuf), S3 { @@ -46,6 +69,15 @@ enum DeltaDestination { }, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DeltaFormat { + Csv, + Parquet, + JsonL, + Avro, + Orc, +} + pub struct DeltaSinkOperator { table_name: String, destination: DeltaDestination, @@ -56,18 +88,16 @@ pub struct DeltaSinkOperator { early_flush_threshold_bytes: usize, file_counter: u64, format: DeltaFormat, -} - -#[derive(Debug, Clone, Copy)] -pub enum DeltaFormat { - Csv, - Parquet, - JsonL, - Avro, - Orc, + committer: Option, + table_uri: Option, + storage_options: HashMap, + s3_bucket: Option, + sink_path: String, + catalog_schema: Option>, } impl DeltaSinkOperator { + /// Synchronous, side-effect-free constructor. Async setup runs in [`Operator::on_start`]. pub fn try_new( table_name: String, path: String, @@ -75,15 +105,20 @@ impl DeltaSinkOperator { parquet_compression: Compression, sink_memory_bytes: u64, options: HashMap, - ) -> Result { - let destination = if let Some(bucket) = options.get(opt::S3_BUCKET) { + catalog_schema: Option>, + ) -> Result { + let s3_bucket = options.get(opt::S3_BUCKET).cloned(); + let storage_options = build_delta_storage_options(&options); + + let destination = if let Some(bucket) = &s3_bucket { let region = options .get(opt::S3_REGION) - .cloned() - .unwrap_or_else(|| "us-east-1".to_string()); + .map(|s| s.as_str()) + .unwrap_or("us-east-1"); let mut builder = AmazonS3Builder::new() .with_bucket_name(bucket.clone()) .with_region(region); + if let Some(endpoint) = options.get(opt::S3_ENDPOINT) { builder = builder.with_endpoint(endpoint); if endpoint.to_ascii_lowercase().starts_with("http://") { @@ -99,86 +134,106 @@ impl DeltaSinkOperator { if let Some(v) = options.get(opt::S3_SESSION_TOKEN) { builder = builder.with_token(v); } - let client = builder - .build() - .context("failed to build s3 client for delta sink")?; + + let client = builder.build().map_err(DeltaSinkError::ObjectStore)?; + DeltaDestination::S3 { prefix: path.trim_matches('/').to_string(), client: Arc::new(client), } } else { - let root = PathBuf::from(path.clone()); - create_dir_all(&root) - .with_context(|| format!("failed to create delta sink dir {}", root.display()))?; - DeltaDestination::Local(root) + DeltaDestination::Local(PathBuf::from(path.clone())) }; let mut sink_memory_block = None; let reserve_bytes = usize::try_from(sink_memory_bytes).unwrap_or(DEFAULT_MAX_BUFFER_BYTES); let mut early_flush_threshold_bytes = reserve_bytes; - if let Ok(pool) = try_global_memory_pool() - && let Ok(block) = pool.try_request_block(reserve_bytes as u64) - { - early_flush_threshold_bytes = ((block.capacity() as usize) * 8) / 10; - sink_memory_block = Some(block); + + if let Ok(pool) = try_global_memory_pool() { + if let Ok(block) = pool.try_request_block(reserve_bytes as u64) { + early_flush_threshold_bytes = ((block.capacity() as usize) * 8) / 10; + sink_memory_block = Some(block); + } } Ok(Self { table_name, destination, parquet_compression, - pending: Vec::new(), + pending: Vec::with_capacity(32), pending_bytes: 0, sink_memory_block, early_flush_threshold_bytes, file_counter: 0, format, + committer: None, + table_uri: None, + storage_options, + s3_bucket, + sink_path: path, + catalog_schema, }) } - async fn flush_epoch(&mut self, epoch: u64, subtask_idx: usize) -> Result<()> { + /// Flush physical data files only; transaction commit is deferred to checkpoint. + #[instrument(skip(self), fields(table = %self.table_name))] + async fn flush_data_file( + &mut self, + epoch: u64, + subtask_idx: usize, + ) -> Result<(), DeltaSinkError> { if self.pending.is_empty() { return Ok(()); } + let fallback_schema = self + .catalog_schema + .is_none() + .then(|| self.pending[0].schema()); let batches = std::mem::take(&mut self.pending); + self.pending_bytes = 0; + + let record_count: u64 = batches.iter().map(|b| b.num_rows() as u64).sum(); let format = self.format; let compression = self.parquet_compression; - let bytes = tokio::task::spawn_blocking(move || -> Result> { - match format { - DeltaFormat::Csv => FormatEncoder::encode_csv(&batches), - DeltaFormat::Parquet => FormatEncoder::encode_parquet(&batches, compression), - DeltaFormat::JsonL => FormatEncoder::encode_jsonl(&batches), - DeltaFormat::Avro => FormatEncoder::encode_avro(&batches), - DeltaFormat::Orc => FormatEncoder::encode_orc(&batches), - } + + let bytes = tokio::task::spawn_blocking(move || match format { + DeltaFormat::Csv => FormatEncoder::encode_csv(&batches), + DeltaFormat::Parquet => FormatEncoder::encode_parquet(&batches, compression), + DeltaFormat::JsonL => FormatEncoder::encode_jsonl(&batches), + DeltaFormat::Avro => FormatEncoder::encode_avro(&batches), + DeltaFormat::Orc => FormatEncoder::encode_orc(&batches), }) .await - .context("tokio blocking task panicked during serialization")??; + .map_err(|e| DeltaSinkError::SerializationPanic(e.to_string()))? + .map_err(|e| DeltaSinkError::SerializationPanic(e.to_string()))?; + + if bytes.is_empty() { + return Ok(()); + } self.file_counter += 1; + let ext = match self.format { + DeltaFormat::Csv => "csv", + DeltaFormat::Parquet => "parquet", + DeltaFormat::JsonL => "jsonl", + DeltaFormat::Avro => "avro", + DeltaFormat::Orc => "orc", + }; + let file_name = format!( - "delta-part-{:05}-epoch-{:010}-{:06}.{}", - subtask_idx, - epoch, - self.file_counter, - match self.format { - DeltaFormat::Csv => "csv", - DeltaFormat::Parquet => "parquet", - DeltaFormat::JsonL => "jsonl", - DeltaFormat::Avro => "avro", - DeltaFormat::Orc => "orc", - } + "part-{subtask_idx:05}-epoch-{epoch:010}-{counter:06}.{ext}", + counter = self.file_counter ); + let file_size = bytes.len() as u64; + let mut relative_path = file_name.clone(); + match &self.destination { DeltaDestination::Local(root) => { - let out = root.join(file_name); - let mut f = tokio::fs::File::create(&out).await.with_context(|| { - format!("failed creating delta sink file {}", out.display()) - })?; - f.write_all(&bytes) - .await - .with_context(|| format!("failed writing delta sink file {}", out.display()))?; + let out = root.join(&file_name); + let mut f = tokio::fs::File::create(&out).await?; + f.write_all(&bytes).await?; + f.flush().await?; } DeltaDestination::S3 { prefix, client } => { let key = if prefix.is_empty() { @@ -186,16 +241,42 @@ impl DeltaSinkOperator { } else { format!("{prefix}/{file_name}") }; + relative_path = key.clone(); client .put( &ObjectStorePath::from(key), PutPayload::from(Bytes::from(bytes)), ) - .await - .context("failed writing object to s3")?; + .await?; + } + } + + if let Some(committer) = self.committer.as_mut() { + if let Some(schema) = fallback_schema { + committer.update_schema(schema)?; + } + committer.register_uncommitted(UncommittedDataFile { + path: relative_path, + size_bytes: file_size, + record_count, + }); + } + + Ok(()) + } + + async fn flush_and_commit_checkpoint( + &mut self, + epoch: u64, + subtask_idx: usize, + ) -> Result<(), DeltaSinkError> { + self.flush_data_file(epoch, subtask_idx).await?; + + if let Some(committer) = self.committer.as_mut() { + if committer.has_uncommitted() { + committer.commit_checkpoint(epoch).await?; } } - self.pending_bytes = 0; Ok(()) } } @@ -206,18 +287,32 @@ impl Operator for DeltaSinkOperator { factory_operator_name::CONNECTOR_SINK } - async fn on_start(&mut self, _ctx: &mut TaskContext) -> Result<()> { - let reserved_block_bytes = self - .sink_memory_block - .as_ref() - .map(|b| b.capacity()) - .unwrap_or(0); + async fn on_start(&mut self, _ctx: &mut TaskContext) -> anyhow::Result<()> { + if let DeltaDestination::Local(root) = &self.destination { + tokio::fs::create_dir_all(root).await?; + } + + let table_uri = resolve_delta_table_uri(&self.sink_path, self.s3_bucket.as_deref())?; + self.table_uri = Some(table_uri.clone()); + + if self.format == DeltaFormat::Parquet { + self.committer = Some(DeltaTableCommitter::try_new( + table_uri, + self.storage_options.clone(), + self.catalog_schema.clone(), + )?); + } else { + warn!( + format = ?self.format, + "format is not Parquet; writing raw files WITHOUT Delta transaction logs" + ); + } + info!( table = %self.table_name, - format = ?self.format, - reserved_block_bytes, - early_flush_threshold_bytes = self.early_flush_threshold_bytes, - "Starting delta sink operator" + threshold = self.early_flush_threshold_bytes, + is_true_delta = self.committer.is_some(), + "delta sink operator started successfully" ); Ok(()) } @@ -228,18 +323,16 @@ impl Operator for DeltaSinkOperator { batch: RecordBatch, ctx: &mut TaskContext, _collector: &mut dyn Collector, - ) -> Result<()> { - let batch_size = batch.get_array_memory_size(); + ) -> anyhow::Result<()> { + self.pending_bytes += batch.get_array_memory_size(); self.pending.push(batch); - self.pending_bytes += batch_size; if self.pending_bytes > self.early_flush_threshold_bytes { debug!( - table = %self.table_name, bytes = self.pending_bytes, - "memory watermark reached, triggering early flush" + "memory watermark reached, executing early flush (commit deferred to checkpoint)" ); - self.flush_epoch(0, ctx.subtask_index as usize).await?; + self.flush_data_file(0, ctx.subtask_index as usize).await?; } Ok(()) } @@ -249,23 +342,29 @@ impl Operator for DeltaSinkOperator { _watermark: Watermark, _ctx: &mut TaskContext, _collector: &mut dyn Collector, - ) -> Result<()> { + ) -> anyhow::Result<()> { Ok(()) } + #[instrument(skip(self, ctx), fields(epoch = barrier.epoch))] async fn snapshot_state( &mut self, barrier: CheckpointBarrier, ctx: &mut TaskContext, - ) -> Result<()> { - self.flush_epoch(barrier.epoch, ctx.subtask_index as usize) + ) -> anyhow::Result<()> { + self.flush_and_commit_checkpoint(barrier.epoch, ctx.subtask_index as usize) .await + .map_err(anyhow::Error::from) } - async fn on_close(&mut self, ctx: &mut TaskContext) -> Result> { - if !self.pending.is_empty() { - warn!(table = %self.table_name, "flushing remaining delta sink batches on close"); - self.flush_epoch(u64::MAX, ctx.subtask_index as usize) + async fn on_close(&mut self, ctx: &mut TaskContext) -> anyhow::Result> { + if !self.pending.is_empty() || self.committer.as_ref().is_some_and(|c| c.has_uncommitted()) + { + warn!( + table = %self.table_name, + "operator closing, forcing final flush and commit" + ); + self.flush_and_commit_checkpoint(u64::MAX, ctx.subtask_index as usize) .await?; } Ok(vec![]) diff --git a/src/streaming_runtime_operators/src/operators/sink/filesystem/mod.rs b/src/streaming_runtime_operators/src/operators/sink/filesystem/mod.rs index 642724af..7aabfc3c 100644 --- a/src/streaming_runtime_operators/src/operators/sink/filesystem/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/filesystem/mod.rs @@ -20,13 +20,13 @@ use parquet::basic::Compression; use tokio::io::AsyncWriteExt; use tracing::{debug, info, warn}; -use crate::memory::{MemoryBlock, try_global_memory_pool}; -use crate::sql::common::constants::factory_operator_name; -use crate::sql::common::{CheckpointBarrier, Watermark}; use crate::core::StreamOutput; use crate::core::api::context::TaskContext; use crate::core::api::operator::{Collector, Operator}; use crate::format::encoder::FormatEncoder; +use crate::memory::{MemoryBlock, try_global_memory_pool}; +use crate::sql::common::constants::factory_operator_name; +use crate::sql::common::{CheckpointBarrier, Watermark}; const DEFAULT_MAX_BUFFER_BYTES: usize = 64 * 1024 * 1024; diff --git a/src/streaming_runtime_operators/src/operators/sink/iceberg/mod.rs b/src/streaming_runtime_operators/src/operators/sink/iceberg/mod.rs index 044ebe4b..c89df325 100644 --- a/src/streaming_runtime_operators/src/operators/sink/iceberg/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/iceberg/mod.rs @@ -26,14 +26,14 @@ use parquet::basic::Compression; use tokio::io::AsyncWriteExt; use tracing::{debug, info, warn}; -use crate::memory::{MemoryBlock, try_global_memory_pool}; -use crate::sql::common::constants::factory_operator_name; -use crate::sql::common::with_option_keys as opt; -use crate::sql::common::{CheckpointBarrier, Watermark}; use crate::core::StreamOutput; use crate::core::api::context::TaskContext; use crate::core::api::operator::{Collector, Operator}; use crate::format::encoder::FormatEncoder; +use crate::memory::{MemoryBlock, try_global_memory_pool}; +use crate::sql::common::constants::factory_operator_name; +use crate::sql::common::with_option_keys as opt; +use crate::sql::common::{CheckpointBarrier, Watermark}; const DEFAULT_MAX_BUFFER_BYTES: usize = 64 * 1024 * 1024; diff --git a/src/streaming_runtime_operators/src/operators/sink/kafka/mod.rs b/src/streaming_runtime_operators/src/operators/sink/kafka/mod.rs index 5ab646a4..e973244f 100644 --- a/src/streaming_runtime_operators/src/operators/sink/kafka/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/kafka/mod.rs @@ -36,12 +36,12 @@ use std::time::Duration; use tokio::time::sleep; use tracing::{info, warn}; -use crate::sql::common::constants::factory_operator_name; -use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark}; use crate::core::StreamOutput; use crate::core::api::context::TaskContext; use crate::core::api::operator::{Collector, Operator}; use crate::format::DataSerializer; +use crate::sql::common::constants::factory_operator_name; +use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark}; // ============================================================================ // ============================================================================ diff --git a/src/streaming_runtime_operators/src/operators/sink/lancedb/mod.rs b/src/streaming_runtime_operators/src/operators/sink/lancedb/mod.rs index a30f9dd9..4c9ed7a9 100644 --- a/src/streaming_runtime_operators/src/operators/sink/lancedb/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/lancedb/mod.rs @@ -23,11 +23,11 @@ use lance::dataset::{WriteMode, WriteParams}; use lance::io::{ObjectStoreParams, StorageOptionsAccessor}; use tracing::{info, warn}; -use crate::sql::common::constants::factory_operator_name; -use crate::sql::common::{CheckpointBarrier, Watermark}; use crate::core::StreamOutput; use crate::core::api::context::TaskContext; use crate::core::api::operator::{Collector, Operator}; +use crate::sql::common::constants::factory_operator_name; +use crate::sql::common::{CheckpointBarrier, Watermark}; pub struct LanceDbSinkOperator { table_name: String, diff --git a/src/streaming_runtime_operators/src/operators/sink/s3/mod.rs b/src/streaming_runtime_operators/src/operators/sink/s3/mod.rs index 61aa72d8..1a454c17 100644 --- a/src/streaming_runtime_operators/src/operators/sink/s3/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/s3/mod.rs @@ -26,12 +26,12 @@ use parquet::basic::Compression; use parquet::file::properties::WriterProperties; use tracing::{info, warn}; -use crate::sql::common::constants::factory_operator_name; -use crate::sql::common::with_option_keys as opt; -use crate::sql::common::{CheckpointBarrier, Watermark}; use crate::core::StreamOutput; use crate::core::api::context::TaskContext; use crate::core::api::operator::{Collector, Operator}; +use crate::sql::common::constants::factory_operator_name; +use crate::sql::common::with_option_keys as opt; +use crate::sql::common::{CheckpointBarrier, Watermark}; #[derive(Debug, Clone, Copy)] pub enum S3Format { diff --git a/src/streaming_runtime_operators/src/operators/source/kafka/mod.rs b/src/streaming_runtime_operators/src/operators/source/kafka/mod.rs index ebcf6363..dd952239 100644 --- a/src/streaming_runtime_operators/src/operators/source/kafka/mod.rs +++ b/src/streaming_runtime_operators/src/operators/source/kafka/mod.rs @@ -31,13 +31,11 @@ use std::num::NonZeroU32; use std::time::{Duration, Instant}; use tracing::{debug, error, info, warn}; -use crate::sql::common::fs_schema::FieldValueType; -use crate::sql::common::{CheckpointBarrier, MetadataField}; use crate::core::api::context::TaskContext; -use crate::core::api::source::{ - SourceCheckpointReport, SourceEvent, SourceOffset, SourceOperator, -}; +use crate::core::api::source::{SourceCheckpointReport, SourceEvent, SourceOffset, SourceOperator}; use crate::format::{BadDataPolicy, DataDeserializer, Format}; +use crate::sql::common::fs_schema::FieldValueType; +use crate::sql::common::{CheckpointBarrier, MetadataField}; pub trait BatchDeserializer: Send + 'static { fn deserialize_slice( diff --git a/src/streaming_runtime_operators/src/operators/stateless_physical_executor.rs b/src/streaming_runtime_operators/src/operators/stateless_physical_executor.rs index 1107ae68..11b1e307 100644 --- a/src/streaming_runtime_operators/src/operators/stateless_physical_executor.rs +++ b/src/streaming_runtime_operators/src/operators/stateless_physical_executor.rs @@ -24,8 +24,8 @@ use datafusion_proto::protobuf::PhysicalPlanNode; use futures::StreamExt; use prost::Message; -use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; use crate::factory::Registry; +use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; pub struct StatelessPhysicalExecutor { batch: Arc>>, diff --git a/src/streaming_runtime_operators/src/operators/value_execution.rs b/src/streaming_runtime_operators/src/operators/value_execution.rs index 79a948dd..dc93eeb6 100644 --- a/src/streaming_runtime_operators/src/operators/value_execution.rs +++ b/src/streaming_runtime_operators/src/operators/value_execution.rs @@ -15,11 +15,11 @@ use arrow_array::RecordBatch; use async_trait::async_trait; use futures::StreamExt; -use crate::sql::common::{CheckpointBarrier, Watermark}; use crate::core::StreamOutput; use crate::core::api::context::TaskContext; use crate::core::api::operator::{Collector, Operator}; use crate::operators::StatelessPhysicalExecutor; +use crate::sql::common::{CheckpointBarrier, Watermark}; pub struct ValueExecutionOperator { name: String, diff --git a/src/streaming_runtime_operators/src/operators/watermark/watermark_generator.rs b/src/streaming_runtime_operators/src/operators/watermark/watermark_generator.rs index d2720367..451a479b 100644 --- a/src/streaming_runtime_operators/src/operators/watermark/watermark_generator.rs +++ b/src/streaming_runtime_operators/src/operators/watermark/watermark_generator.rs @@ -25,11 +25,11 @@ use std::sync::Arc; use std::time::{Duration, SystemTime}; use tracing::debug; -use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark, from_nanos, to_millis}; use crate::core::StreamOutput; use crate::core::api::context::TaskContext; use crate::core::api::operator::{Collector, Operator}; use crate::factory::Registry; +use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark, from_nanos, to_millis}; use async_trait::async_trait; use protocol::function_stream_graph::ExpressionWatermarkConfig; diff --git a/src/streaming_runtime_operators/src/operators/windows/session_aggregating_window.rs b/src/streaming_runtime_operators/src/operators/windows/session_aggregating_window.rs index 9da970ad..9b32107c 100644 --- a/src/streaming_runtime_operators/src/operators/windows/session_aggregating_window.rs +++ b/src/streaming_runtime_operators/src/operators/windows/session_aggregating_window.rs @@ -36,17 +36,17 @@ use std::time::{Duration, SystemTime}; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; use tracing::info; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::core::state::OperatorStateStore; +use crate::factory::Registry; use crate::sql::common::converter::Converter; use crate::sql::common::{ CheckpointBarrier, FsSchema, FsSchemaRef, Watermark, from_nanos, to_nanos, }; use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; use crate::sql::schema::utils::window_arrow_struct; -use crate::core::StreamOutput; -use crate::core::api::context::TaskContext; -use crate::core::api::operator::{Collector, Operator}; -use crate::factory::Registry; -use crate::core::state::OperatorStateStore; use async_trait::async_trait; use protocol::function_stream_graph::SessionWindowAggregateOperator; // ============================================================================ diff --git a/src/streaming_runtime_operators/src/operators/windows/sliding_aggregating_window.rs b/src/streaming_runtime_operators/src/operators/windows/sliding_aggregating_window.rs index 371024a5..475554a0 100644 --- a/src/streaming_runtime_operators/src/operators/windows/sliding_aggregating_window.rs +++ b/src/streaming_runtime_operators/src/operators/windows/sliding_aggregating_window.rs @@ -33,13 +33,13 @@ use std::time::{Duration, SystemTime}; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; use tracing::info; -use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark, from_nanos, to_nanos}; -use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; use crate::core::StreamOutput; use crate::core::api::context::TaskContext; use crate::core::api::operator::{Collector, Operator}; -use crate::factory::Registry; use crate::core::state::OperatorStateStore; +use crate::factory::Registry; +use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark, from_nanos, to_nanos}; +use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; use async_trait::async_trait; use protocol::function_stream_graph::SlidingWindowAggregateOperator; // ============================================================================ diff --git a/src/streaming_runtime_operators/src/operators/windows/tumbling_aggregating_window.rs b/src/streaming_runtime_operators/src/operators/windows/tumbling_aggregating_window.rs index 1d3a76b6..ece41718 100644 --- a/src/streaming_runtime_operators/src/operators/windows/tumbling_aggregating_window.rs +++ b/src/streaming_runtime_operators/src/operators/windows/tumbling_aggregating_window.rs @@ -34,15 +34,15 @@ use std::time::{Duration, SystemTime}; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; use tracing::{info, warn}; -use crate::sql::common::time_utils::print_time; -use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark, from_nanos, to_nanos}; -use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; -use crate::sql::schema::utils::add_timestamp_field_arrow; use crate::core::StreamOutput; use crate::core::api::context::TaskContext; use crate::core::api::operator::{Collector, Operator}; -use crate::factory::Registry; use crate::core::state::OperatorStateStore; +use crate::factory::Registry; +use crate::sql::common::time_utils::print_time; +use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark, from_nanos, to_nanos}; +use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; +use crate::sql::schema::utils::add_timestamp_field_arrow; use async_trait::async_trait; use protocol::function_stream_graph::TumblingWindowAggregateOperator; diff --git a/src/streaming_runtime_operators/src/operators/windows/window_function.rs b/src/streaming_runtime_operators/src/operators/windows/window_function.rs index df4f5a67..30bf601a 100644 --- a/src/streaming_runtime_operators/src/operators/windows/window_function.rs +++ b/src/streaming_runtime_operators/src/operators/windows/window_function.rs @@ -26,16 +26,16 @@ use std::time::SystemTime; use tokio::sync::mpsc::{UnboundedReceiver, unbounded_channel}; use tracing::{info, warn}; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::core::state::OperatorStateStore; +use crate::factory::Registry; use crate::sql::common::time_utils::print_time; use crate::sql::common::{ CheckpointBarrier, FsSchema, FsSchemaRef, Watermark, from_nanos, to_nanos, }; use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; -use crate::core::StreamOutput; -use crate::core::api::context::TaskContext; -use crate::core::api::operator::{Collector, Operator}; -use crate::factory::Registry; -use crate::core::state::OperatorStateStore; use async_trait::async_trait; // ============================================================================ From f6636a2e6491a7586cd18290410977d212b36550 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 19 May 2026 23:20:52 +0800 Subject: [PATCH 04/16] update --- .../src/operators/sink/delta/delta_commit.rs | 39 +++++++++++++++++-- .../src/operators/sink/delta/mod.rs | 6 +++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs b/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs index 5c5574fd..2fa3b6c8 100644 --- a/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs +++ b/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs @@ -23,7 +23,7 @@ use deltalake::kernel::transaction::CommitBuilder; use deltalake::kernel::{Action, Add, StructField, StructType}; use deltalake::protocol::{DeltaOperation, SaveMode}; use deltalake::{DeltaTable, open_table_with_storage_options}; -use tracing::info; +use tracing::{info, instrument}; use url::Url; use super::DeltaSinkError; @@ -93,6 +93,12 @@ impl DeltaTableCommitter { match open_result { Ok(table) => { + let version = table.version(); + info!( + table_uri = %self.table_uri, + ?version, + "opened existing delta table" + ); self.table = Some(table); return Ok(()); } @@ -127,16 +133,39 @@ impl DeltaTableCommitter { .await .map_err(|e| DeltaSinkError::CommitterFailed(e.to_string()))?; + info!( + table_uri = %self.table_uri, + column_count = columns.len(), + version = table.version(), + "created new delta table" + ); self.table = Some(table); Ok(()) } + #[instrument(skip(self), fields(table_uri = %self.table_uri, epoch))] pub async fn commit_checkpoint(&mut self, epoch: u64) -> Result<(), DeltaSinkError> { if self.uncommitted.is_empty() { return Ok(()); } let files = std::mem::take(&mut self.uncommitted); + let file_count = files.len(); + let total_bytes: u64 = files.iter().map(|f| f.size_bytes).sum(); + let total_records: u64 = files.iter().map(|f| f.record_count).sum(); + let paths: Vec<&str> = files.iter().map(|f| f.path.as_str()).collect(); + + let prev_version = self.table.as_ref().and_then(|t| t.version()); + + info!( + file_count, + total_bytes, + total_records, + ?prev_version, + paths = ?paths, + "committing delta checkpoint" + ); + self.ensure_table().await?; let table = self @@ -190,8 +219,12 @@ impl DeltaTableCommitter { info!( epoch, version, - file_count = files.len(), - "successfully committed delta checkpoint (2PC completed)" + file_count, + total_bytes, + total_records, + ?prev_version, + paths = ?paths, + "delta checkpoint committed" ); Ok(()) } diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs index 10b6d2fd..427de558 100644 --- a/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs @@ -274,6 +274,12 @@ impl DeltaSinkOperator { if let Some(committer) = self.committer.as_mut() { if committer.has_uncommitted() { + info!( + table = %self.table_name, + epoch, + subtask_idx, + "delta sink triggering checkpoint commit" + ); committer.commit_checkpoint(epoch).await?; } } From 72ce3ef0a33c72a3b98170a395bf2382bfdbbd18 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 20 May 2026 00:05:29 +0800 Subject: [PATCH 05/16] update --- .../src/operators/sink/delta/mod.rs | 14 ++++++++++++- .../src/operators/source/kafka/mod.rs | 20 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs index 427de558..f6e74c1d 100644 --- a/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs @@ -330,9 +330,21 @@ impl Operator for DeltaSinkOperator { ctx: &mut TaskContext, _collector: &mut dyn Collector, ) -> anyhow::Result<()> { - self.pending_bytes += batch.get_array_memory_size(); + let batch_rows = batch.num_rows(); + let batch_bytes = batch.get_array_memory_size(); + self.pending_bytes += batch_bytes; self.pending.push(batch); + debug!( + table = %self.table_name, + subtask_idx = ctx.subtask_index, + batch_rows, + batch_bytes, + pending_batches = self.pending.len(), + pending_bytes = self.pending_bytes, + "delta sink received data" + ); + if self.pending_bytes > self.early_flush_threshold_bytes { debug!( bytes = self.pending_bytes, diff --git a/src/streaming_runtime_operators/src/operators/source/kafka/mod.rs b/src/streaming_runtime_operators/src/operators/source/kafka/mod.rs index dd952239..78423540 100644 --- a/src/streaming_runtime_operators/src/operators/source/kafka/mod.rs +++ b/src/streaming_runtime_operators/src/operators/source/kafka/mod.rs @@ -341,6 +341,14 @@ impl SourceOperator for KafkaSourceOperator { if let Some(payload) = msg.payload() { let topic = msg.topic(); + debug!( + topic, + partition, + offset, + payload_bytes = payload.len(), + timestamp_ms = timestamp, + "kafka source consumed message" + ); let connector_metadata = if !self.metadata_fields.is_empty() { let mut meta = HashMap::new(); @@ -384,6 +392,13 @@ impl SourceOperator for KafkaSourceOperator { && let Some(batch) = self.deserializer.flush_buffer()? { self.last_flush_time = Instant::now(); + debug!( + num_rows = batch.num_rows(), + num_columns = batch.num_columns(), + flush_by_size = should_flush_by_size, + flush_by_time = should_flush_by_time, + "kafka source emitting record batch" + ); return Ok(SourceEvent::Data(batch)); } @@ -398,6 +413,11 @@ impl SourceOperator for KafkaSourceOperator { && let Some(batch) = self.deserializer.flush_buffer()? { self.last_flush_time = Instant::now(); + debug!( + num_rows = batch.num_rows(), + num_columns = batch.num_columns(), + "kafka source emitting record batch (poll timeout flush)" + ); return Ok(SourceEvent::Data(batch)); } Ok(SourceEvent::Idle) From 76d020414efd5b2c18db3924c628eaa5842b571b Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 20 May 2026 00:40:39 +0800 Subject: [PATCH 06/16] update --- .../src/operators/sink/delta/delta_commit.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs b/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs index 2fa3b6c8..c837ce9f 100644 --- a/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs +++ b/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs @@ -19,6 +19,7 @@ use arrow_ipc::writer::StreamWriter; use arrow_schema::Schema as ArrowSchema; use deltalake::errors::DeltaTableError; use deltalake::kernel::engine::arrow_conversion::TryIntoKernel as _; +use deltalake::kernel::schema::cast::normalize_for_delta; use deltalake::kernel::transaction::CommitBuilder; use deltalake::kernel::{Action, Add, StructField, StructType}; use deltalake::protocol::{DeltaOperation, SaveMode}; @@ -250,7 +251,10 @@ fn arrow_schema_to_delta_columns(schema: &ArrowSchema) -> Result Date: Wed, 20 May 2026 10:12:11 +0800 Subject: [PATCH 07/16] fix(delta): strip system columns and normalize timestamps for writes Exclude _timestamp/_updating_meta from Delta catalog and Parquet schema, cast user columns to Delta-compatible types, and support multiple timestamp units in watermark max extraction. Co-authored-by: Cursor --- .../src/factory/connector/delta.rs | 6 +- .../src/operators/sink/delta/delta_commit.rs | 165 +++++++++++++++++- .../src/operators/sink/delta/mod.rs | 48 +++-- .../watermark/watermark_generator.rs | 35 +++- 4 files changed, 230 insertions(+), 24 deletions(-) diff --git a/src/streaming_runtime_operators/src/factory/connector/delta.rs b/src/streaming_runtime_operators/src/factory/connector/delta.rs index 1b5aab0a..a44e5464 100644 --- a/src/streaming_runtime_operators/src/factory/connector/delta.rs +++ b/src/streaming_runtime_operators/src/factory/connector/delta.rs @@ -24,7 +24,9 @@ use crate::factory::connector::sink_props_codec::{ }; use crate::factory::global::Registry; use crate::factory::operator_constructor::OperatorConstructor; -use crate::operators::sink::delta::{DeltaFormat, DeltaSinkOperator}; +use crate::operators::sink::delta::{ + DeltaFormat, DeltaSinkOperator, strip_streaming_system_columns_arc, +}; use crate::operators::sink::filesystem::compression_from_str; use crate::sql::common::FsSchema; use crate::sql::common::constants::connection_format_value; @@ -47,7 +49,7 @@ impl OperatorConstructor for DeltaSinkDispatcher { .map(|fs| FsSchema::try_from(fs.clone())) .transpose() .map_err(|e| anyhow::anyhow!("invalid fs_schema for delta sink: {e}"))? - .map(|fs| fs.schema); + .and_then(|fs| strip_streaming_system_columns_arc(fs.schema)); let format = props .get(opt::FORMAT) diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs b/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs index c837ce9f..cf26f7ad 100644 --- a/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs +++ b/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs @@ -14,9 +14,10 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; +use arrow::compute::cast; use arrow_array::RecordBatch; use arrow_ipc::writer::StreamWriter; -use arrow_schema::Schema as ArrowSchema; +use arrow_schema::{DataType, Field, FieldRef, Schema as ArrowSchema, TimeUnit}; use deltalake::errors::DeltaTableError; use deltalake::kernel::engine::arrow_conversion::TryIntoKernel as _; use deltalake::kernel::schema::cast::normalize_for_delta; @@ -27,8 +28,42 @@ use deltalake::{DeltaTable, open_table_with_storage_options}; use tracing::{info, instrument}; use url::Url; +use crate::sql::common::{TIMESTAMP_FIELD, UPDATING_META_FIELD}; + use super::DeltaSinkError; +/// Streaming-internal columns that must not be persisted to external sinks. +pub fn is_streaming_system_column(name: &str) -> bool { + name == TIMESTAMP_FIELD || name == UPDATING_META_FIELD +} + +/// Remove `_timestamp` / `_updating_meta` from a schema (e.g. connector `fs_schema` may inject them). +pub fn strip_streaming_system_columns(schema: &ArrowSchema) -> ArrowSchema { + let fields: Vec = schema + .fields() + .iter() + .filter(|f| !is_streaming_system_column(f.name())) + .cloned() + .collect(); + ArrowSchema::new(fields) +} + +pub fn strip_streaming_system_columns_arc(schema: Arc) -> Option> { + let stripped = strip_streaming_system_columns(schema.as_ref()); + if stripped.fields().is_empty() { + return None; + } + let had_system = schema + .fields() + .iter() + .any(|f| is_streaming_system_column(f.name())); + if had_system { + Some(Arc::new(stripped)) + } else { + Some(schema) + } +} + pub struct UncommittedDataFile { pub path: String, pub size_bytes: u64, @@ -42,6 +77,8 @@ pub struct DeltaTableCommitter { table: Option, /// Precomputed from catalog `fs_schema` at startup, or lazily from the first batch. delta_columns: Option>, + /// Arrow 55 schema for Parquet writes (timestamps normalized to microsecond). + write_schema: Option>, } impl DeltaTableCommitter { @@ -50,7 +87,11 @@ impl DeltaTableCommitter { storage_options: HashMap, catalog_schema: Option>, ) -> Result { - let delta_columns = catalog_schema + let user_schema = catalog_schema.and_then(strip_streaming_system_columns_arc); + let write_schema = user_schema + .as_ref() + .map(|s| Arc::new(normalize_arrow_schema_for_delta(s))); + let delta_columns = user_schema .as_deref() .map(arrow_schema_to_delta_columns) .transpose()?; @@ -61,13 +102,30 @@ impl DeltaTableCommitter { uncommitted: Vec::new(), table: None, delta_columns, + write_schema, }) } - /// Fallback when catalog schema is absent: derive columns from the first flushed batch. + pub fn write_schema(&self) -> Option> { + self.write_schema.clone() + } + + /// Fallback when catalog schema is absent: derive user columns from the first flushed batch. pub fn update_schema(&mut self, schema: Arc) -> Result<(), DeltaSinkError> { + let user_schema = strip_streaming_system_columns_arc(schema).ok_or_else(|| { + DeltaSinkError::CommitterFailed( + "cannot derive delta table schema: no user columns after removing streaming \ + system columns (_timestamp, _updating_meta)" + .into(), + ) + })?; if self.delta_columns.is_none() { - self.delta_columns = Some(arrow_schema_to_delta_columns(&schema)?); + self.delta_columns = Some(arrow_schema_to_delta_columns(user_schema.as_ref())?); + } + if self.write_schema.is_none() { + self.write_schema = Some(Arc::new(normalize_arrow_schema_for_delta( + user_schema.as_ref(), + ))); } Ok(()) } @@ -231,6 +289,105 @@ impl DeltaTableCommitter { } } +/// Normalize Arrow 55 schema for Delta Parquet writes (align with deltalake `normalize_for_delta`). +pub fn normalize_arrow_schema_for_delta(schema: &ArrowSchema) -> ArrowSchema { + let fields: Vec = schema + .fields() + .iter() + .map(|f| Arc::new(normalize_field_for_delta(f.as_ref()))) + .collect(); + ArrowSchema::new(fields) +} + +fn normalize_field_for_delta(field: &Field) -> Field { + let data_type = normalize_datatype_for_delta(field.data_type()); + if data_type == *field.data_type() { + field.clone() + } else { + field.clone().with_data_type(data_type) + } +} + +fn normalize_datatype_for_delta(dt: &DataType) -> DataType { + match dt { + DataType::Date64 => DataType::Date32, + DataType::Timestamp(TimeUnit::Second, tz) + | DataType::Timestamp(TimeUnit::Millisecond, tz) + | DataType::Timestamp(TimeUnit::Nanosecond, tz) => { + DataType::Timestamp(TimeUnit::Microsecond, tz.clone()) + } + DataType::Struct(fields) => { + let normalized: Vec = fields + .iter() + .map(|f| Arc::new(normalize_field_for_delta(f.as_ref()))) + .collect(); + DataType::Struct(normalized.into()) + } + DataType::List(inner) => { + DataType::List(Arc::new(normalize_field_for_delta(inner.as_ref()))) + } + DataType::LargeList(inner) => { + DataType::LargeList(Arc::new(normalize_field_for_delta(inner.as_ref()))) + } + DataType::FixedSizeList(inner, len) => { + DataType::FixedSizeList(Arc::new(normalize_field_for_delta(inner.as_ref())), *len) + } + DataType::Map(entries, sorted) => DataType::Map( + Arc::new(normalize_field_for_delta(entries.as_ref())), + *sorted, + ), + _ => dt.clone(), + } +} + +/// Cast record batches so on-disk Parquet matches the Delta table schema. +pub fn cast_batches_for_delta_write( + batches: &[RecordBatch], + target_schema: &ArrowSchema, +) -> Result, DeltaSinkError> { + let target = Arc::new(target_schema.clone()); + batches + .iter() + .map(|batch| cast_batch_for_delta_write(batch, &target)) + .collect() +} + +fn cast_batch_for_delta_write( + batch: &RecordBatch, + target_schema: &Arc, +) -> Result { + if batch.schema().as_ref() == target_schema.as_ref() { + return Ok(batch.clone()); + } + + let mut columns = Vec::with_capacity(target_schema.fields().len()); + for field in target_schema.fields() { + let col = batch.column_by_name(field.name()).ok_or_else(|| { + DeltaSinkError::CommitterFailed(format!( + "batch missing column '{}' required by delta write schema", + field.name() + )) + })?; + let casted = if col.data_type() == field.data_type() { + col.clone() + } else { + cast(col, field.data_type()).map_err(|e| { + DeltaSinkError::CommitterFailed(format!( + "failed to cast column '{}' from {:?} to {:?}: {e}", + field.name(), + col.data_type(), + field.data_type() + )) + })? + }; + columns.push(casted); + } + + RecordBatch::try_new(target_schema.clone(), columns).map_err(|e| { + DeltaSinkError::CommitterFailed(format!("failed to build delta write batch: {e}")) + }) +} + /// Bridge arrow 55 (runtime) schema to deltalake kernel schema via IPC. fn arrow_schema_to_delta_columns(schema: &ArrowSchema) -> Result, DeltaSinkError> { let empty = RecordBatch::new_empty(Arc::new(schema.clone())); diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs index f6e74c1d..30914f0f 100644 --- a/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs @@ -12,6 +12,8 @@ mod delta_commit; +pub use delta_commit::strip_streaming_system_columns_arc; + use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; @@ -21,7 +23,8 @@ use arrow_schema::Schema as ArrowSchema; use async_trait::async_trait; use bytes::Bytes; use delta_commit::{ - DeltaTableCommitter, UncommittedDataFile, build_delta_storage_options, resolve_delta_table_uri, + DeltaTableCommitter, UncommittedDataFile, build_delta_storage_options, + cast_batches_for_delta_write, resolve_delta_table_uri, }; use object_store::aws::AmazonS3Builder; use object_store::path::Path as ObjectStorePath; @@ -94,6 +97,8 @@ pub struct DeltaSinkOperator { s3_bucket: Option, sink_path: String, catalog_schema: Option>, + /// Normalized schema for Parquet files (microsecond timestamps, etc.). + parquet_write_schema: Option>, } impl DeltaSinkOperator { @@ -171,7 +176,8 @@ impl DeltaSinkOperator { storage_options, s3_bucket, sink_path: path, - catalog_schema, + catalog_schema: catalog_schema.and_then(strip_streaming_system_columns_arc), + parquet_write_schema: None, }) } @@ -186,23 +192,35 @@ impl DeltaSinkOperator { return Ok(()); } - let fallback_schema = self - .catalog_schema - .is_none() - .then(|| self.pending[0].schema()); + let fallback_schema = self.catalog_schema.is_none().then(|| { + strip_streaming_system_columns_arc(self.pending[0].schema()) + .expect("batch has no user columns after removing streaming system columns") + }); let batches = std::mem::take(&mut self.pending); self.pending_bytes = 0; let record_count: u64 = batches.iter().map(|b| b.num_rows() as u64).sum(); let format = self.format; let compression = self.parquet_compression; + let parquet_write_schema = self.parquet_write_schema.clone(); - let bytes = tokio::task::spawn_blocking(move || match format { - DeltaFormat::Csv => FormatEncoder::encode_csv(&batches), - DeltaFormat::Parquet => FormatEncoder::encode_parquet(&batches, compression), - DeltaFormat::JsonL => FormatEncoder::encode_jsonl(&batches), - DeltaFormat::Avro => FormatEncoder::encode_avro(&batches), - DeltaFormat::Orc => FormatEncoder::encode_orc(&batches), + let bytes = tokio::task::spawn_blocking(move || { + let batches = if format == DeltaFormat::Parquet { + if let Some(ref schema) = parquet_write_schema { + cast_batches_for_delta_write(&batches, schema)? + } else { + batches + } + } else { + batches + }; + match format { + DeltaFormat::Csv => FormatEncoder::encode_csv(&batches), + DeltaFormat::Parquet => FormatEncoder::encode_parquet(&batches, compression), + DeltaFormat::JsonL => FormatEncoder::encode_jsonl(&batches), + DeltaFormat::Avro => FormatEncoder::encode_avro(&batches), + DeltaFormat::Orc => FormatEncoder::encode_orc(&batches), + } }) .await .map_err(|e| DeltaSinkError::SerializationPanic(e.to_string()))? @@ -302,11 +320,13 @@ impl Operator for DeltaSinkOperator { self.table_uri = Some(table_uri.clone()); if self.format == DeltaFormat::Parquet { - self.committer = Some(DeltaTableCommitter::try_new( + let committer = DeltaTableCommitter::try_new( table_uri, self.storage_options.clone(), self.catalog_schema.clone(), - )?); + )?; + self.parquet_write_schema = committer.write_schema(); + self.committer = Some(committer); } else { warn!( format = ?self.format, diff --git a/src/streaming_runtime_operators/src/operators/watermark/watermark_generator.rs b/src/streaming_runtime_operators/src/operators/watermark/watermark_generator.rs index 451a479b..b49ca607 100644 --- a/src/streaming_runtime_operators/src/operators/watermark/watermark_generator.rs +++ b/src/streaming_runtime_operators/src/operators/watermark/watermark_generator.rs @@ -13,8 +13,12 @@ use anyhow::{Result, anyhow}; use arrow::compute::kernels::aggregate; use arrow_array::cast::AsArray; -use arrow_array::types::TimestampNanosecondType; +use arrow_array::types::{ + TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, + TimestampSecondType, +}; use arrow_array::{RecordBatch, TimestampNanosecondArray}; +use arrow_schema::{DataType, TimeUnit}; use bincode::{Decode, Encode}; use datafusion::physical_expr::PhysicalExpr; use datafusion_proto::physical_plan::DefaultPhysicalExtensionCodec; @@ -78,9 +82,32 @@ impl WatermarkGeneratorOperator { fn extract_max_timestamp(&self, batch: &RecordBatch) -> Option { let ts_column = batch.column(self.timestamp_index); - let arr = ts_column.as_primitive::(); - let max_ts = aggregate::max(arr)?; - Some(from_nanos(max_ts as u128)) + match ts_column.data_type() { + DataType::Timestamp(TimeUnit::Nanosecond, _) => { + let arr = ts_column.as_primitive::(); + aggregate::max(arr).map(|v| from_nanos(v as u128)) + } + DataType::Timestamp(TimeUnit::Microsecond, _) => { + let arr = ts_column.as_primitive::(); + aggregate::max(arr).map(|v| from_nanos((v as u128) * 1_000)) + } + DataType::Timestamp(TimeUnit::Millisecond, _) => { + let arr = ts_column.as_primitive::(); + aggregate::max(arr).map(|v| from_nanos((v as u128) * 1_000_000)) + } + DataType::Timestamp(TimeUnit::Second, _) => { + let arr = ts_column.as_primitive::(); + aggregate::max(arr).map(|v| from_nanos((v as u128) * 1_000_000_000)) + } + other => { + debug!( + ?other, + index = self.timestamp_index, + "skip max timestamp: column is not a timestamp array" + ); + None + } + } } fn evaluate_watermark(&self, batch: &RecordBatch) -> Result { From 1bd7adef7a64b5b9a4299d7a0c71397b353e3d18 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 20 May 2026 13:07:02 +0800 Subject: [PATCH 08/16] fix(watermark): accept expression results in any timestamp unit Planner may evaluate watermark expressions to millisecond (or other) timestamp arrays after projection; unify max extraction across units instead of requiring TimestampNanosecondArray only. Co-authored-by: Cursor --- .../watermark/watermark_generator.rs | 69 +++++++++---------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/src/streaming_runtime_operators/src/operators/watermark/watermark_generator.rs b/src/streaming_runtime_operators/src/operators/watermark/watermark_generator.rs index b49ca607..3120d524 100644 --- a/src/streaming_runtime_operators/src/operators/watermark/watermark_generator.rs +++ b/src/streaming_runtime_operators/src/operators/watermark/watermark_generator.rs @@ -17,7 +17,7 @@ use arrow_array::types::{ TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, }; -use arrow_array::{RecordBatch, TimestampNanosecondArray}; +use arrow_array::{Array, RecordBatch}; use arrow_schema::{DataType, TimeUnit}; use bincode::{Decode, Encode}; use datafusion::physical_expr::PhysicalExpr; @@ -81,33 +81,7 @@ impl WatermarkGeneratorOperator { } fn extract_max_timestamp(&self, batch: &RecordBatch) -> Option { - let ts_column = batch.column(self.timestamp_index); - match ts_column.data_type() { - DataType::Timestamp(TimeUnit::Nanosecond, _) => { - let arr = ts_column.as_primitive::(); - aggregate::max(arr).map(|v| from_nanos(v as u128)) - } - DataType::Timestamp(TimeUnit::Microsecond, _) => { - let arr = ts_column.as_primitive::(); - aggregate::max(arr).map(|v| from_nanos((v as u128) * 1_000)) - } - DataType::Timestamp(TimeUnit::Millisecond, _) => { - let arr = ts_column.as_primitive::(); - aggregate::max(arr).map(|v| from_nanos((v as u128) * 1_000_000)) - } - DataType::Timestamp(TimeUnit::Second, _) => { - let arr = ts_column.as_primitive::(); - aggregate::max(arr).map(|v| from_nanos((v as u128) * 1_000_000_000)) - } - other => { - debug!( - ?other, - index = self.timestamp_index, - "skip max timestamp: column is not a timestamp array" - ); - None - } - } + max_timestamp_from_array(batch.column(self.timestamp_index).as_ref()).map(from_nanos) } fn evaluate_watermark(&self, batch: &RecordBatch) -> Result { @@ -116,15 +90,40 @@ impl WatermarkGeneratorOperator { .evaluate(batch)? .into_array(batch.num_rows())?; - let typed_array = watermark_array - .as_any() - .downcast_ref::() - .ok_or_else(|| anyhow!("watermark expression must return TimestampNanosecondArray"))?; + let max_nanos = max_timestamp_from_array(watermark_array.as_ref()).ok_or_else(|| { + anyhow!( + "watermark expression must return a timestamp array, got {:?}", + watermark_array.data_type() + ) + })?; - let max_watermark_nanos = aggregate::max(typed_array) - .ok_or_else(|| anyhow!("failed to extract max watermark from batch"))?; + Ok(from_nanos(max_nanos)) + } +} - Ok(from_nanos(max_watermark_nanos as u128)) +/// Returns event time as nanoseconds since epoch (internal watermark representation). +fn max_timestamp_from_array(array: &dyn Array) -> Option { + match array.data_type() { + DataType::Timestamp(TimeUnit::Nanosecond, _) => { + let arr = array.as_primitive::(); + aggregate::max(arr).map(|v| v as u128) + } + DataType::Timestamp(TimeUnit::Microsecond, _) => { + let arr = array.as_primitive::(); + aggregate::max(arr).map(|v| (v as u128) * 1_000) + } + DataType::Timestamp(TimeUnit::Millisecond, _) => { + let arr = array.as_primitive::(); + aggregate::max(arr).map(|v| (v as u128) * 1_000_000) + } + DataType::Timestamp(TimeUnit::Second, _) => { + let arr = array.as_primitive::(); + aggregate::max(arr).map(|v| (v as u128) * 1_000_000_000) + } + other => { + debug!(?other, "skip max timestamp: not a timestamp array"); + None + } } } From dd4b04456da61a63c1b68c35006c8df7ca1b50e4 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 20 May 2026 21:51:11 +0800 Subject: [PATCH 09/16] update --- Cargo.lock | 1208 ++++++++++------- src/streaming_runtime_operators/Cargo.toml | 2 +- .../src/operators/sink/delta/mod.rs | 53 + 3 files changed, 750 insertions(+), 513 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 47828130..4f1a1dfe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -140,12 +140,12 @@ checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" dependencies = [ "bigdecimal", "bon", - "digest", + "digest 0.10.7", "log", "miniz_oxide", "num-bigint", "quad-rand", - "rand 0.9.2", + "rand 0.9.4", "regex-lite", "serde", "serde_bytes", @@ -215,23 +215,23 @@ dependencies = [ [[package]] name = "arrow" -version = "57.3.0" +version = "57.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4754a624e5ae42081f464514be454b39711daae0458906dacde5f4c632f33a8" +checksum = "3bd47f2a6ddc39244bd722a27ee5da66c03369d087b9e024eafdb03e98b98ea7" dependencies = [ - "arrow-arith 57.3.0", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-cast 57.3.0", - "arrow-csv 57.3.0", - "arrow-data 57.3.0", - "arrow-ipc 57.3.0", - "arrow-json 57.3.0", - "arrow-ord 57.3.0", - "arrow-row 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", - "arrow-string 57.3.0", + "arrow-arith 57.3.1", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-cast 57.3.1", + "arrow-csv 57.3.1", + "arrow-data 57.3.1", + "arrow-ipc 57.3.1", + "arrow-json 57.3.1", + "arrow-ord 57.3.1", + "arrow-row 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", + "arrow-string 57.3.1", ] [[package]] @@ -271,14 +271,14 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "57.3.0" +version = "57.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7b3141e0ec5145a22d8694ea8b6d6f69305971c4fa1c1a13ef0195aef2d678b" +checksum = "7c7bbd679c5418b8639b92be01f361d60013c4906574b578b77b63c78356594c" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-data 57.3.1", + "arrow-schema 57.3.1", "chrono", "num-traits", ] @@ -332,14 +332,14 @@ dependencies = [ [[package]] name = "arrow-array" -version = "57.3.0" +version = "57.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8955af33b25f3b175ee10af580577280b4bd01f7e823d94c7cdef7cf8c9aef" +checksum = "c8a4ab47b3f3eac60f7fd31b81e9028fda018607bcc63451aca4f2b755269862" dependencies = [ "ahash", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", + "arrow-buffer 57.3.1", + "arrow-data 57.3.1", + "arrow-schema 57.3.1", "chrono", "chrono-tz", "half", @@ -392,9 +392,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "57.3.0" +version = "57.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c697ddca96183182f35b3a18e50b9110b11e916d7b7799cbfd4d34662f2c56c2" +checksum = "0d18b89b4c4f4811d0858175e79541fe98e33e18db3b011708bc287b1240593f" dependencies = [ "bytes", "half", @@ -457,16 +457,16 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "57.3.0" +version = "57.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "646bbb821e86fd57189c10b4fcdaa941deaf4181924917b0daa92735baa6ada5" +checksum = "722b5c41dd1d14d0a879a1bce92c6fe33f546101bb2acce57a209825edd075b3" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-ord 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-data 57.3.1", + "arrow-ord 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", "atoi", "base64", "chrono", @@ -515,13 +515,13 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "57.3.0" +version = "57.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da746f4180004e3ce7b83c977daf6394d768332349d3d913998b10a120b790a" +checksum = "27ddb80a4848e03b1655af496d5ac2563a779e5742fcb48f2ca2e089c9cd2197" dependencies = [ - "arrow-array 57.3.0", - "arrow-cast 57.3.0", - "arrow-schema 57.3.0", + "arrow-array 57.3.1", + "arrow-cast 57.3.1", + "arrow-schema 57.3.1", "chrono", "csv", "csv-core", @@ -569,12 +569,12 @@ dependencies = [ [[package]] name = "arrow-data" -version = "57.3.0" +version = "57.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fdd994a9d28e6365aa78e15da3f3950c0fdcea6b963a12fa1c391afb637b304" +checksum = "c1683705c63dcf0d18972759eda48489028cbbff67af7d6bef2c6b7b74ab778a" dependencies = [ - "arrow-buffer 57.3.0", - "arrow-schema 57.3.0", + "arrow-buffer 57.3.1", + "arrow-schema 57.3.1", "half", "num-integer", "num-traits", @@ -623,17 +623,17 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "57.3.0" +version = "57.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf7df950701ab528bf7c0cf7eeadc0445d03ef5d6ffc151eaae6b38a58feff1" +checksum = "8cf72d04c07229fbf4dbebe7145cac37d7cf7ec582fe705c6b92cb314af096ab" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-data 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", "flatbuffers 25.12.19", - "lz4_flex 0.12.1", + "lz4_flex 0.12.2", "zstd", ] @@ -664,7 +664,7 @@ dependencies = [ "arrow-schema 55.2.0", "chrono", "half", - "indexmap 2.13.1", + "indexmap 2.14.0", "lexical-core 1.0.6", "memchr", "num", @@ -675,18 +675,18 @@ dependencies = [ [[package]] name = "arrow-json" -version = "57.3.0" +version = "57.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ff8357658bedc49792b13e2e862b80df908171275f8e6e075c460da5ee4bf86" +checksum = "a84a905f41fedfcd7679813c89a61dc369c0f932b27aa8dcc6aa051cc781a97d" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-cast 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-cast 57.3.1", + "arrow-data 57.3.1", + "arrow-schema 57.3.1", "chrono", "half", - "indexmap 2.13.1", + "indexmap 2.14.0", "itoa", "lexical-core 1.0.6", "memchr", @@ -711,7 +711,7 @@ dependencies = [ "arrow-select 58.3.0", "chrono", "half", - "indexmap 2.13.1", + "indexmap 2.14.0", "itoa", "lexical-core 1.0.6", "memchr", @@ -737,15 +737,15 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "57.3.0" +version = "57.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d8f1870e03d4cbed632959498bcc84083b5a24bded52905ae1695bd29da45b" +checksum = "082342947d4e5a2bcccf029a0a0397e21cb3bb8421edd9571d34fb5dd2670256" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-data 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", ] [[package]] @@ -776,14 +776,14 @@ dependencies = [ [[package]] name = "arrow-row" -version = "57.3.0" +version = "57.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18228633bad92bff92a95746bbeb16e5fc318e8382b75619dec26db79e4de4c0" +checksum = "e3a931b520a2a5e22033e01a6f2486b4cdc26f9106b759abeebc320f125e94d7" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-data 57.3.1", + "arrow-schema 57.3.1", "half", ] @@ -818,11 +818,11 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "57.3.0" +version = "57.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c872d36b7bf2a6a6a2b40de9156265f0242910791db366a2c17476ba8330d68" +checksum = "e4cf0d4a6609679e03002167a61074a21d7b1ad9ea65e462b2c0a97f8a3b2bc6" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "serde_core", "serde_json", ] @@ -833,7 +833,7 @@ version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "serde", "serde_core", ] @@ -868,15 +868,15 @@ dependencies = [ [[package]] name = "arrow-select" -version = "57.3.0" +version = "57.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68bf3e3efbd1278f770d67e5dc410257300b161b93baedb3aae836144edcaf4b" +checksum = "0b320d86a9806923663bb0fd9baa65ecaba81cb0cd77ff8c1768b9716b4ef891" dependencies = [ "ahash", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-data 57.3.1", + "arrow-schema 57.3.1", "num-traits", ] @@ -913,15 +913,15 @@ dependencies = [ [[package]] name = "arrow-string" -version = "57.3.0" +version = "57.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85e968097061b3c0e9fe3079cf2e703e487890700546b5b0647f60fca1b5a8d8" +checksum = "b493e99162e5764077e7823e50ba284858d365922631c7aaefe9487b1abd02c2" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-data 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", "memchr", "num-traits", "regex", @@ -1061,9 +1061,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-config" -version = "1.8.14" +version = "1.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a8fc176d53d6fe85017f230405e3255cedb4a02221cb55ed6d76dccbbb099b2" +checksum = "50f156acdd2cf55f5aa53ee416c4ac851cf1222694506c0b1f78c85695e9ca9d" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1081,7 +1081,7 @@ dependencies = [ "fastrand", "hex", "http 1.4.0", - "ring", + "sha1", "time", "tokio", "tracing", @@ -1091,9 +1091,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.13" +version = "1.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d203b0bf2626dcba8665f5cd0871d7c2c0930223d6b6be9097592fea21242d0" +checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -1103,9 +1103,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.16.3" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" dependencies = [ "aws-lc-sys", "zeroize", @@ -1113,9 +1113,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.40.0" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" dependencies = [ "cc", "cmake", @@ -1125,9 +1125,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.7.1" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede2ddc593e6c8acc6ce3358c28d6677a6dc49b65ba4b37a2befe14a11297e75" +checksum = "5dcd93c82209ac7413532388067dce79be5a8780c1786e5fae3df22e4dee2864" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -1148,11 +1148,35 @@ dependencies = [ "uuid", ] +[[package]] +name = "aws-sdk-dynamodb" +version = "1.111.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc418346e3cb248c7d59e642acbcb06488b7c7cd2ba6ebc79e8003f618c60099" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + [[package]] name = "aws-sdk-sso" -version = "1.95.0" +version = "1.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00c5ff27c6ba2cbd95e6e26e2e736676fdf6bcf96495b187733f521cfe4ce448" +checksum = "d69c77aafa20460c68b6b3213c84f6423b6e76dbf89accd3e1789a686ffd9489" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1174,9 +1198,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.97.0" +version = "1.100.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d186f1e5a3694a188e5a0640b3115ccc6e084d104e16fd6ba968dca072ffef8" +checksum = "1c7e7b09346d5ca22a2a08267555843a6a0127fb20d8964cb6ecfb8fdb190225" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1198,9 +1222,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.99.0" +version = "1.103.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9acba7c62f3d4e2408fa998a3a8caacd8b9a5b5549cf36e2372fbdae329d5449" +checksum = "c2249b81a2e73a8027c41c378463a81ec39b8510f184f2caab87de912af0f49b" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1223,9 +1247,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.4.1" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37411f8e0f4bea0c3ca0958ce7f18f6439db24d555dbd809787262cd00926aa9" +checksum = "68dc0b907359b120170613b5c09ccc61304eac3998ff6274b97d93ee6490115a" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -1234,20 +1258,20 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac", + "hmac 0.13.0", "http 0.2.12", "http 1.4.0", "percent-encoding", - "sha2", + "sha2 0.11.0", "time", "tracing", ] [[package]] name = "aws-smithy-async" -version = "1.2.13" +version = "1.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc50d0f63e714784b84223abd7abbc8577de8c35d699e0edd19f0a88a08ae13" +checksum = "2ffcaf626bdda484571968400c326a244598634dc75fd451325a54ad1a59acfc" dependencies = [ "futures-util", "pin-project-lite", @@ -1256,9 +1280,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.63.5" +version = "0.63.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d619373d490ad70966994801bc126846afaa0d1ee920697a031f0cf63f2568e7" +checksum = "ba1ab2dc1c2c3749ead27180d333c42f11be8b0e934058fb4b2258ee8dbe5231" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -1277,9 +1301,9 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.1.11" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00ccbb08c10f6bcf912f398188e42ee2eab5f1767ce215a02a73bc5df1bbdd95" +checksum = "6a2f165a7feee6f263028b899d0a181987f4fa7179a6411a32a439fba7c5f769" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -1301,27 +1325,29 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.62.4" +version = "0.62.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b3a779093e18cad88bbae08dc4261e1d95018c4c5b9356a52bcae7c0b6e9bb" +checksum = "517089205f18ab4adc5a3e02888cb139bbbbb2e168eac9f396216925d1fbeaf5" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", ] [[package]] name = "aws-smithy-observability" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d3f39d5bb871aaf461d59144557f16d5927a5248a983a40654d9cf3b9ba183b" +checksum = "a06c2315d173edbf1920da8ba3a7189695827002e4c0fc961973ab1c54abca9c" dependencies = [ "aws-smithy-runtime-api", ] [[package]] name = "aws-smithy-query" -version = "0.60.14" +version = "0.60.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f76a580e3d8f8961e5d48763214025a2af65c2fa4cd1fb7f270a0e107a71b0" +checksum = "1a56d79744fb3edb5d722ef79d86081e121d3b9422cb209eb03aea6aa4f21ebd" dependencies = [ "aws-smithy-types", "urlencoding", @@ -1329,15 +1355,16 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.10.2" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ccf7f6eba8b2dcf8ce9b74806c6c185659c311665c4bf8d6e71ebd454db6bf" +checksum = "b8e6f5caf6fea86f8c2206541ab5857cfcda9013426cdbe8fa0098b9e2d32182" dependencies = [ "aws-smithy-async", "aws-smithy-http", "aws-smithy-http-client", "aws-smithy-observability", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "bytes", "fastrand", @@ -1354,11 +1381,12 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.11.5" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4af6e5def28be846479bbeac55aa4603d6f7986fc5da4601ba324dd5d377516" +checksum = "dc117c179ecf39a62a0a3f49f600e9ac26a7ad7dd172177999f83933af776c32" dependencies = [ "aws-smithy-async", + "aws-smithy-runtime-api-macros", "aws-smithy-types", "bytes", "http 0.2.12", @@ -1369,15 +1397,38 @@ dependencies = [ "zeroize", ] +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "aws-smithy-schema" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.4.0", +] + [[package]] name = "aws-smithy-types" -version = "1.4.5" +version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ca2734c16913a45343b37313605d84e7d8b34a4611598ce1d25b35860a2bed3" +checksum = "056b66dbce2f81cc0c1e2b05bb402eb58f8a3530479d650efadd5bbae9a4050b" dependencies = [ "base64-simd", "bytes", "bytes-utils", + "futures-core", "http 0.2.12", "http 1.4.0", "http-body 0.4.6", @@ -1390,22 +1441,24 @@ dependencies = [ "ryu", "serde", "time", + "tokio", + "tokio-util", ] [[package]] name = "aws-smithy-xml" -version = "0.60.14" +version = "0.60.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b53543b4b86ed43f051644f704a98c7291b3618b67adf057ee77a366fa52fcaa" +checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "1.3.13" +version = "1.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0470cc047657c6e286346bdf10a8719d26efd6a91626992e0e64481e44323e96" +checksum = "2f4bbcaa9304ea40902d3d5f42a0428d1bd895a2b0f6999436fb279ffddc58ac" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -1550,7 +1603,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1570,9 +1623,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] name = "bitmaps" @@ -1610,14 +1663,14 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] name = "blake3" -version = "1.8.4" +version = "1.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" dependencies = [ "arrayref", "arrayvec", @@ -1636,6 +1689,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bon" version = "3.9.1" @@ -1703,12 +1765,12 @@ dependencies = [ "chrono", "crc", "futures", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "object_store 0.13.2", "parquet 58.3.0", "percent-encoding", - "rand 0.9.2", + "rand 0.9.4", "reqwest 0.13.3", "roaring", "rustc_version", @@ -1831,7 +1893,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8144c22e24bbcf26ade86cb6501a0916c46b7e4787abdb0045a467eb1645a1d" dependencies = [ "ambient-authority", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -1862,9 +1924,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.59" +version = "1.2.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" dependencies = [ "find-msvc-tools", "jobserver", @@ -1953,9 +2015,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", "clap_derive", @@ -1975,9 +2037,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ "heck", "proc-macro2", @@ -2009,6 +2071,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746" + [[package]] name = "cobs" version = "0.3.0" @@ -2060,6 +2128,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -2209,7 +2283,7 @@ dependencies = [ "rustc-hash 2.1.2", "serde", "serde_derive", - "sha2", + "sha2 0.10.9", "smallvec", "target-lexicon", "wasmtime-internal-math", @@ -2381,7 +2455,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "crossterm_winapi", "document-features", "parking_lot", @@ -2414,6 +2488,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.4.0" @@ -2452,11 +2535,20 @@ version = "0.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a949c44fcacbbbb7ada007dc7acb34603dd97cd47de5d054f2b6493ecebb483" +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "curl-sys" -version = "0.4.87+curl-8.19.0" +version = "0.4.88+curl-8.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a460380f0ef783703dcbe909107f39c162adeac050d73c850055118b5b6327" +checksum = "644816de6547255eff4e491a1dda1c19b7237f00b62a61e6e64859ce4f2906d0" dependencies = [ "cc", "libc", @@ -2464,7 +2556,7 @@ dependencies = [ "openssl-sys", "pkg-config", "vcpkg", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2538,9 +2630,9 @@ dependencies = [ [[package]] name = "dashmap" -version = "6.1.0" +version = "6.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -2592,7 +2684,7 @@ dependencies = [ "object_store 0.12.5", "parking_lot", "parquet 55.2.0", - "rand 0.9.2", + "rand 0.9.4", "regex", "sqlparser 0.55.0", "tempfile", @@ -2609,8 +2701,8 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7541353e77dc7262b71ca27be07d8393661737e3a73b5d1b1c6f7d814c64fa2a" dependencies = [ - "arrow 57.3.0", - "arrow-schema 57.3.0", + "arrow 57.3.1", + "arrow-schema 57.3.1", "async-trait", "bytes", "chrono", @@ -2643,7 +2735,7 @@ dependencies = [ "log", "object_store 0.12.5", "parking_lot", - "rand 0.9.2", + "rand 0.9.4", "regex", "sqlparser 0.59.0", "tempfile", @@ -2683,7 +2775,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9997731f90fa5398ef831ad0e69600f92c861b79c0d38bd1a29b6f0e3a0ce4c8" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "async-trait", "dashmap", "datafusion-common 52.5.0", @@ -2730,7 +2822,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b30a3dd50dec860c9559275c8d97d9de602e611237a6ecfbda0b3b63b872352" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "async-trait", "datafusion-catalog 52.5.0", "datafusion-common 52.5.0", @@ -2758,7 +2850,7 @@ dependencies = [ "base64", "half", "hashbrown 0.14.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "libc", "log", "object_store 0.12.5", @@ -2777,12 +2869,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d551054acec0398ca604512310b77ce05c46f66e54b54d48200a686e385cca4e" dependencies = [ "ahash", - "arrow 57.3.0", - "arrow-ipc 57.3.0", + "arrow 57.3.1", + "arrow-ipc 57.3.1", "chrono", "half", "hashbrown 0.16.1", - "indexmap 2.13.1", + "indexmap 2.14.0", "libc", "log", "object_store 0.12.5", @@ -2839,7 +2931,7 @@ dependencies = [ "log", "object_store 0.12.5", "parquet 55.2.0", - "rand 0.9.2", + "rand 0.9.4", "tempfile", "tokio", "tokio-util", @@ -2854,7 +2946,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27d2668f51b3b30befae2207472569e37807fdedd1d14da58acc6f8ca6257eae" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "async-trait", "bytes", "chrono", @@ -2872,7 +2964,7 @@ dependencies = [ "itertools 0.14.0", "log", "object_store 0.12.5", - "rand 0.9.2", + "rand 0.9.4", "tokio", "url", ] @@ -2883,8 +2975,8 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e02e1b3e3a8ec55f1f62de4252b0407c8567363d056078769a197e24fc834a0f" dependencies = [ - "arrow 57.3.0", - "arrow-ipc 57.3.0", + "arrow 57.3.1", + "arrow-ipc 57.3.1", "async-trait", "bytes", "datafusion-common 52.5.0", @@ -2931,7 +3023,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b559d7bf87d4f900f847baba8509634f838d9718695389e903604cdcccdb01f3" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "async-trait", "bytes", "datafusion-common 52.5.0", @@ -2978,7 +3070,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "250e2d7591ba8b638f063854650faa40bca4e8bd4059b2ece8836f6388d02db4" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "async-trait", "bytes", "datafusion-common 52.5.0", @@ -3020,7 +3112,7 @@ dependencies = [ "object_store 0.12.5", "parking_lot", "parquet 55.2.0", - "rand 0.9.2", + "rand 0.9.4", "tokio", ] @@ -3048,7 +3140,7 @@ dependencies = [ "log", "object_store 0.12.5", "parking_lot", - "rand 0.9.2", + "rand 0.9.4", "tempfile", "url", ] @@ -3059,7 +3151,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc45d23c516ed8d3637751e44e09e21b45b3f58b473c802dddd1f1ad4fe435ff" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "async-trait", "chrono", "dashmap", @@ -3069,7 +3161,7 @@ dependencies = [ "log", "object_store 0.12.5", "parking_lot", - "rand 0.9.2", + "rand 0.9.4", "tempfile", "url", ] @@ -3087,7 +3179,7 @@ dependencies = [ "datafusion-functions-aggregate-common 48.0.1", "datafusion-functions-window-common 48.0.1", "datafusion-physical-expr-common 48.0.1", - "indexmap 2.13.1", + "indexmap 2.14.0", "paste", "recursive", "serde_json", @@ -3100,7 +3192,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63dd30526d2db4fda6440806a41e4676334a94bc0596cc9cc2a0efed20ef2c44" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "async-trait", "chrono", "datafusion-common 52.5.0", @@ -3109,7 +3201,7 @@ dependencies = [ "datafusion-functions-aggregate-common 52.5.0", "datafusion-functions-window-common 52.5.0", "datafusion-physical-expr-common 52.5.0", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "paste", "serde_json", @@ -3123,7 +3215,7 @@ source = "git+https://github.com/FunctionStream/datafusion?branch=48.0.1%2Ffs#c3 dependencies = [ "arrow 55.2.0", "datafusion-common 48.0.1", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "paste", ] @@ -3134,9 +3226,9 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b486b5f6255d40976b88bb83813b0d035a8333e0ec39864824e78068cf42fa6" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "datafusion-common 52.5.0", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "paste", ] @@ -3162,9 +3254,9 @@ dependencies = [ "itertools 0.14.0", "log", "md-5", - "rand 0.9.2", + "rand 0.9.4", "regex", - "sha2", + "sha2 0.10.9", "unicode-segmentation", "uuid", ] @@ -3175,8 +3267,8 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07356c94118d881130dd0ffbff127540407d969c8978736e324edcd6c41cd48f" dependencies = [ - "arrow 57.3.0", - "arrow-buffer 57.3.0", + "arrow 57.3.1", + "arrow-buffer 57.3.1", "base64", "blake2", "blake3", @@ -3193,9 +3285,9 @@ dependencies = [ "log", "md-5", "num-traits", - "rand 0.9.2", + "rand 0.9.4", "regex", - "sha2", + "sha2 0.10.9", "unicode-segmentation", "uuid", ] @@ -3227,7 +3319,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b644f9cf696df9233ce6958b9807666d78563b56f923267474dd6c07795f1f8f" dependencies = [ "ahash", - "arrow 57.3.0", + "arrow 57.3.1", "datafusion-common 52.5.0", "datafusion-doc 52.5.0", "datafusion-execution 52.5.0", @@ -3260,7 +3352,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1de2deaaabe8923ce9ea9f29c47bbb4ee14f67ea2fe1ab5398d9bbebcf86e56" dependencies = [ "ahash", - "arrow 57.3.0", + "arrow 57.3.1", "datafusion-common 52.5.0", "datafusion-expr-common 52.5.0", "datafusion-physical-expr-common 52.5.0", @@ -3292,8 +3384,8 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "552f8d92e4331ee91d23c02d12bb6acf32cbfd5215117e01c0fb63cd4b15af1a" dependencies = [ - "arrow 57.3.0", - "arrow-ord 57.3.0", + "arrow 57.3.1", + "arrow-ord 57.3.1", "datafusion-common 52.5.0", "datafusion-doc 52.5.0", "datafusion-execution 52.5.0", @@ -3330,7 +3422,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "970fd0cdd3df8802b9a9975ff600998289ba9d46682a4f7285cba4820c9ada78" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "async-trait", "datafusion-catalog 52.5.0", "datafusion-common 52.5.0", @@ -3363,7 +3455,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40b4c21a7c8a986a1866c0a87ab756d0bbf7b5f41f306009fa2d9af79c52ed31" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "datafusion-common 52.5.0", "datafusion-doc 52.5.0", "datafusion-expr 52.5.0", @@ -3425,7 +3517,7 @@ dependencies = [ "datafusion-common 48.0.1", "datafusion-expr 48.0.1", "datafusion-physical-expr 48.0.1", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "log", "recursive", @@ -3439,13 +3531,13 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff9aa82b240252a88dee118372f9b9757c545ab9e53c0736bebab2e7da0ef1f2" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "chrono", "datafusion-common 52.5.0", "datafusion-expr 52.5.0", "datafusion-expr-common 52.5.0", "datafusion-physical-expr 52.5.0", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "log", "regex", @@ -3466,7 +3558,7 @@ dependencies = [ "datafusion-physical-expr-common 48.0.1", "half", "hashbrown 0.14.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "log", "paste", @@ -3480,7 +3572,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d48022b8af9988c1d852644f9e8b5584c490659769a550c5e8d39457a1da0a5" dependencies = [ "ahash", - "arrow 57.3.0", + "arrow 57.3.1", "datafusion-common 52.5.0", "datafusion-expr 52.5.0", "datafusion-expr-common 52.5.0", @@ -3488,7 +3580,7 @@ dependencies = [ "datafusion-physical-expr-common 52.5.0", "half", "hashbrown 0.16.1", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", "paste", @@ -3502,7 +3594,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae7a8abc0b4fe624000972a9b145b30b7f1b680bffaa950ea53f78d9b21c27c3" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "datafusion-common 52.5.0", "datafusion-expr 52.5.0", "datafusion-functions 52.5.0", @@ -3531,12 +3623,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147253ca3e6b9d59c162de64c02800973018660e13340dd1886dd038d17ac429" dependencies = [ "ahash", - "arrow 57.3.0", + "arrow 57.3.1", "chrono", "datafusion-common 52.5.0", "datafusion-expr-common 52.5.0", "hashbrown 0.16.1", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", ] @@ -3565,7 +3657,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "689156bb2282107b6239db8d7ef44b4dab10a9b33d3491a0c74acac5e4fedd72" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "datafusion-common 52.5.0", "datafusion-execution 52.5.0", "datafusion-expr 52.5.0", @@ -3598,7 +3690,7 @@ dependencies = [ "futures", "half", "hashbrown 0.14.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "log", "parking_lot", @@ -3613,9 +3705,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68253dc0ee5330aa558b2549c9b0da5af9fc17d753ae73022939014ad616fc28" dependencies = [ "ahash", - "arrow 57.3.0", - "arrow-ord 57.3.0", - "arrow-schema 57.3.0", + "arrow 57.3.1", + "arrow-ord 57.3.1", + "arrow-schema 57.3.1", "async-trait", "datafusion-common 52.5.0", "datafusion-common-runtime 52.5.0", @@ -3629,7 +3721,7 @@ dependencies = [ "futures", "half", "hashbrown 0.16.1", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "log", "parking_lot", @@ -3668,7 +3760,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fcad240a54d0b1d3e8f668398900260a53122d522b2102ab57218590decacd6" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "datafusion-common 52.5.0", "datafusion-datasource 52.5.0", "datafusion-expr-common 52.5.0", @@ -3725,7 +3817,7 @@ dependencies = [ "bigdecimal", "datafusion-common 48.0.1", "datafusion-expr 48.0.1", - "indexmap 2.13.1", + "indexmap 2.14.0", "log", "recursive", "regex", @@ -3738,12 +3830,12 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be53e9eb55db0fbb8980bb6d87f2435b0524acf4c718ed54a57cabbb299b2ab3" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "bigdecimal", "chrono", "datafusion-common 52.5.0", "datafusion-expr 52.5.0", - "indexmap 2.13.1", + "indexmap 2.14.0", "log", "regex", "sqlparser 0.59.0", @@ -3786,7 +3878,35 @@ checksum = "33449d2bcc676a8a0088ed644c3f7c5bf094ae4b115c1730735ba30e8a1866bb" dependencies = [ "buoyant_kernel", "ctor", + "deltalake-aws", + "deltalake-core", +] + +[[package]] +name = "deltalake-aws" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ef837d5d58bb0cd62dc9ee3bc201805751090f4bab2ca0ff09669661814fd4" +dependencies = [ + "async-trait", + "aws-config", + "aws-credential-types", + "aws-sdk-dynamodb", + "aws-sdk-sts", + "aws-smithy-runtime-api", + "backon", + "bytes", + "chrono", "deltalake-core", + "futures", + "object_store 0.13.2", + "regex", + "thiserror 2.0.18", + "tokio", + "tracing", + "typed-builder", + "url", + "uuid", ] [[package]] @@ -3816,7 +3936,8 @@ dependencies = [ "dirs", "either", "futures", - "indexmap 2.13.1", + "humantime", + "indexmap 2.14.0", "itertools 0.14.0", "num_cpus", "object_store 0.13.2", @@ -3868,12 +3989,24 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.0", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "directories-next" version = "2.0.0" @@ -4054,9 +4187,9 @@ checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" [[package]] name = "ethnum" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca81e6b4777c89fd810c25a4be2b1bd93ea034fbe58e6a75216a34c6b82c539b" +checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f" [[package]] name = "event-listener" @@ -4099,9 +4232,9 @@ checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" [[package]] name = "fastrand" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a043dc74da1e37d6afe657061213aa6f425f855399a11d3463c6ecccc4dfda1f" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "fd-lock" @@ -4148,7 +4281,7 @@ version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "rustc_version", ] @@ -4219,12 +4352,12 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2195cc7f87e84bd695586137de99605e7e9579b26ec5e01b82960ddb4d0922f2" +checksum = "2b3a6f3550e61b999febd7168d462db953948eff4fc3448276b3d10d10324dbb" dependencies = [ - "arrow-array 57.3.0", - "rand 0.9.2", + "arrow-array 57.3.1", + "rand 0.9.4", ] [[package]] @@ -4245,9 +4378,9 @@ dependencies = [ "apache-avro", "arrow 55.2.0", "arrow-array 55.2.0", - "arrow-array 57.3.0", + "arrow-array 57.3.1", "arrow-ipc 55.2.0", - "arrow-ipc 57.3.0", + "arrow-ipc 57.3.1", "arrow-json 55.2.0", "arrow-schema 55.2.0", "async-trait", @@ -4281,7 +4414,7 @@ dependencies = [ "proctitle", "prost 0.13.5", "protocol", - "rand 0.8.5", + "rand 0.8.6", "rdkafka", "rocksdb", "serde", @@ -4413,7 +4546,7 @@ dependencies = [ "petgraph 0.7.1", "prost 0.13.5", "protocol", - "rand 0.8.5", + "rand 0.8.6", "serde", "serde_json", "serde_json_path", @@ -4485,9 +4618,9 @@ dependencies = [ "apache-avro", "arrow 55.2.0", "arrow-array 55.2.0", - "arrow-array 57.3.0", + "arrow-array 57.3.1", "arrow-ipc 55.2.0", - "arrow-ipc 57.3.0", + "arrow-ipc 57.3.1", "arrow-json 55.2.0", "arrow-schema 55.2.0", "async-trait", @@ -4605,9 +4738,9 @@ checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-timer" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" [[package]] name = "futures-util" @@ -4632,7 +4765,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25234f20a3ec0a962a61770cfe39ecf03cb529a6e474ad8cff025ed497eda557" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "debugid", "rustc-hash 2.1.2", "serde", @@ -4713,7 +4846,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" dependencies = [ "fallible-iterator", - "indexmap 2.13.1", + "indexmap 2.14.0", "stable_deref_trait", ] @@ -4752,7 +4885,7 @@ dependencies = [ "parking_lot", "portable-atomic", "quanta", - "rand 0.9.2", + "rand 0.9.4", "smallvec", "spinning_top", "web-time", @@ -4760,9 +4893,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" dependencies = [ "atomic-waker", "bytes", @@ -4770,7 +4903,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.4.0", - "indexmap 2.13.1", + "indexmap 2.14.0", "slab", "tokio", "tokio-util", @@ -4858,7 +4991,16 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", ] [[package]] @@ -4949,6 +5091,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.9.0" @@ -4973,16 +5124,15 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.7" +version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ "http 1.4.0", "hyper", "hyper-util", "rustls", "rustls-native-certs", - "rustls-pki-types", "tokio", "tokio-rustls", "tower-service", @@ -5167,9 +5317,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", @@ -5201,12 +5351,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45a8a2b9cb3e0b0c1803dbb0758ffac5de2f425b23c28f518faabd9d805342ff" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "serde", "serde_core", ] @@ -5248,16 +5398,6 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" -[[package]] -name = "iri-string" -version = "0.7.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "is-terminal" version = "0.4.17" @@ -5321,9 +5461,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" dependencies = [ "jiff-static", "jiff-tzdb-platform", @@ -5336,9 +5476,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" dependencies = [ "proc-macro2", "quote", @@ -5421,9 +5561,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.94" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" dependencies = [ "cfg-if", "futures-util", @@ -5445,7 +5585,7 @@ dependencies = [ "nom 8.0.0", "num-traits", "ordered-float 5.3.0", - "rand 0.9.2", + "rand 0.9.4", "serde", "serde_json", "zmij", @@ -5453,19 +5593,19 @@ dependencies = [ [[package]] name = "lance" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efe6c3ddd79cdfd2b7e1c23cafae52806906bc40fbd97de9e8cf2f8c7a75fc04" -dependencies = [ - "arrow 57.3.0", - "arrow-arith 57.3.0", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-ipc 57.3.0", - "arrow-ord 57.3.0", - "arrow-row 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63e285ceee2b4ca8eb3a8742266cc1ac8161599767a8ecb4d8c2f9fd43d8b29" +dependencies = [ + "arrow 57.3.1", + "arrow-arith 57.3.1", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-ipc 57.3.1", + "arrow-ord 57.3.1", + "arrow-row 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", "async-recursion", "async-trait", "async_cell", @@ -5503,7 +5643,7 @@ dependencies = [ "pin-project", "prost 0.14.3", "prost-types 0.14.3", - "rand 0.9.2", + "rand 0.9.4", "roaring", "semver", "serde", @@ -5520,31 +5660,31 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d9f5d95bdda2a2b790f1fb8028b5b6dcf661abeb3133a8bca0f3d24b054af87" +checksum = "5c55e62fc04422ef4cd4af6f863ada32641ae23124f9b2e9c567a40d617e8c97" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-cast 57.3.0", - "arrow-data 57.3.0", - "arrow-ord 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-cast 57.3.1", + "arrow-data 57.3.1", + "arrow-ord 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", "bytes", "futures", "getrandom 0.2.17", "half", "jsonb", "num-traits", - "rand 0.9.2", + "rand 0.9.4", ] [[package]] name = "lance-bitpacking" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f827d6ab9f8f337a9509d5ad66a12f3314db8713868260521c344ef6135eb4e4" +checksum = "a48d232a2908645af0040f96c60a6387fea2df75e762d7033e93e17bb420c6a1" dependencies = [ "arrayref", "paste", @@ -5553,13 +5693,13 @@ dependencies = [ [[package]] name = "lance-core" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f1e25df6a79bf72ee6bcde0851f19b1cd36c5848c1b7db83340882d3c9fdecb" +checksum = "ce071baaff88fcdcf67f1dd0af54e17656f52ae75aaeb75f25f9cf4da29241f2" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-schema 57.3.0", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-schema 57.3.1", "async-trait", "byteorder", "bytes", @@ -5578,7 +5718,7 @@ dependencies = [ "object_store 0.12.5", "pin-project", "prost 0.14.3", - "rand 0.9.2", + "rand 0.9.4", "roaring", "serde_json", "snafu", @@ -5592,16 +5732,16 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93146de8ae720cb90edef81c2f2d0a1b065fc2f23ecff2419546f389b0fa70a4" +checksum = "11ebc97ee94fa8e1af6fd0520066c7e7e0eab38a100e750ba9aabad644c5aa57" dependencies = [ - "arrow 57.3.0", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-ord 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow 57.3.1", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-ord 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", "async-trait", "chrono", "datafusion 52.5.0", @@ -5624,19 +5764,19 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccec8ce4d8e0a87a99c431dab2364398029f2ffb649c1a693c60c79e05ed30dd" +checksum = "9b90dbb2829875b3a3d00f88fd3a3e39a9e4c7d34c266f67da6550fcda54c76e" dependencies = [ - "arrow 57.3.0", - "arrow-array 57.3.0", - "arrow-cast 57.3.0", - "arrow-schema 57.3.0", + "arrow 57.3.1", + "arrow-array 57.3.1", + "arrow-cast 57.3.1", + "arrow-schema 57.3.1", "chrono", "futures", "half", "hex", - "rand 0.9.2", + "rand 0.9.4", "rand_distr 0.5.1", "rand_xoshiro 0.7.0", "random_word", @@ -5644,17 +5784,17 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c1aec0bbbac6bce829bc10f1ba066258126100596c375fb71908ecf11c2c2a5" +checksum = "65ec429cc2e18ad1b7e43cc7ec57a2f2e49229cfbd934da45e619751a886b8cd" dependencies = [ - "arrow-arith 57.3.0", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-cast 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-arith 57.3.1", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-cast 57.3.1", + "arrow-data 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", "bytemuck", "byteorder", "bytes", @@ -5672,7 +5812,7 @@ dependencies = [ "prost 0.14.3", "prost-build 0.14.3", "prost-types 0.14.3", - "rand 0.9.2", + "rand 0.9.4", "snafu", "strum 0.26.3", "tokio", @@ -5683,16 +5823,16 @@ dependencies = [ [[package]] name = "lance-file" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14a8c548804f5b17486dc2d3282356ed1957095a852780283bc401fdd69e9075" +checksum = "418afe3f82487615fa09222b95a4b5853103f3f0425996d24a537ca750381f83" dependencies = [ - "arrow-arith 57.3.0", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-arith 57.3.1", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-data 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", "async-recursion", "async-trait", "byteorder", @@ -5717,16 +5857,16 @@ dependencies = [ [[package]] name = "lance-index" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da212f0090ea59f79ac3686660f596520c167fe1cb5f408900cf71d215f0e03" +checksum = "936b3deeb6ee075646d18f27b01cf2d2e846c3f5f6c5fa45b30aa41dd5b4c4e2" dependencies = [ - "arrow 57.3.0", - "arrow-arith 57.3.0", - "arrow-array 57.3.0", - "arrow-ord 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow 57.3.1", + "arrow-arith 57.3.1", + "arrow-array 57.3.1", + "arrow-ord 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", "async-channel", "async-recursion", "async-trait", @@ -5764,7 +5904,7 @@ dependencies = [ "prost 0.14.3", "prost-build 0.14.3", "prost-types 0.14.3", - "rand 0.9.2", + "rand 0.9.4", "rand_distr 0.5.1", "rangemap", "rayon", @@ -5783,18 +5923,18 @@ dependencies = [ [[package]] name = "lance-io" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d958eb4b56f03bbe0f5f85eb2b4e9657882812297b6f711f201ffc995f259f" -dependencies = [ - "arrow 57.3.0", - "arrow-arith 57.3.0", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-cast 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4103e4cebe146af15bfb198c8142d6ea37d5b25fa04158bf2d9be4597bf174d3" +dependencies = [ + "arrow 57.3.1", + "arrow-arith 57.3.1", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-cast 57.3.1", + "arrow-data 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", "async-recursion", "async-trait", "aws-config", @@ -5815,7 +5955,7 @@ dependencies = [ "path_abs", "pin-project", "prost 0.14.3", - "rand 0.9.2", + "rand 0.9.4", "serde", "snafu", "tempfile", @@ -5826,29 +5966,29 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0285b70da35def7ed95e150fae1d5308089554e1290470403ed3c50cb235bc5e" +checksum = "c00c7ad71eca93635404519e77add6689947c9342134bb2133578f81249bf809" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-schema 57.3.0", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-schema 57.3.1", "cc", "deepsize", "half", "lance-arrow", "lance-core", "num-traits", - "rand 0.9.2", + "rand 0.9.4", ] [[package]] name = "lance-namespace" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f78e2a828b654e062a495462c6e3eb4fcf0e7e907d761b8f217fc09ccd3ceac" +checksum = "e0c59a574e72a4b72da8096bcaaa1b1e5b44f6a83da164cc714c286fab30c369" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "async-trait", "bytes", "lance-core", @@ -5872,15 +6012,15 @@ dependencies = [ [[package]] name = "lance-table" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df9c4adca3eb2074b3850432a9fb34248a3d90c3d6427d158b13ff9355664ee" +checksum = "943b9c503f23ebab9e0dbee356f528bc4cbcafded87a6848451f205b0bb473d7" dependencies = [ - "arrow 57.3.0", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-ipc 57.3.0", - "arrow-schema 57.3.0", + "arrow 57.3.1", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-ipc 57.3.1", + "arrow-schema 57.3.1", "async-trait", "byteorder", "bytes", @@ -5896,7 +6036,7 @@ dependencies = [ "prost 0.14.3", "prost-build 0.14.3", "prost-types 0.14.3", - "rand 0.9.2", + "rand 0.9.4", "rangemap", "roaring", "semver", @@ -5923,9 +6063,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "leb128" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" +checksum = "6cc46bac87ef8093eed6f272babb833b6443374399985ac8ed28471ee0918545" [[package]] name = "leb128fmt" @@ -6062,9 +6202,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.184" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libloading" @@ -6084,9 +6224,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08" +checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" dependencies = [ "libc", ] @@ -6222,9 +6362,9 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c23545df7ecf1b16c303910a69b079e8e251d60f7dd2cc9b4177f2afaf1746" +checksum = "90071f8077f8e40adfc4b7fe9cd495ce316263f19e75c2211eeff3fdf475a3d9" dependencies = [ "twox-hash", ] @@ -6299,7 +6439,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", ] [[package]] @@ -6446,7 +6586,7 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cfg-if", "cfg_aliases 0.1.1", "libc", @@ -6528,9 +6668,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" @@ -6613,7 +6753,7 @@ checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" dependencies = [ "crc32fast", "hashbrown 0.15.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "memchr", ] @@ -6638,7 +6778,7 @@ dependencies = [ "parking_lot", "percent-encoding", "quick-xml 0.38.4", - "rand 0.9.2", + "rand 0.9.4", "reqwest 0.12.28", "ring", "serde", @@ -6764,9 +6904,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.112" +version = "0.9.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb" +checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4" dependencies = [ "cc", "libc", @@ -6983,7 +7123,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset 0.4.2", - "indexmap 2.13.1", + "indexmap 2.14.0", ] [[package]] @@ -6993,7 +7133,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset 0.5.7", - "indexmap 2.13.1", + "indexmap 2.14.0", ] [[package]] @@ -7004,7 +7144,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset 0.5.7", "hashbrown 0.15.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "serde", ] @@ -7028,18 +7168,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.11" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.11" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", @@ -7060,9 +7200,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "portable-atomic" @@ -7293,9 +7433,9 @@ dependencies = [ [[package]] name = "psm" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3852766467df634d74f0b2d7819bf8dc483a0eb2e3b0f50f756f9cfe8b0d18d8" +checksum = "645dbe486e346d9b5de3ef16ede18c26e6c70ad97418f4874b8b1889d6e761ea" dependencies = [ "ar_archive_writer", "cc", @@ -7405,7 +7545,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand 0.9.4", "ring", "rustc-hash 2.1.2", "rustls", @@ -7470,9 +7610,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -7481,9 +7621,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -7551,7 +7691,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" dependencies = [ "num-traits", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -7561,7 +7701,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.2", + "rand 0.9.4", ] [[package]] @@ -7591,7 +7731,7 @@ dependencies = [ "ahash", "brotli", "paste", - "rand 0.9.2", + "rand 0.9.4", "unicase", ] @@ -7607,7 +7747,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", ] [[package]] @@ -7618,9 +7758,9 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" [[package]] name = "rayon" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" dependencies = [ "either", "rayon-core", @@ -7696,7 +7836,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", ] [[package]] @@ -7784,19 +7924,19 @@ dependencies = [ "form_urlencoded", "getrandom 0.2.17", "hex", - "hmac", + "hmac 0.12.1", "home", "http 1.4.0", "log", "percent-encoding", "quick-xml 0.37.5", - "rand 0.8.5", + "rand 0.8.6", "reqwest 0.12.28", "rust-ini", "serde", "serde_json", "sha1", - "sha2", + "sha2 0.10.9", "tokio", ] @@ -7900,9 +8040,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.11.3" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ba9ce64a8f45d7fc86358410bb1a82e8c987504c0d4900e9141d69a9f26c885" +checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9" dependencies = [ "bytemuck", "byteorder", @@ -7971,7 +8111,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys 0.4.15", @@ -7984,7 +8124,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys 0.12.1", @@ -8003,9 +8143,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.37" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "aws-lc-rs", "once_cell", @@ -8030,9 +8170,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ "web-time", "zeroize", @@ -8067,9 +8207,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.11" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20a6af516fea4b20eccceaf166e8aa666ac996208e8a644ce3ef5aa783bc7cd4" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", "ring", @@ -8089,7 +8229,7 @@ version = "14.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7803e8936da37efd9b6d4478277f4b2b9bb5cdb37a113e8d63222e58da647e63" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cfg-if", "clipboard-win", "fd-lock", @@ -8159,7 +8299,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -8333,7 +8473,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.13.1", + "indexmap 2.14.0", "itoa", "ryu", "serde", @@ -8348,7 +8488,7 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", ] [[package]] @@ -8359,7 +8499,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -8443,9 +8594,9 @@ checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "siphasher" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" [[package]] name = "sized-chunks" @@ -8596,15 +8747,15 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stacker" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013" +checksum = "640c8cdd92b6b12f5bcb1803ca3bbf5ab96e5e6b6b96b9ab77dabe9e880b3190" dependencies = [ "cc", "cfg-if", "libc", "psm", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -8680,6 +8831,12 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "1.0.109" @@ -8728,7 +8885,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -8749,7 +8906,7 @@ version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc4592f674ce18521c2a81483873a49596655b179f71c5e05d10c1fe66c78745" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cap-fs-ext", "cap-std", "fd-lock", @@ -9081,9 +9238,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.51.0" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bd1c4c0fc4a7ab90fc15ef6daaa3ec3b893f004f915f2392557ed23237820cd" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -9147,7 +9304,7 @@ version = "0.9.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" dependencies = [ - "indexmap 2.13.1", + "indexmap 2.14.0", "serde_core", "serde_spanned", "toml_datetime 0.7.5+spec-1.1.0", @@ -9176,14 +9333,14 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.10+spec-1.1.0" +version = "0.25.11+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82418ca169e235e6c399a84e395ab6debeb3bc90edc959bf0f48647c6a32d1b" +checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" dependencies = [ - "indexmap 2.13.1", + "indexmap 2.14.0", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", - "winnow 1.0.1", + "winnow 1.0.3", ] [[package]] @@ -9192,7 +9349,7 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "winnow 1.0.1", + "winnow 1.0.3", ] [[package]] @@ -9256,7 +9413,7 @@ dependencies = [ "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand 0.8.5", + "rand 0.8.6", "slab", "tokio", "tokio-util", @@ -9282,25 +9439,25 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "async-compression", - "bitflags 2.11.0", + "bitflags 2.11.1", "bytes", "futures-core", "futures-util", "http 1.4.0", "http-body 1.0.1", "http-body-util", - "iri-string", "pin-project-lite", "tokio", "tokio-util", "tower 0.5.3", "tower-layer", "tower-service", + "url", ] [[package]] @@ -9329,11 +9486,12 @@ dependencies = [ [[package]] name = "tracing-appender" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" dependencies = [ "crossbeam-channel", + "symlink", "thiserror 2.0.18", "time", "tracing-subscriber", @@ -9401,14 +9559,34 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" dependencies = [ - "rand 0.9.2", + "rand 0.9.4", +] + +[[package]] +name = "typed-builder" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31aa81521b70f94402501d848ccc0ecaa8f93c8eb6999eb9747e72287757ffda" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076a02dc54dd46795c2e9c8282ed40bcfb1e22747e955de9389a1de28190fb26" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", ] [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" [[package]] name = "unicase" @@ -9503,9 +9681,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -9601,11 +9779,11 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.2+wasi-0.2.9" +version = "1.0.3+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.57.1", ] [[package]] @@ -9614,14 +9792,14 @@ version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.51.0", ] [[package]] name = "wasm-bindgen" -version = "0.2.117" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" dependencies = [ "cfg-if", "once_cell", @@ -9632,9 +9810,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.67" +version = "0.4.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03623de6905b7206edd0a75f69f747f134b7f0a2323392d664448bf2d3c5d87e" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" dependencies = [ "js-sys", "wasm-bindgen", @@ -9642,9 +9820,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.117" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -9652,9 +9830,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.117" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" dependencies = [ "bumpalo", "proc-macro2", @@ -9665,9 +9843,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.117" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" dependencies = [ "unicode-ident", ] @@ -9681,7 +9859,7 @@ dependencies = [ "anyhow", "heck", "im-rc", - "indexmap 2.13.1", + "indexmap 2.14.0", "log", "petgraph 0.6.5", "serde", @@ -9715,12 +9893,12 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.246.2" +version = "0.249.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61fb705ce81adde29d2a8e99d87995e39a6e927358c91398f374474746070ef7" +checksum = "69830ccbbf41c55eb585991659fb70867ef628193af3a495f09a6956f7615e59" dependencies = [ "leb128fmt", - "wasmparser 0.246.2", + "wasmparser 0.249.0", ] [[package]] @@ -9730,7 +9908,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap 2.13.1", + "indexmap 2.14.0", "wasm-encoder 0.244.0", "wasmparser 0.244.0", ] @@ -9754,9 +9932,9 @@ version = "0.243.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6d8db401b0528ec316dfbe579e6ab4152d61739cfe076706d2009127970159d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "hashbrown 0.15.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "semver", "serde", ] @@ -9767,20 +9945,20 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "hashbrown 0.15.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "semver", ] [[package]] name = "wasmparser" -version = "0.246.2" +version = "0.249.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71cde4757396defafd25417cfb36aa3161027d06d865b0c24baaae229aac005d" +checksum = "30538cae9a794215f490b532df01c557e2e2bfac92569482554acd0992a102ea" dependencies = [ - "bitflags 2.11.0", - "indexmap 2.13.1", + "bitflags 2.11.1", + "indexmap 2.14.0", "semver", ] @@ -9804,7 +9982,7 @@ dependencies = [ "addr2line", "anyhow", "async-trait", - "bitflags 2.11.0", + "bitflags 2.11.1", "bumpalo", "cc", "cfg-if", @@ -9813,7 +9991,7 @@ dependencies = [ "fxprof-processed-profile", "gimli", "hashbrown 0.15.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "ittapi", "libc", "log", @@ -9863,7 +10041,7 @@ dependencies = [ "cranelift-bitset", "cranelift-entity", "gimli", - "indexmap 2.13.1", + "indexmap 2.14.0", "log", "object", "postcard", @@ -9892,7 +10070,7 @@ dependencies = [ "rustix 1.1.4", "serde", "serde_derive", - "sha2", + "sha2 0.10.9", "toml", "wasmtime-environ", "windows-sys 0.61.2", @@ -10049,9 +10227,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "587699ca7cae16b4a234ffcc834f37e75675933d533809919b52975f5609e2ef" dependencies = [ "anyhow", - "bitflags 2.11.0", + "bitflags 2.11.1", "heck", - "indexmap 2.13.1", + "indexmap 2.14.0", "wit-parser 0.243.0", ] @@ -10063,7 +10241,7 @@ checksum = "fc2eb9dc95baed3cd86fdfebf9f9f333337eb308bf8bd973e0c7b06d9418c35f" dependencies = [ "anyhow", "async-trait", - "bitflags 2.11.0", + "bitflags 2.11.1", "bytes", "cap-fs-ext", "cap-net-ext", @@ -10110,31 +10288,31 @@ dependencies = [ [[package]] name = "wast" -version = "246.0.2" +version = "249.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe3fe8e3bf88ad96d031b4181ddbd64634b17cb0d06dfc3de589ef43591a9a62" +checksum = "2474a321bf9ae2808e9fa23ac4ec2b27300e70985e30bcb5a38d43b76bfc901a" dependencies = [ "bumpalo", "leb128fmt", "memchr", "unicode-width 0.2.2", - "wasm-encoder 0.246.2", + "wasm-encoder 0.249.0", ] [[package]] name = "wat" -version = "1.246.2" +version = "1.249.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd7fda1199b94fff395c2d19a153f05dbe7807630316fa9673367666fd2ad8c" +checksum = "28af699d0a9c7e4e250b7b8e36167ae5215fbb4b7ae526bb4ce7b234ba0afc90" dependencies = [ - "wast 246.0.2", + "wast 249.0.0", ] [[package]] name = "web-sys" -version = "0.3.94" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd70027e39b12f0849461e08ffc50b9cd7688d942c1c8e3c7b22273236b4dd0a" +checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" dependencies = [ "js-sys", "wasm-bindgen", @@ -10175,7 +10353,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a69a60bcbe1475c5dc9ec89210ade54823d44f742e283cba64f98f89697c4cec" dependencies = [ "anyhow", - "bitflags 2.11.0", + "bitflags 2.11.1", "thiserror 2.0.18", "tracing", "wasmtime", @@ -10502,9 +10680,9 @@ checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" [[package]] name = "winnow" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" dependencies = [ "memchr", ] @@ -10515,7 +10693,7 @@ version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "windows-sys 0.59.0", ] @@ -10528,6 +10706,12 @@ dependencies = [ "wit-bindgen-rust-macro", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "wit-bindgen-core" version = "0.51.0" @@ -10547,7 +10731,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck", - "indexmap 2.13.1", + "indexmap 2.14.0", "prettyplease", "syn 2.0.117", "wasm-metadata", @@ -10577,8 +10761,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags 2.11.0", - "indexmap 2.13.1", + "bitflags 2.11.1", + "indexmap 2.14.0", "log", "serde", "serde_derive", @@ -10597,7 +10781,7 @@ checksum = "df983a8608e513d8997f435bb74207bf0933d0e49ca97aa9d8a6157164b9b7fc" dependencies = [ "anyhow", "id-arena", - "indexmap 2.13.1", + "indexmap 2.14.0", "log", "semver", "serde", @@ -10615,7 +10799,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap 2.13.1", + "indexmap 2.14.0", "log", "semver", "serde", @@ -10724,9 +10908,9 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] diff --git a/src/streaming_runtime_operators/Cargo.toml b/src/streaming_runtime_operators/Cargo.toml index d20f491c..53f6fbba 100644 --- a/src/streaming_runtime_operators/Cargo.toml +++ b/src/streaming_runtime_operators/Cargo.toml @@ -28,7 +28,7 @@ datafusion-execution = { git = "https://github.com/FunctionStream/datafusion", b datafusion-expr = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } datafusion-physical-expr = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } datafusion-proto = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } -deltalake = { version = "0.32.3", default-features = false, features = ["rustls"] } +deltalake = { version = "0.32.3", default-features = false, features = ["rustls", "s3"] } function-stream-config = { path = "../config" } function-stream-runtime-common = { path = "../runtime_common" } function-stream-streaming-planner = { path = "../streaming_planner" } diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs index 30914f0f..65d4df66 100644 --- a/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs @@ -17,6 +17,7 @@ pub use delta_commit::strip_streaming_system_columns_arc; use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; +use std::sync::Once; use arrow_array::RecordBatch; use arrow_schema::Schema as ArrowSchema; @@ -45,6 +46,15 @@ use crate::sql::common::{CheckpointBarrier, Watermark}; const DEFAULT_MAX_BUFFER_BYTES: usize = 64 * 1024 * 1024; +/// Registers deltalake protocol handlers (e.g. `s3://`) exactly once per process. +/// Without this, `deltalake::open_table` returns `Cannot infer storage location from: s3://...`. +fn ensure_delta_handlers_registered() { + static REGISTER: Once = Once::new(); + REGISTER.call_once(|| { + deltalake::aws::register_handlers(None); + }); +} + /// Strongly typed error domain for the Delta sink. #[derive(thiserror::Error, Debug)] pub enum DeltaSinkError { @@ -312,10 +322,16 @@ impl Operator for DeltaSinkOperator { } async fn on_start(&mut self, _ctx: &mut TaskContext) -> anyhow::Result<()> { + ensure_delta_handlers_registered(); + if let DeltaDestination::Local(root) = &self.destination { tokio::fs::create_dir_all(root).await?; } + if let DeltaDestination::S3 { client, prefix } = &self.destination { + preflight_s3_bucket(client.as_ref(), prefix, self.s3_bucket.as_deref()).await?; + } + let table_uri = resolve_delta_table_uri(&self.sink_path, self.s3_bucket.as_deref())?; self.table_uri = Some(table_uri.clone()); @@ -408,3 +424,40 @@ impl Operator for DeltaSinkOperator { Ok(vec![]) } } + +/// Verify the S3 bucket is reachable and produce a clear error before +/// the deltalake layer fails with a less obvious message. +async fn preflight_s3_bucket( + client: &dyn ObjectStore, + prefix: &str, + bucket: Option<&str>, +) -> Result<(), DeltaSinkError> { + let probe_path = if prefix.is_empty() { + ObjectStorePath::from("") + } else { + ObjectStorePath::from(prefix.to_string()) + }; + + let mut stream = client.list(Some(&probe_path)); + match futures::StreamExt::next(&mut stream).await { + None => Ok(()), + Some(Ok(_)) => Ok(()), + Some(Err(object_store::Error::NotFound { .. })) => Ok(()), + Some(Err(e)) => { + let msg = e.to_string(); + let lower = msg.to_ascii_lowercase(); + if lower.contains("nosuchbucket") || lower.contains("the specified bucket") { + return Err(DeltaSinkError::Config(format!( + "S3 bucket '{}' does not exist (prefix '{}'): {msg}", + bucket.unwrap_or(""), + prefix + ))); + } + Err(DeltaSinkError::Config(format!( + "failed to access S3 bucket '{}' (prefix '{}'): {msg}", + bucket.unwrap_or(""), + prefix + ))) + } + } +} From ce36fc5f71ff5a46957c6fca80446b8943593ccf Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 20 May 2026 22:35:29 +0800 Subject: [PATCH 10/16] fix(delta): enable unsafe rename by default for S3/MinIO commits deltalake-aws 0.15+ requires either a DynamoDB lock client or AWS_S3_ALLOW_UNSAFE_RENAME=true. Function-Stream sinks are single-writer, so default the flag on; users can opt into ETag conditional puts via the s3.conditional.put option (e.g. MinIO). Co-authored-by: Cursor --- .../src/operators/sink/delta/delta_commit.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs b/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs index cf26f7ad..5bcaf201 100644 --- a/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs +++ b/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs @@ -443,6 +443,22 @@ pub fn build_delta_storage_options(options: &HashMap) -> HashMap if let Some(v) = options.get(opt::S3_SESSION_TOKEN) { storage.insert("AWS_SESSION_TOKEN".to_string(), v.clone()); } + + // deltalake-aws 0.15+ requires either a DynamoDB lock client or this flag for S3 commits. + // Function-Stream Delta sinks are single-writer per table, so unsafe rename is safe. + // Users may opt into ETag-based conditional puts (e.g. MinIO) via `s3.conditional.put=etag`. + let conditional_put = options + .get("s3.conditional.put") + .map(String::as_str) + .unwrap_or(""); + if !conditional_put.is_empty() { + storage.insert("conditional_put".to_string(), conditional_put.to_string()); + } else { + storage + .entry("AWS_S3_ALLOW_UNSAFE_RENAME".to_string()) + .or_insert_with(|| "true".to_string()); + } + storage } From 34c7aaa398d2da14dc319aa140076e5079f495d2 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 20 May 2026 22:42:12 +0800 Subject: [PATCH 11/16] feat(delta): model commit coordination strategy for S3-compatible stores Replace the bare AWS_S3_ALLOW_UNSAFE_RENAME default with an explicit DeltaCommitStrategy abstraction: single-writer (default, leveraging the pipeline scheduler's per-table writer guarantee) or DynamoDB (for multi-process writers). Adds delta.commit.strategy and delta.dynamodb.* sink options for clear cloud/on-prem deployments. Co-authored-by: Cursor --- .../src/common/with_option_keys.rs | 18 +++ .../src/operators/sink/delta/delta_commit.rs | 131 ++++++++++++++++-- .../src/operators/sink/delta/mod.rs | 7 +- 3 files changed, 139 insertions(+), 17 deletions(-) diff --git a/src/streaming_planner/src/common/with_option_keys.rs b/src/streaming_planner/src/common/with_option_keys.rs index 21bfa691..8a8ebd27 100644 --- a/src/streaming_planner/src/common/with_option_keys.rs +++ b/src/streaming_planner/src/common/with_option_keys.rs @@ -82,6 +82,24 @@ pub const S3_ACCESS_KEY_ID: &str = "s3.access_key_id"; pub const S3_SECRET_ACCESS_KEY: &str = "s3.secret_access_key"; pub const S3_SESSION_TOKEN: &str = "s3.session_token"; +// ── Delta Lake commit coordination (S3-compatible backends) ─────────────── +// +// `delta.commit.strategy` selects how concurrent-writer safety is enforced +// when committing to `_delta_log` on S3-compatible object stores: +// +// - `single-writer` (default): rely on Function-Stream's pipeline scheduler +// that guarantees a single writer per Delta table. Commits use direct +// PUT and are atomic per object. Suitable for AWS S3, MinIO, R2, GCS-S3. +// - `dynamodb`: use deltalake-aws DynamoDB lock client for multi-writer +// safety. Requires `delta.dynamodb.table` and AWS credentials/region. +// +pub const DELTA_COMMIT_STRATEGY: &str = "delta.commit.strategy"; +pub const DELTA_DYNAMODB_TABLE: &str = "delta.dynamodb.table"; +pub const DELTA_DYNAMODB_REGION: &str = "delta.dynamodb.region"; +pub const DELTA_DYNAMODB_ENDPOINT: &str = "delta.dynamodb.endpoint"; +pub const DELTA_DYNAMODB_ACCESS_KEY_ID: &str = "delta.dynamodb.access_key_id"; +pub const DELTA_DYNAMODB_SECRET_ACCESS_KEY: &str = "delta.dynamodb.secret_access_key"; + // ── Protobuf ──────────────────────────────────────────────────────────────── pub const PROTOBUF_INTO_UNSTRUCTURED_JSON: &str = "protobuf.into_unstructured_json"; diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs b/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs index 5bcaf201..17ee001f 100644 --- a/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs +++ b/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs @@ -32,6 +32,83 @@ use crate::sql::common::{TIMESTAMP_FIELD, UPDATING_META_FIELD}; use super::DeltaSinkError; +/// Coordination strategy for Delta `_delta_log` commits on S3-compatible stores. +/// +/// `deltalake-aws` 0.15 only ships two safe primitives for S3 commits: +/// - a DynamoDB-backed lock client, or +/// - a single-writer flag (`AWS_S3_ALLOW_UNSAFE_RENAME=true`) that +/// guarantees correctness only when there is exactly one writer per table. +/// +/// Function-Stream's planner+scheduler guarantees a single writer per Delta +/// table, so `SingleWriter` is the safe default for all S3-compatible backends +/// including MinIO, R2 and GCS-S3. For multi-process deployments writing to the +/// same table, switch to `DynamoDb`. +#[derive(Debug, Clone)] +pub enum DeltaCommitStrategy { + /// Single-writer per table (default). Function-Stream's scheduler enforces + /// this invariant, so direct PUTs to `_delta_log/*.json` are race-free. + SingleWriter, + + /// DynamoDB-backed multi-writer concurrency control via deltalake-aws. + DynamoDb(DynamoDbLockConfig), +} + +#[derive(Debug, Clone, Default)] +pub struct DynamoDbLockConfig { + pub table_name: Option, + pub region: Option, + pub endpoint: Option, + pub access_key_id: Option, + pub secret_access_key: Option, +} + +impl DeltaCommitStrategy { + /// Resolve the commit strategy from sink options. + /// + /// `delta.commit.strategy`: + /// - `single-writer` / `single_writer` / `none` / unset → SingleWriter + /// - `dynamodb` / `dynamo` → DynamoDb + pub fn from_options(options: &HashMap) -> Result { + use crate::sql::common::with_option_keys as opt; + + let raw = options + .get(opt::DELTA_COMMIT_STRATEGY) + .map(|v| v.trim().to_ascii_lowercase()) + .unwrap_or_default(); + + match raw.as_str() { + "" | "single-writer" | "single_writer" | "singlewriter" | "none" => { + Ok(Self::SingleWriter) + } + "dynamodb" | "dynamo" => { + let cfg = DynamoDbLockConfig { + table_name: options.get(opt::DELTA_DYNAMODB_TABLE).cloned(), + region: options.get(opt::DELTA_DYNAMODB_REGION).cloned(), + endpoint: options.get(opt::DELTA_DYNAMODB_ENDPOINT).cloned(), + access_key_id: options.get(opt::DELTA_DYNAMODB_ACCESS_KEY_ID).cloned(), + secret_access_key: options.get(opt::DELTA_DYNAMODB_SECRET_ACCESS_KEY).cloned(), + }; + if cfg.table_name.is_none() { + return Err(DeltaSinkError::Config( + "delta.commit.strategy=dynamodb requires delta.dynamodb.table".into(), + )); + } + Ok(Self::DynamoDb(cfg)) + } + other => Err(DeltaSinkError::Config(format!( + "unknown delta.commit.strategy '{other}'; expected single-writer or dynamodb" + ))), + } + } + + pub fn label(&self) -> &'static str { + match self { + Self::SingleWriter => "single-writer", + Self::DynamoDb(_) => "dynamodb", + } + } +} + /// Streaming-internal columns that must not be persisted to external sinks. pub fn is_streaming_system_column(name: &str) -> bool { name == TIMESTAMP_FIELD || name == UPDATING_META_FIELD @@ -421,10 +498,17 @@ fn simple_stats_json(record_count: u64) -> Option { Some(format!(r#"{{"numRecords":{record_count}}}"#)) } -pub fn build_delta_storage_options(options: &HashMap) -> HashMap { +/// Build the storage options handed to deltalake. +/// +/// Returns both the assembled HashMap and the resolved commit strategy so the +/// caller can log/expose it. +pub fn build_delta_storage_options( + options: &HashMap, +) -> Result<(HashMap, DeltaCommitStrategy), DeltaSinkError> { use crate::sql::common::with_option_keys as opt; let mut storage = HashMap::new(); + if let Some(v) = options.get(opt::S3_ACCESS_KEY_ID) { storage.insert("AWS_ACCESS_KEY_ID".to_string(), v.clone()); } @@ -444,22 +528,39 @@ pub fn build_delta_storage_options(options: &HashMap) -> HashMap storage.insert("AWS_SESSION_TOKEN".to_string(), v.clone()); } - // deltalake-aws 0.15+ requires either a DynamoDB lock client or this flag for S3 commits. - // Function-Stream Delta sinks are single-writer per table, so unsafe rename is safe. - // Users may opt into ETag-based conditional puts (e.g. MinIO) via `s3.conditional.put=etag`. - let conditional_put = options - .get("s3.conditional.put") - .map(String::as_str) - .unwrap_or(""); - if !conditional_put.is_empty() { - storage.insert("conditional_put".to_string(), conditional_put.to_string()); - } else { - storage - .entry("AWS_S3_ALLOW_UNSAFE_RENAME".to_string()) - .or_insert_with(|| "true".to_string()); + let strategy = DeltaCommitStrategy::from_options(options)?; + match &strategy { + DeltaCommitStrategy::SingleWriter => { + // Function-Stream guarantees a single writer per Delta table via its + // pipeline scheduler. deltalake-aws still gates S3 commits unless one + // of the two supported flags is set, so we set the single-writer flag + // here; the architectural invariant makes this safe. + storage.insert("AWS_S3_ALLOW_UNSAFE_RENAME".to_string(), "true".to_string()); + } + DeltaCommitStrategy::DynamoDb(cfg) => { + storage.insert( + "AWS_S3_LOCKING_PROVIDER".to_string(), + "dynamodb".to_string(), + ); + if let Some(table) = &cfg.table_name { + storage.insert("DELTA_DYNAMO_TABLE_NAME".to_string(), table.clone()); + } + if let Some(region) = &cfg.region { + storage.insert("AWS_REGION_DYNAMODB".to_string(), region.clone()); + } + if let Some(endpoint) = &cfg.endpoint { + storage.insert("AWS_ENDPOINT_URL_DYNAMODB".to_string(), endpoint.clone()); + } + if let Some(v) = &cfg.access_key_id { + storage.insert("AWS_ACCESS_KEY_ID_DYNAMODB".to_string(), v.clone()); + } + if let Some(v) = &cfg.secret_access_key { + storage.insert("AWS_SECRET_ACCESS_KEY_DYNAMODB".to_string(), v.clone()); + } + } } - storage + Ok((storage, strategy)) } pub fn resolve_delta_table_uri(path: &str, bucket: Option<&str>) -> Result { diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs index 65d4df66..17dfe430 100644 --- a/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs @@ -24,7 +24,7 @@ use arrow_schema::Schema as ArrowSchema; use async_trait::async_trait; use bytes::Bytes; use delta_commit::{ - DeltaTableCommitter, UncommittedDataFile, build_delta_storage_options, + DeltaCommitStrategy, DeltaTableCommitter, UncommittedDataFile, build_delta_storage_options, cast_batches_for_delta_write, resolve_delta_table_uri, }; use object_store::aws::AmazonS3Builder; @@ -104,6 +104,7 @@ pub struct DeltaSinkOperator { committer: Option, table_uri: Option, storage_options: HashMap, + commit_strategy: DeltaCommitStrategy, s3_bucket: Option, sink_path: String, catalog_schema: Option>, @@ -123,7 +124,7 @@ impl DeltaSinkOperator { catalog_schema: Option>, ) -> Result { let s3_bucket = options.get(opt::S3_BUCKET).cloned(); - let storage_options = build_delta_storage_options(&options); + let (storage_options, commit_strategy) = build_delta_storage_options(&options)?; let destination = if let Some(bucket) = &s3_bucket { let region = options @@ -184,6 +185,7 @@ impl DeltaSinkOperator { committer: None, table_uri: None, storage_options, + commit_strategy, s3_bucket, sink_path: path, catalog_schema: catalog_schema.and_then(strip_streaming_system_columns_arc), @@ -354,6 +356,7 @@ impl Operator for DeltaSinkOperator { table = %self.table_name, threshold = self.early_flush_threshold_bytes, is_true_delta = self.committer.is_some(), + commit_strategy = %self.commit_strategy.label(), "delta sink operator started successfully" ); Ok(()) From 54965cd9a506a47060bc3737ef108b8b0158764b Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 20 May 2026 22:49:54 +0800 Subject: [PATCH 12/16] feat(delta): production-grade S3/MinIO networking and bucket preflight - AmazonS3Builder now uses an exponential-backoff RetryConfig (100ms..5s, 5 retries, 30s overall) so transient network blips don't fail commits. - Custom S3 endpoints force path-style addressing on both the object_store client and the deltalake storage options (AWS_VIRTUAL_HOSTED_STYLE_REQUEST =false), matching MinIO/Ceph/R2/private gateway deployments. - Preflight bucket check now surfaces InvalidAccessKeyId, SignatureDoes NotMatch and AccessDenied as clear configuration errors. - on_close logs commit failures with error! before bubbling up to make data-loss risk visible in operator dashboards. Co-authored-by: Cursor --- .../src/operators/sink/delta/delta_commit.rs | 7 +++ .../src/operators/sink/delta/mod.rs | 49 +++++++++++++++++-- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs b/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs index 17ee001f..e73d5058 100644 --- a/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs +++ b/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs @@ -523,6 +523,13 @@ pub fn build_delta_storage_options( if !v.starts_with("https://") { storage.insert("AWS_ALLOW_HTTP".to_string(), "true".to_string()); } + // Custom endpoint implies MinIO / Ceph / R2 / private S3 gateway — these + // backends route via path style; virtual-hosted style requires per-bucket + // DNS that they typically do not provide. + storage.insert( + "AWS_VIRTUAL_HOSTED_STYLE_REQUEST".to_string(), + "false".to_string(), + ); } if let Some(v) = options.get(opt::S3_SESSION_TOKEN) { storage.insert("AWS_SESSION_TOKEN".to_string(), v.clone()); diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs index 17dfe430..522c3331 100644 --- a/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs @@ -18,6 +18,7 @@ use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; use std::sync::Once; +use std::time::Duration; use arrow_array::RecordBatch; use arrow_schema::Schema as ArrowSchema; @@ -29,10 +30,10 @@ use delta_commit::{ }; use object_store::aws::AmazonS3Builder; use object_store::path::Path as ObjectStorePath; -use object_store::{ObjectStore, PutPayload}; +use object_store::{BackoffConfig, ObjectStore, PutPayload, RetryConfig}; use parquet::basic::Compression; use tokio::io::AsyncWriteExt; -use tracing::{debug, info, instrument, warn}; +use tracing::{debug, error, info, instrument, warn}; use url::Url; use crate::core::StreamOutput; @@ -131,15 +132,34 @@ impl DeltaSinkOperator { .get(opt::S3_REGION) .map(|s| s.as_str()) .unwrap_or("us-east-1"); + + // Production-grade retry with exponential backoff for transient network + // blips on S3 / MinIO / R2. Caps retries to keep checkpoint progress + // bounded; pipeline-level retries handle longer outages. + let retry_config = RetryConfig { + backoff: BackoffConfig { + init_backoff: Duration::from_millis(100), + max_backoff: Duration::from_secs(5), + base: 2.0, + }, + max_retries: 5, + retry_timeout: Duration::from_secs(30), + }; + let mut builder = AmazonS3Builder::new() .with_bucket_name(bucket.clone()) - .with_region(region); + .with_region(region) + .with_retry(retry_config); if let Some(endpoint) = options.get(opt::S3_ENDPOINT) { builder = builder.with_endpoint(endpoint); if endpoint.to_ascii_lowercase().starts_with("http://") { builder = builder.with_allow_http(true); } + // Custom endpoint typically means MinIO / Ceph / R2 / private S3 + // gateways that route via path style; virtual-hosted style would + // require per-bucket DNS that these backends usually lack. + builder = builder.with_virtual_hosted_style_request(false); } if let Some(v) = options.get(opt::S3_ACCESS_KEY_ID) { builder = builder.with_access_key_id(v); @@ -421,8 +441,17 @@ impl Operator for DeltaSinkOperator { table = %self.table_name, "operator closing, forcing final flush and commit" ); - self.flush_and_commit_checkpoint(u64::MAX, ctx.subtask_index as usize) - .await?; + if let Err(e) = self + .flush_and_commit_checkpoint(u64::MAX, ctx.subtask_index as usize) + .await + { + error!( + table = %self.table_name, + error = %e, + "fatal: final flush/commit on close failed; data may be lost" + ); + return Err(e.into()); + } } Ok(vec![]) } @@ -456,6 +485,16 @@ async fn preflight_s3_bucket( prefix ))); } + if lower.contains("invalidaccesskeyid") + || lower.contains("signaturedoesnotmatch") + || lower.contains("access denied") + || lower.contains("accessdenied") + { + return Err(DeltaSinkError::Config(format!( + "S3 credential check failed for bucket '{}': {msg}", + bucket.unwrap_or("") + ))); + } Err(DeltaSinkError::Config(format!( "failed to access S3 bucket '{}' (prefix '{}'): {msg}", bucket.unwrap_or(""), From c48272be33aa0de2219f44bfee980ba4dc3b07ff Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 20 May 2026 23:12:54 +0800 Subject: [PATCH 13/16] refactor(delta): Arroyo-style physical sink separated from commit layer Introduce physical_sink module with FinishedFile contract, SingleThreadPhysicalSink (S3/MinIO encode+PUT without _delta_log), and LocalPhysicalSink. DeltaSinkOperator buffers batches, delegates early flush to the physical layer, and commits via DeltaTableCommitter only at checkpoint boundaries. Co-authored-by: Cursor --- .../src/operators/sink/delta/mod.rs | 254 +++--------- .../src/operators/sink/delta/physical_sink.rs | 376 ++++++++++++++++++ 2 files changed, 431 insertions(+), 199 deletions(-) create mode 100644 src/streaming_runtime_operators/src/operators/sink/delta/physical_sink.rs diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs index 522c3331..5c79f713 100644 --- a/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs @@ -10,45 +10,43 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! Delta Lake sink with Arroyo-style separation: +//! - [`physical_sink`]: encode + PUT part files (no `_delta_log` I/O) +//! - [`delta_commit`]: checkpoint-time transaction commit + mod delta_commit; +mod physical_sink; pub use delta_commit::strip_streaming_system_columns_arc; +pub use physical_sink::FinishedFile; use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; use std::sync::Once; -use std::time::Duration; use arrow_array::RecordBatch; use arrow_schema::Schema as ArrowSchema; use async_trait::async_trait; -use bytes::Bytes; use delta_commit::{ - DeltaCommitStrategy, DeltaTableCommitter, UncommittedDataFile, build_delta_storage_options, - cast_batches_for_delta_write, resolve_delta_table_uri, + DeltaCommitStrategy, DeltaTableCommitter, build_delta_storage_options, resolve_delta_table_uri, }; -use object_store::aws::AmazonS3Builder; -use object_store::path::Path as ObjectStorePath; -use object_store::{BackoffConfig, ObjectStore, PutPayload, RetryConfig}; use parquet::basic::Compression; -use tokio::io::AsyncWriteExt; use tracing::{debug, error, info, instrument, warn}; use url::Url; use crate::core::StreamOutput; use crate::core::api::context::TaskContext; use crate::core::api::operator::{Collector, Operator}; -use crate::format::encoder::FormatEncoder; use crate::memory::{MemoryBlock, try_global_memory_pool}; use crate::sql::common::constants::factory_operator_name; use crate::sql::common::with_option_keys as opt; use crate::sql::common::{CheckpointBarrier, Watermark}; +use physical_sink::{LocalPhysicalSink, SingleThreadPhysicalSink, preflight_s3_bucket}; + const DEFAULT_MAX_BUFFER_BYTES: usize = 64 * 1024 * 1024; -/// Registers deltalake protocol handlers (e.g. `s3://`) exactly once per process. -/// Without this, `deltalake::open_table` returns `Cannot infer storage location from: s3://...`. fn ensure_delta_handlers_registered() { static REGISTER: Once = Once::new(); REGISTER.call_once(|| { @@ -56,7 +54,6 @@ fn ensure_delta_handlers_registered() { }); } -/// Strongly typed error domain for the Delta sink. #[derive(thiserror::Error, Debug)] pub enum DeltaSinkError { #[error("local filesystem I/O error: {0}")] @@ -75,12 +72,10 @@ pub enum DeltaSinkError { Config(String), } -enum DeltaDestination { - Local(PathBuf), - S3 { - prefix: String, - client: Arc, - }, +/// Physical storage backend: part-file writer only. +enum PhysicalBackend { + Local(LocalPhysicalSink), + S3(SingleThreadPhysicalSink), } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -92,15 +87,16 @@ pub enum DeltaFormat { Orc, } +/// Coordinator operator: buffers batches, delegates physical writes to [`PhysicalBackend`], +/// and commits `_delta_log` at checkpoint via [`DeltaTableCommitter`]. pub struct DeltaSinkOperator { table_name: String, - destination: DeltaDestination, + physical: PhysicalBackend, parquet_compression: Compression, pending: Vec, pending_bytes: usize, sink_memory_block: Option>, early_flush_threshold_bytes: usize, - file_counter: u64, format: DeltaFormat, committer: Option, table_uri: Option, @@ -109,12 +105,12 @@ pub struct DeltaSinkOperator { s3_bucket: Option, sink_path: String, catalog_schema: Option>, - /// Normalized schema for Parquet files (microsecond timestamps, etc.). parquet_write_schema: Option>, + /// Retained for local path creation in [`Operator::on_start`]. + local_root: Option, } impl DeltaSinkOperator { - /// Synchronous, side-effect-free constructor. Async setup runs in [`Operator::on_start`]. pub fn try_new( table_name: String, path: String, @@ -127,58 +123,15 @@ impl DeltaSinkOperator { let s3_bucket = options.get(opt::S3_BUCKET).cloned(); let (storage_options, commit_strategy) = build_delta_storage_options(&options)?; - let destination = if let Some(bucket) = &s3_bucket { - let region = options - .get(opt::S3_REGION) - .map(|s| s.as_str()) - .unwrap_or("us-east-1"); - - // Production-grade retry with exponential backoff for transient network - // blips on S3 / MinIO / R2. Caps retries to keep checkpoint progress - // bounded; pipeline-level retries handle longer outages. - let retry_config = RetryConfig { - backoff: BackoffConfig { - init_backoff: Duration::from_millis(100), - max_backoff: Duration::from_secs(5), - base: 2.0, - }, - max_retries: 5, - retry_timeout: Duration::from_secs(30), - }; - - let mut builder = AmazonS3Builder::new() - .with_bucket_name(bucket.clone()) - .with_region(region) - .with_retry(retry_config); - - if let Some(endpoint) = options.get(opt::S3_ENDPOINT) { - builder = builder.with_endpoint(endpoint); - if endpoint.to_ascii_lowercase().starts_with("http://") { - builder = builder.with_allow_http(true); - } - // Custom endpoint typically means MinIO / Ceph / R2 / private S3 - // gateways that route via path style; virtual-hosted style would - // require per-bucket DNS that these backends usually lack. - builder = builder.with_virtual_hosted_style_request(false); - } - if let Some(v) = options.get(opt::S3_ACCESS_KEY_ID) { - builder = builder.with_access_key_id(v); - } - if let Some(v) = options.get(opt::S3_SECRET_ACCESS_KEY) { - builder = builder.with_secret_access_key(v); - } - if let Some(v) = options.get(opt::S3_SESSION_TOKEN) { - builder = builder.with_token(v); - } - - let client = builder.build().map_err(DeltaSinkError::ObjectStore)?; - - DeltaDestination::S3 { - prefix: path.trim_matches('/').to_string(), - client: Arc::new(client), - } + let (physical, local_root) = if let Some(bucket) = &s3_bucket { + let sink = SingleThreadPhysicalSink::try_new(bucket.clone(), path.clone(), &options)?; + (PhysicalBackend::S3(sink), None) } else { - DeltaDestination::Local(PathBuf::from(path.clone())) + let root = PathBuf::from(path.clone()); + ( + PhysicalBackend::Local(LocalPhysicalSink::new(root.clone())), + Some(root), + ) }; let mut sink_memory_block = None; @@ -194,13 +147,12 @@ impl DeltaSinkOperator { Ok(Self { table_name, - destination, + physical, parquet_compression, - pending: Vec::with_capacity(32), + pending: Vec::with_capacity(64), pending_bytes: 0, sink_memory_block, early_flush_threshold_bytes, - file_counter: 0, format, committer: None, table_uri: None, @@ -210,10 +162,10 @@ impl DeltaSinkOperator { sink_path: path, catalog_schema: catalog_schema.and_then(strip_streaming_system_columns_arc), parquet_write_schema: None, + local_root, }) } - /// Flush physical data files only; transaction commit is deferred to checkpoint. #[instrument(skip(self), fields(table = %self.table_name))] async fn flush_data_file( &mut self, @@ -228,88 +180,34 @@ impl DeltaSinkOperator { strip_streaming_system_columns_arc(self.pending[0].schema()) .expect("batch has no user columns after removing streaming system columns") }); + let batches = std::mem::take(&mut self.pending); self.pending_bytes = 0; - let record_count: u64 = batches.iter().map(|b| b.num_rows() as u64).sum(); - let format = self.format; let compression = self.parquet_compression; - let parquet_write_schema = self.parquet_write_schema.clone(); - - let bytes = tokio::task::spawn_blocking(move || { - let batches = if format == DeltaFormat::Parquet { - if let Some(ref schema) = parquet_write_schema { - cast_batches_for_delta_write(&batches, schema)? - } else { - batches - } - } else { - batches - }; - match format { - DeltaFormat::Csv => FormatEncoder::encode_csv(&batches), - DeltaFormat::Parquet => FormatEncoder::encode_parquet(&batches, compression), - DeltaFormat::JsonL => FormatEncoder::encode_jsonl(&batches), - DeltaFormat::Avro => FormatEncoder::encode_avro(&batches), - DeltaFormat::Orc => FormatEncoder::encode_orc(&batches), + let format = self.format; + let schema = self.parquet_write_schema.clone(); + + let finished = match &mut self.physical { + PhysicalBackend::S3(sink) => { + sink.write_batches(&batches, format, compression, schema, epoch, subtask_idx) + .await? } - }) - .await - .map_err(|e| DeltaSinkError::SerializationPanic(e.to_string()))? - .map_err(|e| DeltaSinkError::SerializationPanic(e.to_string()))?; + PhysicalBackend::Local(sink) => { + sink.write_batches(&batches, format, compression, schema, epoch, subtask_idx) + .await? + } + }; - if bytes.is_empty() { + let Some(finished) = finished else { return Ok(()); - } - - self.file_counter += 1; - let ext = match self.format { - DeltaFormat::Csv => "csv", - DeltaFormat::Parquet => "parquet", - DeltaFormat::JsonL => "jsonl", - DeltaFormat::Avro => "avro", - DeltaFormat::Orc => "orc", }; - let file_name = format!( - "part-{subtask_idx:05}-epoch-{epoch:010}-{counter:06}.{ext}", - counter = self.file_counter - ); - let file_size = bytes.len() as u64; - let mut relative_path = file_name.clone(); - - match &self.destination { - DeltaDestination::Local(root) => { - let out = root.join(&file_name); - let mut f = tokio::fs::File::create(&out).await?; - f.write_all(&bytes).await?; - f.flush().await?; - } - DeltaDestination::S3 { prefix, client } => { - let key = if prefix.is_empty() { - file_name - } else { - format!("{prefix}/{file_name}") - }; - relative_path = key.clone(); - client - .put( - &ObjectStorePath::from(key), - PutPayload::from(Bytes::from(bytes)), - ) - .await?; - } - } - if let Some(committer) = self.committer.as_mut() { if let Some(schema) = fallback_schema { committer.update_schema(schema)?; } - committer.register_uncommitted(UncommittedDataFile { - path: relative_path, - size_bytes: file_size, - record_count, - }); + committer.register_uncommitted(finished.into_uncommitted()); } Ok(()) @@ -346,12 +244,17 @@ impl Operator for DeltaSinkOperator { async fn on_start(&mut self, _ctx: &mut TaskContext) -> anyhow::Result<()> { ensure_delta_handlers_registered(); - if let DeltaDestination::Local(root) = &self.destination { + if let Some(root) = &self.local_root { tokio::fs::create_dir_all(root).await?; } - if let DeltaDestination::S3 { client, prefix } = &self.destination { - preflight_s3_bucket(client.as_ref(), prefix, self.s3_bucket.as_deref()).await?; + if let PhysicalBackend::S3(sink) = &self.physical { + preflight_s3_bucket( + sink.client().as_ref(), + sink.prefix_path(), + self.s3_bucket.as_deref(), + ) + .await?; } let table_uri = resolve_delta_table_uri(&self.sink_path, self.s3_bucket.as_deref())?; @@ -374,10 +277,10 @@ impl Operator for DeltaSinkOperator { info!( table = %self.table_name, - threshold = self.early_flush_threshold_bytes, + threshold_bytes = self.early_flush_threshold_bytes, is_true_delta = self.committer.is_some(), commit_strategy = %self.commit_strategy.label(), - "delta sink operator started successfully" + "delta sink operator started (physical write + checkpoint commit)" ); Ok(()) } @@ -407,7 +310,7 @@ impl Operator for DeltaSinkOperator { if self.pending_bytes > self.early_flush_threshold_bytes { debug!( bytes = self.pending_bytes, - "memory watermark reached, executing early flush (commit deferred to checkpoint)" + "memory high-watermark breached, early flush (commit deferred to checkpoint)" ); self.flush_data_file(0, ctx.subtask_index as usize).await?; } @@ -456,50 +359,3 @@ impl Operator for DeltaSinkOperator { Ok(vec![]) } } - -/// Verify the S3 bucket is reachable and produce a clear error before -/// the deltalake layer fails with a less obvious message. -async fn preflight_s3_bucket( - client: &dyn ObjectStore, - prefix: &str, - bucket: Option<&str>, -) -> Result<(), DeltaSinkError> { - let probe_path = if prefix.is_empty() { - ObjectStorePath::from("") - } else { - ObjectStorePath::from(prefix.to_string()) - }; - - let mut stream = client.list(Some(&probe_path)); - match futures::StreamExt::next(&mut stream).await { - None => Ok(()), - Some(Ok(_)) => Ok(()), - Some(Err(object_store::Error::NotFound { .. })) => Ok(()), - Some(Err(e)) => { - let msg = e.to_string(); - let lower = msg.to_ascii_lowercase(); - if lower.contains("nosuchbucket") || lower.contains("the specified bucket") { - return Err(DeltaSinkError::Config(format!( - "S3 bucket '{}' does not exist (prefix '{}'): {msg}", - bucket.unwrap_or(""), - prefix - ))); - } - if lower.contains("invalidaccesskeyid") - || lower.contains("signaturedoesnotmatch") - || lower.contains("access denied") - || lower.contains("accessdenied") - { - return Err(DeltaSinkError::Config(format!( - "S3 credential check failed for bucket '{}': {msg}", - bucket.unwrap_or("") - ))); - } - Err(DeltaSinkError::Config(format!( - "failed to access S3 bucket '{}' (prefix '{}'): {msg}", - bucket.unwrap_or(""), - prefix - ))) - } - } -} diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/physical_sink.rs b/src/streaming_runtime_operators/src/operators/sink/delta/physical_sink.rs new file mode 100644 index 00000000..c4d481ec --- /dev/null +++ b/src/streaming_runtime_operators/src/operators/sink/delta/physical_sink.rs @@ -0,0 +1,376 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Arroyo-style physical file sink: encode + PUT to object storage without touching +//! `_delta_log`. The coordinator ([`super::DeltaSinkOperator`] + [`super::delta_commit::DeltaTableCommitter`]) +//! commits metadata at checkpoint boundaries. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use arrow_array::RecordBatch; +use arrow_schema::Schema as ArrowSchema; +use bytes::Bytes; +use object_store::aws::AmazonS3Builder; +use object_store::path::Path as ObjectStorePath; +use object_store::{BackoffConfig, ObjectStore, PutPayload, RetryConfig}; +use parquet::basic::Compression; +use tokio::io::AsyncWriteExt; +use tracing::{debug, info}; + +use crate::format::encoder::FormatEncoder; +use crate::sql::common::with_option_keys as opt; + +use super::delta_commit::cast_batches_for_delta_write; +use super::{DeltaFormat, DeltaSinkError}; + +/// Descriptor returned after a successful physical write (Arroyo `FinishedFile` contract). +/// Stateless and safe to hand to the transaction committer at checkpoint time. +#[derive(Debug, Clone)] +pub struct FinishedFile { + pub filename: String, + pub size: usize, + pub record_count: u64, +} + +impl FinishedFile { + pub fn into_uncommitted(self) -> super::delta_commit::UncommittedDataFile { + super::delta_commit::UncommittedDataFile { + path: self.filename, + size_bytes: self.size as u64, + record_count: self.record_count, + } + } +} + +/// Single-writer physical sink targeting S3-compatible stores (AWS S3, MinIO, R2, Ceph). +/// +/// Uses plain `ObjectStore::put` for unique `part-*.parquet` keys — no `_delta_log` I/O and +/// no rename-based locking on this path. Delta transaction logs are written only by +/// [`super::delta_commit::DeltaTableCommitter`] at checkpoint. +pub struct SingleThreadPhysicalSink { + bucket_name: String, + prefix_path: String, + object_client: Arc, + file_counter: u64, +} + +impl SingleThreadPhysicalSink { + pub fn try_new( + bucket: String, + prefix: String, + options: &HashMap, + ) -> Result { + let client = build_s3_object_store(bucket.as_str(), options)?; + Ok(Self { + bucket_name: bucket, + prefix_path: prefix.trim_matches('/').to_string(), + object_client: client, + file_counter: 0, + }) + } + + pub fn client(&self) -> Arc { + self.object_client.clone() + } + + pub fn bucket_name(&self) -> &str { + &self.bucket_name + } + + pub fn prefix_path(&self) -> &str { + &self.prefix_path + } + + /// Encode batches off the async runtime, then atomically PUT one object. + pub async fn write_batches( + &mut self, + batches: &[RecordBatch], + format: DeltaFormat, + compression: Compression, + parquet_write_schema: Option>, + epoch: u64, + subtask_idx: usize, + ) -> Result, DeltaSinkError> { + if batches.is_empty() { + return Ok(None); + } + + let record_count: u64 = batches.iter().map(|b| b.num_rows() as u64).sum(); + let owned_batches = batches.to_vec(); + + let encoded = tokio::task::spawn_blocking(move || { + encode_batches( + owned_batches, + format, + compression, + parquet_write_schema.as_deref(), + ) + }) + .await + .map_err(|e| DeltaSinkError::SerializationPanic(e.to_string()))? + .map_err(DeltaSinkError::SerializationPanic)?; + + self.put_encoded(encoded, record_count, format, epoch, subtask_idx) + .await + } + + async fn put_encoded( + &mut self, + encoded: Vec, + record_count: u64, + format: DeltaFormat, + epoch: u64, + subtask_idx: usize, + ) -> Result, DeltaSinkError> { + if encoded.is_empty() { + return Ok(None); + } + + self.file_counter += 1; + let ext = format.file_extension(); + let file_name = format!( + "part-{subtask_idx:05}-epoch-{epoch:010}-{counter:06}.{ext}", + counter = self.file_counter + ); + let object_key = if self.prefix_path.is_empty() { + file_name + } else { + format!("{}/{}", self.prefix_path, file_name) + }; + + let size = encoded.len(); + debug!( + target_key = %object_key, + size, + record_count, + "uploading data file to S3-compatible storage" + ); + + self.object_client + .put( + &ObjectStorePath::from(object_key.clone()), + PutPayload::from(Bytes::from(encoded)), + ) + .await?; + + info!(file = %object_key, size, record_count, "data file persisted to object storage"); + + Ok(Some(FinishedFile { + filename: object_key, + size, + record_count, + })) + } +} + +/// Local filesystem physical sink (same contract as [`SingleThreadPhysicalSink`]). +pub struct LocalPhysicalSink { + root: PathBuf, + file_counter: u64, +} + +impl LocalPhysicalSink { + pub fn new(root: PathBuf) -> Self { + Self { + root, + file_counter: 0, + } + } + + pub async fn write_batches( + &mut self, + batches: &[RecordBatch], + format: DeltaFormat, + compression: Compression, + parquet_write_schema: Option>, + epoch: u64, + subtask_idx: usize, + ) -> Result, DeltaSinkError> { + if batches.is_empty() { + return Ok(None); + } + + let record_count: u64 = batches.iter().map(|b| b.num_rows() as u64).sum(); + let owned_batches = batches.to_vec(); + + let encoded = tokio::task::spawn_blocking(move || { + encode_batches( + owned_batches, + format, + compression, + parquet_write_schema.as_deref(), + ) + }) + .await + .map_err(|e| DeltaSinkError::SerializationPanic(e.to_string()))? + .map_err(DeltaSinkError::SerializationPanic)?; + + if encoded.is_empty() { + return Ok(None); + } + + self.file_counter += 1; + let ext = format.file_extension(); + let file_name = format!( + "part-{subtask_idx:05}-epoch-{epoch:010}-{counter:06}.{ext}", + counter = self.file_counter + ); + let out = self.root.join(&file_name); + let size = encoded.len(); + + let mut f = tokio::fs::File::create(&out).await?; + f.write_all(&encoded).await?; + f.flush().await?; + + info!(file = %file_name, size, record_count, "data file persisted to local storage"); + + Ok(Some(FinishedFile { + filename: file_name, + size, + record_count, + })) + } +} + +/// Build an S3-compatible [`ObjectStore`] client with production defaults for MinIO / private cloud. +pub fn build_s3_object_store( + bucket: &str, + options: &HashMap, +) -> Result, DeltaSinkError> { + let region = options + .get(opt::S3_REGION) + .map(|s| s.as_str()) + .unwrap_or("us-east-1"); + + let retry_config = RetryConfig { + backoff: BackoffConfig { + init_backoff: Duration::from_millis(100), + max_backoff: Duration::from_secs(3), + base: 2.0, + }, + max_retries: 4, + retry_timeout: Duration::from_secs(15), + }; + + let mut builder = AmazonS3Builder::new() + .with_bucket_name(bucket) + .with_region(region) + .with_retry(retry_config); + + if let Some(endpoint) = options.get(opt::S3_ENDPOINT) { + builder = builder.with_endpoint(endpoint); + if endpoint.to_ascii_lowercase().starts_with("http://") { + builder = builder.with_allow_http(true); + } + builder = builder.with_virtual_hosted_style_request(false); + } + if let Some(v) = options.get(opt::S3_ACCESS_KEY_ID) { + builder = builder.with_access_key_id(v); + } + if let Some(v) = options.get(opt::S3_SECRET_ACCESS_KEY) { + builder = builder.with_secret_access_key(v); + } + if let Some(v) = options.get(opt::S3_SESSION_TOKEN) { + builder = builder.with_token(v); + } + + Ok(Arc::new( + builder.build().map_err(DeltaSinkError::ObjectStore)?, + )) +} + +/// Preflight bucket connectivity before the pipeline accepts traffic. +pub async fn preflight_s3_bucket( + client: &dyn ObjectStore, + prefix: &str, + bucket: Option<&str>, +) -> Result<(), DeltaSinkError> { + let probe_path = if prefix.is_empty() { + ObjectStorePath::from("") + } else { + ObjectStorePath::from(prefix.to_string()) + }; + + let mut stream = client.list(Some(&probe_path)); + match futures::StreamExt::next(&mut stream).await { + None | Some(Ok(_)) => Ok(()), + Some(Err(object_store::Error::NotFound { .. })) => Ok(()), + Some(Err(e)) => { + let msg = e.to_string(); + let lower = msg.to_ascii_lowercase(); + if lower.contains("nosuchbucket") || lower.contains("the specified bucket") { + return Err(DeltaSinkError::Config(format!( + "S3 bucket '{}' does not exist (prefix '{}'): {msg}", + bucket.unwrap_or(""), + prefix + ))); + } + if lower.contains("invalidaccesskeyid") + || lower.contains("signaturedoesnotmatch") + || lower.contains("access denied") + || lower.contains("accessdenied") + { + return Err(DeltaSinkError::Config(format!( + "S3 credential check failed for bucket '{}': {msg}", + bucket.unwrap_or("") + ))); + } + Err(DeltaSinkError::Config(format!( + "failed to access S3 bucket '{}' (prefix '{}'): {msg}", + bucket.unwrap_or(""), + prefix + ))) + } + } +} + +fn encode_batches( + batches: Vec, + format: DeltaFormat, + compression: Compression, + parquet_write_schema: Option<&ArrowSchema>, +) -> Result, String> { + let batches = if format == DeltaFormat::Parquet { + if let Some(schema) = parquet_write_schema { + cast_batches_for_delta_write(&batches, schema).map_err(|e| e.to_string())? + } else { + batches + } + } else { + batches + }; + + match format { + DeltaFormat::Csv => FormatEncoder::encode_csv(&batches).map_err(|e| e.to_string()), + DeltaFormat::Parquet => { + FormatEncoder::encode_parquet(&batches, compression).map_err(|e| e.to_string()) + } + DeltaFormat::JsonL => FormatEncoder::encode_jsonl(&batches).map_err(|e| e.to_string()), + DeltaFormat::Avro => FormatEncoder::encode_avro(&batches).map_err(|e| e.to_string()), + DeltaFormat::Orc => FormatEncoder::encode_orc(&batches).map_err(|e| e.to_string()), + } +} + +impl DeltaFormat { + fn file_extension(self) -> &'static str { + match self { + DeltaFormat::Csv => "csv", + DeltaFormat::Parquet => "parquet", + DeltaFormat::JsonL => "jsonl", + DeltaFormat::Avro => "avro", + DeltaFormat::Orc => "orc", + } + } +} From e7bc25a4cd1cf2c157a6d154d558b88738b23174 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 20 May 2026 23:24:48 +0800 Subject: [PATCH 14/16] update --- .../{physical_sink.rs => delta_writer.rs} | 205 +++++++++--------- .../src/operators/sink/delta/mod.rs | 40 ++-- 2 files changed, 123 insertions(+), 122 deletions(-) rename src/streaming_runtime_operators/src/operators/sink/delta/{physical_sink.rs => delta_writer.rs} (65%) diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/physical_sink.rs b/src/streaming_runtime_operators/src/operators/sink/delta/delta_writer.rs similarity index 65% rename from src/streaming_runtime_operators/src/operators/sink/delta/physical_sink.rs rename to src/streaming_runtime_operators/src/operators/sink/delta/delta_writer.rs index c4d481ec..05a6313f 100644 --- a/src/streaming_runtime_operators/src/operators/sink/delta/physical_sink.rs +++ b/src/streaming_runtime_operators/src/operators/sink/delta/delta_writer.rs @@ -10,14 +10,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Arroyo-style physical file sink: encode + PUT to object storage without touching -//! `_delta_log`. The coordinator ([`super::DeltaSinkOperator`] + [`super::delta_commit::DeltaTableCommitter`]) -//! commits metadata at checkpoint boundaries. +//! Delta data-file writer: encode batches and PUT `part-*` objects (S3/MinIO or local). +//! Does not write `_delta_log`; [`super::delta_commit::DeltaTableCommitter`] handles that +//! at checkpoint. use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, Instant}; use arrow_array::RecordBatch; use arrow_schema::Schema as ArrowSchema; @@ -27,7 +27,7 @@ use object_store::path::Path as ObjectStorePath; use object_store::{BackoffConfig, ObjectStore, PutPayload, RetryConfig}; use parquet::basic::Compression; use tokio::io::AsyncWriteExt; -use tracing::{debug, info}; +use tracing::{debug, info, instrument}; use crate::format::encoder::FormatEncoder; use crate::sql::common::with_option_keys as opt; @@ -35,8 +35,7 @@ use crate::sql::common::with_option_keys as opt; use super::delta_commit::cast_batches_for_delta_write; use super::{DeltaFormat, DeltaSinkError}; -/// Descriptor returned after a successful physical write (Arroyo `FinishedFile` contract). -/// Stateless and safe to hand to the transaction committer at checkpoint time. +/// Descriptor for a completed data-file write (two-phase commit: register at checkpoint). #[derive(Debug, Clone)] pub struct FinishedFile { pub filename: String, @@ -54,19 +53,15 @@ impl FinishedFile { } } -/// Single-writer physical sink targeting S3-compatible stores (AWS S3, MinIO, R2, Ceph). -/// -/// Uses plain `ObjectStore::put` for unique `part-*.parquet` keys — no `_delta_log` I/O and -/// no rename-based locking on this path. Delta transaction logs are written only by -/// [`super::delta_commit::DeltaTableCommitter`] at checkpoint. -pub struct SingleThreadPhysicalSink { +/// S3-compatible data-file writer (AWS S3, MinIO, R2). No `_delta_log` I/O on this path. +pub struct DeltaSink { bucket_name: String, prefix_path: String, object_client: Arc, file_counter: u64, } -impl SingleThreadPhysicalSink { +impl DeltaSink { pub fn try_new( bucket: String, prefix: String, @@ -93,7 +88,7 @@ impl SingleThreadPhysicalSink { &self.prefix_path } - /// Encode batches off the async runtime, then atomically PUT one object. + #[instrument(skip(self, batches, parquet_write_schema), fields(epoch, subtask = subtask_idx))] pub async fn write_batches( &mut self, batches: &[RecordBatch], @@ -108,32 +103,11 @@ impl SingleThreadPhysicalSink { } let record_count: u64 = batches.iter().map(|b| b.num_rows() as u64).sum(); - let owned_batches = batches.to_vec(); - - let encoded = tokio::task::spawn_blocking(move || { - encode_batches( - owned_batches, - format, - compression, - parquet_write_schema.as_deref(), - ) - }) - .await - .map_err(|e| DeltaSinkError::SerializationPanic(e.to_string()))? - .map_err(DeltaSinkError::SerializationPanic)?; - self.put_encoded(encoded, record_count, format, epoch, subtask_idx) - .await - } + let encoded = + encode_in_background(batches.to_vec(), format, compression, parquet_write_schema) + .await?; - async fn put_encoded( - &mut self, - encoded: Vec, - record_count: u64, - format: DeltaFormat, - epoch: u64, - subtask_idx: usize, - ) -> Result, DeltaSinkError> { if encoded.is_empty() { return Ok(None); } @@ -153,19 +127,27 @@ impl SingleThreadPhysicalSink { let size = encoded.len(); debug!( target_key = %object_key, - size, + size_bytes = size, record_count, - "uploading data file to S3-compatible storage" + "uploading delta data file to object storage" ); + let start_io = Instant::now(); self.object_client .put( &ObjectStorePath::from(object_key.clone()), PutPayload::from(Bytes::from(encoded)), ) .await?; - - info!(file = %object_key, size, record_count, "data file persisted to object storage"); + let io_duration = start_io.elapsed(); + + info!( + file = %object_key, + size_bytes = size, + records = record_count, + io_latency_ms = io_duration.as_millis(), + "delta data file persisted to object storage" + ); Ok(Some(FinishedFile { filename: object_key, @@ -175,13 +157,13 @@ impl SingleThreadPhysicalSink { } } -/// Local filesystem physical sink (same contract as [`SingleThreadPhysicalSink`]). -pub struct LocalPhysicalSink { +/// Local filesystem data-file writer with tmp + fsync + atomic rename. +pub struct DeltaLocalSink { root: PathBuf, file_counter: u64, } -impl LocalPhysicalSink { +impl DeltaLocalSink { pub fn new(root: PathBuf) -> Self { Self { root, @@ -189,6 +171,7 @@ impl LocalPhysicalSink { } } + #[instrument(skip(self, batches, parquet_write_schema), fields(epoch, subtask = subtask_idx))] pub async fn write_batches( &mut self, batches: &[RecordBatch], @@ -203,19 +186,10 @@ impl LocalPhysicalSink { } let record_count: u64 = batches.iter().map(|b| b.num_rows() as u64).sum(); - let owned_batches = batches.to_vec(); - - let encoded = tokio::task::spawn_blocking(move || { - encode_batches( - owned_batches, - format, - compression, - parquet_write_schema.as_deref(), - ) - }) - .await - .map_err(|e| DeltaSinkError::SerializationPanic(e.to_string()))? - .map_err(DeltaSinkError::SerializationPanic)?; + + let encoded = + encode_in_background(batches.to_vec(), format, compression, parquet_write_schema) + .await?; if encoded.is_empty() { return Ok(None); @@ -227,14 +201,30 @@ impl LocalPhysicalSink { "part-{subtask_idx:05}-epoch-{epoch:010}-{counter:06}.{ext}", counter = self.file_counter ); - let out = self.root.join(&file_name); + let final_out = self.root.join(&file_name); + let tmp_out = self.root.join(format!("{file_name}.tmp")); let size = encoded.len(); - let mut f = tokio::fs::File::create(&out).await?; - f.write_all(&encoded).await?; - f.flush().await?; + let start_io = Instant::now(); + { + let mut f = tokio::fs::File::create(&tmp_out).await?; + f.write_all(&encoded).await?; + f.sync_all().await?; + } - info!(file = %file_name, size, record_count, "data file persisted to local storage"); + if let Err(e) = tokio::fs::rename(&tmp_out, &final_out).await { + let _ = tokio::fs::remove_file(&tmp_out).await; + return Err(DeltaSinkError::Io(e)); + } + let io_duration = start_io.elapsed(); + + info!( + file = %file_name, + size_bytes = size, + records = record_count, + io_latency_ms = io_duration.as_millis(), + "delta data file atomically persisted locally" + ); Ok(Some(FinishedFile { filename: file_name, @@ -244,7 +234,47 @@ impl LocalPhysicalSink { } } -/// Build an S3-compatible [`ObjectStore`] client with production defaults for MinIO / private cloud. +/// Offload CPU-heavy encoding to the blocking thread pool. +async fn encode_in_background( + owned_batches: Vec, + format: DeltaFormat, + compression: Compression, + parquet_write_schema: Option>, +) -> Result, DeltaSinkError> { + let start_cpu = Instant::now(); + let encoded = tokio::task::spawn_blocking(move || { + let batches = if format == DeltaFormat::Parquet { + if let Some(schema) = parquet_write_schema { + cast_batches_for_delta_write(&owned_batches, schema.as_ref()) + .map_err(|e| e.to_string())? + } else { + owned_batches + } + } else { + owned_batches + }; + + match format { + DeltaFormat::Csv => FormatEncoder::encode_csv(&batches).map_err(|e| e.to_string()), + DeltaFormat::Parquet => { + FormatEncoder::encode_parquet(&batches, compression).map_err(|e| e.to_string()) + } + DeltaFormat::JsonL => FormatEncoder::encode_jsonl(&batches).map_err(|e| e.to_string()), + DeltaFormat::Avro => FormatEncoder::encode_avro(&batches).map_err(|e| e.to_string()), + DeltaFormat::Orc => FormatEncoder::encode_orc(&batches).map_err(|e| e.to_string()), + } + }) + .await + .map_err(|e| DeltaSinkError::SerializationPanic(format!("worker thread panicked: {e}")))? + .map_err(DeltaSinkError::SerializationPanic)?; + + debug!( + cpu_latency_ms = start_cpu.elapsed().as_millis(), + "record batch serialization completed" + ); + Ok(encoded) +} + pub fn build_s3_object_store( bucket: &str, options: &HashMap, @@ -256,12 +286,12 @@ pub fn build_s3_object_store( let retry_config = RetryConfig { backoff: BackoffConfig { - init_backoff: Duration::from_millis(100), - max_backoff: Duration::from_secs(3), + init_backoff: Duration::from_millis(200), + max_backoff: Duration::from_secs(5), base: 2.0, }, - max_retries: 4, - retry_timeout: Duration::from_secs(15), + max_retries: 5, + retry_timeout: Duration::from_secs(30), }; let mut builder = AmazonS3Builder::new() @@ -291,7 +321,6 @@ pub fn build_s3_object_store( )) } -/// Preflight bucket connectivity before the pipeline accepts traffic. pub async fn preflight_s3_bucket( client: &dyn ObjectStore, prefix: &str, @@ -312,9 +341,8 @@ pub async fn preflight_s3_bucket( let lower = msg.to_ascii_lowercase(); if lower.contains("nosuchbucket") || lower.contains("the specified bucket") { return Err(DeltaSinkError::Config(format!( - "S3 bucket '{}' does not exist (prefix '{}'): {msg}", - bucket.unwrap_or(""), - prefix + "S3 bucket '{}' does not exist", + bucket.unwrap_or("") ))); } if lower.contains("invalidaccesskeyid") @@ -323,7 +351,7 @@ pub async fn preflight_s3_bucket( || lower.contains("accessdenied") { return Err(DeltaSinkError::Config(format!( - "S3 credential check failed for bucket '{}': {msg}", + "S3 authentication failed for bucket '{}': {msg}", bucket.unwrap_or("") ))); } @@ -336,35 +364,8 @@ pub async fn preflight_s3_bucket( } } -fn encode_batches( - batches: Vec, - format: DeltaFormat, - compression: Compression, - parquet_write_schema: Option<&ArrowSchema>, -) -> Result, String> { - let batches = if format == DeltaFormat::Parquet { - if let Some(schema) = parquet_write_schema { - cast_batches_for_delta_write(&batches, schema).map_err(|e| e.to_string())? - } else { - batches - } - } else { - batches - }; - - match format { - DeltaFormat::Csv => FormatEncoder::encode_csv(&batches).map_err(|e| e.to_string()), - DeltaFormat::Parquet => { - FormatEncoder::encode_parquet(&batches, compression).map_err(|e| e.to_string()) - } - DeltaFormat::JsonL => FormatEncoder::encode_jsonl(&batches).map_err(|e| e.to_string()), - DeltaFormat::Avro => FormatEncoder::encode_avro(&batches).map_err(|e| e.to_string()), - DeltaFormat::Orc => FormatEncoder::encode_orc(&batches).map_err(|e| e.to_string()), - } -} - impl DeltaFormat { - fn file_extension(self) -> &'static str { + pub(crate) fn file_extension(self) -> &'static str { match self { DeltaFormat::Csv => "csv", DeltaFormat::Parquet => "parquet", diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs index 5c79f713..2e7baba8 100644 --- a/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs @@ -11,14 +11,15 @@ // limitations under the License. //! Delta Lake sink with Arroyo-style separation: -//! - [`physical_sink`]: encode + PUT part files (no `_delta_log` I/O) -//! - [`delta_commit`]: checkpoint-time transaction commit +//! - [`delta_writer`]: [`DeltaSink`] / [`DeltaLocalSink`] write `part-*` data files +//! - [`delta_commit`]: checkpoint-time `_delta_log` transaction commit +//! - [`DeltaSinkOperator`]: stream operator coordinator mod delta_commit; -mod physical_sink; +mod delta_writer; pub use delta_commit::strip_streaming_system_columns_arc; -pub use physical_sink::FinishedFile; +pub use delta_writer::FinishedFile; use std::collections::HashMap; use std::path::PathBuf; @@ -43,7 +44,7 @@ use crate::sql::common::constants::factory_operator_name; use crate::sql::common::with_option_keys as opt; use crate::sql::common::{CheckpointBarrier, Watermark}; -use physical_sink::{LocalPhysicalSink, SingleThreadPhysicalSink, preflight_s3_bucket}; +use delta_writer::{DeltaLocalSink, DeltaSink, preflight_s3_bucket}; const DEFAULT_MAX_BUFFER_BYTES: usize = 64 * 1024 * 1024; @@ -72,10 +73,9 @@ pub enum DeltaSinkError { Config(String), } -/// Physical storage backend: part-file writer only. -enum PhysicalBackend { - Local(LocalPhysicalSink), - S3(SingleThreadPhysicalSink), +enum DeltaSinkBackend { + Local(DeltaLocalSink), + ObjectStore(DeltaSink), } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -87,11 +87,11 @@ pub enum DeltaFormat { Orc, } -/// Coordinator operator: buffers batches, delegates physical writes to [`PhysicalBackend`], +/// Stream operator: buffers batches, delegates data-file writes to [`DeltaSinkBackend`], /// and commits `_delta_log` at checkpoint via [`DeltaTableCommitter`]. pub struct DeltaSinkOperator { table_name: String, - physical: PhysicalBackend, + sink: DeltaSinkBackend, parquet_compression: Compression, pending: Vec, pending_bytes: usize, @@ -123,13 +123,13 @@ impl DeltaSinkOperator { let s3_bucket = options.get(opt::S3_BUCKET).cloned(); let (storage_options, commit_strategy) = build_delta_storage_options(&options)?; - let (physical, local_root) = if let Some(bucket) = &s3_bucket { - let sink = SingleThreadPhysicalSink::try_new(bucket.clone(), path.clone(), &options)?; - (PhysicalBackend::S3(sink), None) + let (sink, local_root) = if let Some(bucket) = &s3_bucket { + let s3 = DeltaSink::try_new(bucket.clone(), path.clone(), &options)?; + (DeltaSinkBackend::ObjectStore(s3), None) } else { let root = PathBuf::from(path.clone()); ( - PhysicalBackend::Local(LocalPhysicalSink::new(root.clone())), + DeltaSinkBackend::Local(DeltaLocalSink::new(root.clone())), Some(root), ) }; @@ -147,7 +147,7 @@ impl DeltaSinkOperator { Ok(Self { table_name, - physical, + sink, parquet_compression, pending: Vec::with_capacity(64), pending_bytes: 0, @@ -188,12 +188,12 @@ impl DeltaSinkOperator { let format = self.format; let schema = self.parquet_write_schema.clone(); - let finished = match &mut self.physical { - PhysicalBackend::S3(sink) => { + let finished = match &mut self.sink { + DeltaSinkBackend::ObjectStore(sink) => { sink.write_batches(&batches, format, compression, schema, epoch, subtask_idx) .await? } - PhysicalBackend::Local(sink) => { + DeltaSinkBackend::Local(sink) => { sink.write_batches(&batches, format, compression, schema, epoch, subtask_idx) .await? } @@ -248,7 +248,7 @@ impl Operator for DeltaSinkOperator { tokio::fs::create_dir_all(root).await?; } - if let PhysicalBackend::S3(sink) = &self.physical { + if let DeltaSinkBackend::ObjectStore(sink) = &self.sink { preflight_s3_bucket( sink.client().as_ref(), sink.prefix_path(), From 3cf300dd8add5efd8fc139e392ab047aa24c7660 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Sun, 31 May 2026 23:38:43 +0800 Subject: [PATCH 15/16] feat(connectors): add robot sources, Kafka group ID defaults, and throughput fixes Add MQTT, HTTP, ROS/ROS2, and robot-bag connectors with ROS1 bag and PCL PCD parsing. Persist auto-generated Kafka consumer group IDs at CREATE TABLE. Remove hot-path debug logging from Kafka source and Delta sink, and tune Delta/Kafka batching for higher throughput. Add bilingual source docs. Co-authored-by: Cursor --- Cargo.lock | 446 ++++++++++++++++- docs/streaming-sql/Source/README-zh.md | 15 +- docs/streaming-sql/Source/README.md | 20 +- docs/streaming-sql/Source/http-source-zh.md | 87 ++++ docs/streaming-sql/Source/kafka-source-zh.md | 19 +- docs/streaming-sql/Source/kafka-source.md | 32 +- docs/streaming-sql/Source/mqtt-source-zh.md | 49 ++ .../Source/robot-bag-source-zh.md | 151 ++++++ docs/streaming-sql/Source/robot-bag-source.md | 153 ++++++ docs/streaming-sql/Source/robot-sources-zh.md | 98 ++++ docs/streaming-sql/Source/robot-sources.md | 79 +++ docs/streaming-sql/Source/ros-source-zh.md | 41 ++ docs/streaming-sql/Source/ros2-source-zh.md | 58 +++ protocol/proto/function_stream_graph.proto | 94 ++++ src/streaming_planner/src/common/constants.rs | 9 + .../src/common/with_option_keys.rs | 58 +++ src/streaming_planner/src/connector/config.rs | 29 +- .../src/connector/registry.rs | 10 + .../src/connector/source/http.rs | 258 ++++++++++ .../src/connector/source/kafka.rs | 98 +++- .../src/connector/source/mod.rs | 6 + .../src/connector/source/mqtt.rs | 206 ++++++++ .../src/connector/source/robot_bag.rs | 184 +++++++ .../src/connector/source/ros.rs | 44 ++ .../src/connector/source/ros2.rs | 45 ++ .../src/connector/source/ros_common.rs | 133 +++++ .../src/schema/introspection/ddl_compiler.rs | 7 + src/streaming_runtime_core/src/api/context.rs | 2 +- src/streaming_runtime_operators/Cargo.toml | 9 + .../src/factory/connector/dispatchers.rs | 9 +- .../src/factory/connector/http.rs | 153 ++++++ .../src/factory/connector/kafka.rs | 8 +- .../src/factory/connector/mod.rs | 8 + .../src/factory/connector/mqtt.rs | 138 ++++++ .../src/factory/connector/robot_bag.rs | 134 +++++ .../src/factory/connector/ros.rs | 126 +++++ .../src/format/encoder.rs | 14 +- .../src/operators/sink/delta/delta_writer.rs | 27 +- .../src/operators/sink/delta/mod.rs | 48 +- .../src/operators/source/batch_buffer.rs | 24 + .../src/operators/source/http/mod.rs | 449 +++++++++++++++++ .../src/operators/source/kafka/mod.rs | 211 ++++---- .../src/operators/source/mod.rs | 5 + .../src/operators/source/mqtt/mod.rs | 243 +++++++++ .../src/operators/source/robot_bag/mod.rs | 460 +++++++++++++++++ .../src/operators/source/robot_bag/pcd.rs | 461 ++++++++++++++++++ .../operators/source/robot_bag/pcd_body.rs | 230 +++++++++ .../operators/source/robot_bag/ros1_bag.rs | 145 ++++++ .../src/operators/source/ros/mod.rs | 272 +++++++++++ 49 files changed, 5417 insertions(+), 188 deletions(-) create mode 100644 docs/streaming-sql/Source/http-source-zh.md create mode 100644 docs/streaming-sql/Source/mqtt-source-zh.md create mode 100644 docs/streaming-sql/Source/robot-bag-source-zh.md create mode 100644 docs/streaming-sql/Source/robot-bag-source.md create mode 100644 docs/streaming-sql/Source/robot-sources-zh.md create mode 100644 docs/streaming-sql/Source/robot-sources.md create mode 100644 docs/streaming-sql/Source/ros-source-zh.md create mode 100644 docs/streaming-sql/Source/ros2-source-zh.md create mode 100644 src/streaming_planner/src/connector/source/http.rs create mode 100644 src/streaming_planner/src/connector/source/mqtt.rs create mode 100644 src/streaming_planner/src/connector/source/robot_bag.rs create mode 100644 src/streaming_planner/src/connector/source/ros.rs create mode 100644 src/streaming_planner/src/connector/source/ros2.rs create mode 100644 src/streaming_planner/src/connector/source/ros_common.rs create mode 100644 src/streaming_runtime_operators/src/factory/connector/http.rs create mode 100644 src/streaming_runtime_operators/src/factory/connector/mqtt.rs create mode 100644 src/streaming_runtime_operators/src/factory/connector/robot_bag.rs create mode 100644 src/streaming_runtime_operators/src/factory/connector/ros.rs create mode 100644 src/streaming_runtime_operators/src/operators/source/batch_buffer.rs create mode 100644 src/streaming_runtime_operators/src/operators/source/http/mod.rs create mode 100644 src/streaming_runtime_operators/src/operators/source/mqtt/mod.rs create mode 100644 src/streaming_runtime_operators/src/operators/source/robot_bag/mod.rs create mode 100644 src/streaming_runtime_operators/src/operators/source/robot_bag/pcd.rs create mode 100644 src/streaming_runtime_operators/src/operators/source/robot_bag/pcd_body.rs create mode 100644 src/streaming_runtime_operators/src/operators/source/robot_bag/ros1_bag.rs create mode 100644 src/streaming_runtime_operators/src/operators/source/ros/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 4f1a1dfe..ea8b82c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -180,6 +180,12 @@ dependencies = [ "rustversion", ] +[[package]] +name = "array-init" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" + [[package]] name = "arrayref" version = "0.3.9" @@ -963,7 +969,7 @@ version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" dependencies = [ - "bzip2", + "bzip2 0.5.2", "flate2", "futures-core", "memchr", @@ -1314,11 +1320,11 @@ dependencies = [ "hyper-rustls", "hyper-util", "pin-project-lite", - "rustls", - "rustls-native-certs", + "rustls 0.23.40", + "rustls-native-certs 0.8.3", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tower 0.5.3", "tracing", ] @@ -1481,6 +1487,8 @@ dependencies = [ "http 1.4.0", "http-body 1.0.1", "http-body-util", + "hyper", + "hyper-util", "itoa", "matchit", "memchr", @@ -1489,10 +1497,15 @@ dependencies = [ "pin-project-lite", "rustversion", "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", "sync_wrapper", + "tokio", "tower 0.5.3", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -1513,6 +1526,7 @@ dependencies = [ "sync_wrapper", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -1526,6 +1540,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + [[package]] name = "base64" version = "0.22.1" @@ -1556,6 +1576,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bimap" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" + [[package]] name = "bincode" version = "2.0.1" @@ -1615,6 +1641,30 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "binrw" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1faf7031c34c71da53eec4e070cf90c3b825729e21ca3aab51b20da4a1d1d9" +dependencies = [ + "array-init", + "binrw_derive", + "bytemuck", +] + +[[package]] +name = "binrw_derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c5eb3446e2f5ea7fa9a6f2cb594648c73bf2dbc60eccf3b2fa41834e5449150" +dependencies = [ + "either", + "owo-colors", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -1825,6 +1875,16 @@ dependencies = [ "either", ] +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + [[package]] name = "bzip2" version = "0.5.2" @@ -2569,6 +2629,16 @@ dependencies = [ "darling_macro 0.20.11", ] +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core 0.21.3", + "darling_macro 0.21.3", +] + [[package]] name = "darling" version = "0.23.0" @@ -2593,6 +2663,19 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "darling_core" version = "0.23.0" @@ -2617,6 +2700,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core 0.21.3", + "quote", + "syn 2.0.117", +] + [[package]] name = "darling_macro" version = "0.23.0" @@ -2642,6 +2736,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + [[package]] name = "datafusion" version = "48.0.1" @@ -2652,7 +2752,7 @@ dependencies = [ "arrow-schema 55.2.0", "async-trait", "bytes", - "bzip2", + "bzip2 0.5.2", "chrono", "datafusion-catalog 48.0.1", "datafusion-catalog-listing 48.0.1", @@ -2914,7 +3014,7 @@ dependencies = [ "async-compression", "async-trait", "bytes", - "bzip2", + "bzip2 0.5.2", "chrono", "datafusion-common 48.0.1", "datafusion-common-runtime 48.0.1", @@ -4150,6 +4250,27 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" +[[package]] +name = "enumset" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "839c4174b41e75c8f7306110b2c51996a293b8d1d850edd529011841d9fede7d" +dependencies = [ + "enumset_derive", +] + +[[package]] +name = "enumset_derive" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd536557b58c682b217b8fb199afdff47cd3eff260623f19e77074eb073d63a" +dependencies = [ + "darling 0.21.3", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "env_logger" version = "0.10.2" @@ -4218,6 +4339,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fast-float2" version = "0.2.3" @@ -4296,6 +4423,17 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -4624,8 +4762,10 @@ dependencies = [ "arrow-json 55.2.0", "arrow-schema 55.2.0", "async-trait", + "axum", "base64", "bincode", + "byteorder", "bytes", "datafusion 48.0.1", "datafusion-common 48.0.1", @@ -4642,15 +4782,22 @@ dependencies = [ "governor", "itertools 0.14.0", "lance", + "lzf", + "mcap", "object_store 0.12.5", "parquet 55.2.0", "prost 0.13.5", "protocol", "rdkafka", + "reqwest 0.12.28", + "rosbag", + "rumqttc", + "rusqlite", "serde", "serde_json", "thiserror 2.0.18", "tokio", + "tokio-tungstenite", "tracing", "url", ] @@ -4967,6 +5114,15 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "heck" version = "0.5.0" @@ -5131,12 +5287,12 @@ dependencies = [ "http 1.4.0", "hyper", "hyper-util", - "rustls", - "rustls-native-certs", + "rustls 0.23.40", + "rustls-native-certs 0.8.3", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tower-service", - "webpki-roots", + "webpki-roots 1.0.7", ] [[package]] @@ -6247,6 +6403,17 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "libz-sys" version = "1.1.28" @@ -6378,6 +6545,12 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "lzf" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c034ef7ca69017088f8522e6f74301232f13ba43b5640941abc4dce26f530521" + [[package]] name = "lzma-sys" version = "0.1.20" @@ -6432,6 +6605,26 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" +[[package]] +name = "mcap" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a5a8225e54ba304d8bf118679f422a884cfcf5353cf5618a1e1b8aa135ee728" +dependencies = [ + "bimap", + "binrw", + "byteorder", + "crc32fast", + "enumset", + "log", + "lz4", + "num_cpus", + "paste", + "static_assertions", + "thiserror 1.0.69", + "zstd", +] + [[package]] name = "md-5" version = "0.10.6" @@ -6896,6 +7089,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + [[package]] name = "openssl-probe" version = "0.2.1" @@ -6973,6 +7172,12 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "owo-colors" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" + [[package]] name = "parking" version = "2.2.1" @@ -7527,7 +7732,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.2", - "rustls", + "rustls 0.23.40", "socket2 0.6.3", "thiserror 2.0.18", "tokio", @@ -7548,7 +7753,7 @@ dependencies = [ "rand 0.9.4", "ring", "rustc-hash 2.1.2", - "rustls", + "rustls 0.23.40", "rustls-pki-types", "slab", "thiserror 2.0.18", @@ -7965,15 +8170,15 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls", - "rustls-native-certs", + "rustls 0.23.40", + "rustls-native-certs 0.8.3", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tokio-util", "tower 0.5.3", "tower-http", @@ -7983,7 +8188,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots", + "webpki-roots 1.0.7", ] [[package]] @@ -8009,12 +8214,12 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls", + "rustls 0.23.40", "rustls-pki-types", "rustls-platform-verifier", "sync_wrapper", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tower 0.5.3", "tower-http", "tower-service", @@ -8058,6 +8263,52 @@ dependencies = [ "librocksdb-sys", ] +[[package]] +name = "rosbag" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef77688c2fe3d1a94991e3d7256face190d6b6ea1a1ac60a6e2bad8e2aeabde7" +dependencies = [ + "base16ct", + "byteorder", + "bzip2 0.4.4", + "log", + "lz4", + "memmap2", +] + +[[package]] +name = "rumqttc" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1568e15fab2d546f940ed3a21f48bbbd1c494c90c99c4481339364a497f94a9" +dependencies = [ + "bytes", + "flume", + "futures-util", + "log", + "rustls-native-certs 0.7.3", + "rustls-pemfile", + "rustls-webpki 0.102.8", + "thiserror 1.0.69", + "tokio", + "tokio-rustls 0.25.0", +] + +[[package]] +name = "rusqlite" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +dependencies = [ + "bitflags 2.11.1", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rust-ini" version = "0.21.3" @@ -8141,6 +8392,20 @@ dependencies = [ "rustix 1.1.4", ] +[[package]] +name = "rustls" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" +dependencies = [ + "log", + "ring", + "rustls-pki-types", + "rustls-webpki 0.102.8", + "subtle", + "zeroize", +] + [[package]] name = "rustls" version = "0.23.40" @@ -8151,21 +8416,43 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.103.13", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" +dependencies = [ + "openssl-probe 0.1.6", + "rustls-pemfile", + "rustls-pki-types", + "schannel", + "security-framework 2.11.1", +] + [[package]] name = "rustls-native-certs" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" dependencies = [ - "openssl-probe", + "openssl-probe 0.2.1", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 3.7.0", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", ] [[package]] @@ -8189,11 +8476,11 @@ dependencies = [ "jni", "log", "once_cell", - "rustls", - "rustls-native-certs", + "rustls 0.23.40", + "rustls-native-certs 0.8.3", "rustls-platform-verifier-android", - "rustls-webpki", - "security-framework", + "rustls-webpki 0.103.13", + "security-framework 3.7.0", "security-framework-sys", "webpki-root-certs", "windows-sys 0.61.2", @@ -8205,6 +8492,17 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" +[[package]] +name = "rustls-webpki" +version = "0.102.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.103.13" @@ -8293,6 +8591,19 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.11.1", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -8435,6 +8746,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_repr" version = "0.1.20" @@ -8679,6 +9001,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + [[package]] name = "spinning_top" version = "0.3.0" @@ -9264,13 +9595,24 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tokio-rustls" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +dependencies = [ + "rustls 0.22.4", + "rustls-pki-types", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls", + "rustls 0.23.40", "tokio", ] @@ -9285,6 +9627,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +dependencies = [ + "futures-util", + "log", + "rustls 0.23.40", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.4", + "tungstenite", + "webpki-roots 0.26.11", +] + [[package]] name = "tokio-util" version = "0.7.18" @@ -9435,6 +9793,7 @@ dependencies = [ "tokio", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -9553,6 +9912,26 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http 1.4.0", + "httparse", + "log", + "rand 0.8.6", + "rustls 0.23.40", + "rustls-pki-types", + "sha1", + "thiserror 1.0.69", + "utf-8", +] + [[package]] name = "twox-hash" version = "2.1.2" @@ -9661,6 +10040,12 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf8-ranges" version = "1.0.5" @@ -10337,6 +10722,15 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.7", +] + [[package]] name = "webpki-roots" version = "1.0.7" diff --git a/docs/streaming-sql/Source/README-zh.md b/docs/streaming-sql/Source/README-zh.md index e38a763f..77075fc3 100644 --- a/docs/streaming-sql/Source/README-zh.md +++ b/docs/streaming-sql/Source/README-zh.md @@ -1,7 +1,20 @@ # Streaming SQL Source 文档 -当前 Streaming SQL Source 仅支持 Kafka。通常先创建 Source 表，再被 `CREATE STREAMING TABLE ... AS SELECT ...` 引用。 +当前 Streaming SQL Source 支持 Kafka、MQTT、HTTP、ROS/ROS2、robot-bag 等。通常先创建 Source 表，再被 `CREATE STREAMING TABLE ... AS SELECT ...` 引用。 ## 文档列表 +- [机器人场景 Source 总览](robot-sources-zh.md) - [Kafka Source](kafka-source-zh.md) +- [MQTT Source](mqtt-source-zh.md) +- [HTTP Source](http-source-zh.md) +- [ROS Source](ros-source-zh.md) +- [ROS2 Source](ros2-source-zh.md) +- [Robot Bag Source（ROS1 bag / PCD / MCAP / ROS2）](robot-bag-source-zh.md) + +## 英文文档 + +- [Source README (EN)](README.md) +- [Robot sources overview (EN)](robot-sources.md) +- [Robot bag source (EN)](robot-bag-source.md) +- [Kafka source (EN)](kafka-source.md) diff --git a/docs/streaming-sql/Source/README.md b/docs/streaming-sql/Source/README.md index a84d3437..1b24f733 100644 --- a/docs/streaming-sql/Source/README.md +++ b/docs/streaming-sql/Source/README.md @@ -1,6 +1,22 @@ # Streaming SQL Source Docs -## Supported sources +Create a source table with `CREATE TABLE ... WITH (connector='...', type='source', ...)`, then reference it in `CREATE STREAMING TABLE ... AS SELECT ...`. -- [Kafka Source](kafka-source.md) +## Documentation +### Overview + +- [Robot & industrial data sources](robot-sources.md) + +### Connectors + +| Connector | Doc (EN) | Doc (中文) | +|-----------|----------|------------| +| Kafka | [kafka-source.md](kafka-source.md) | [kafka-source-zh.md](kafka-source-zh.md) | +| MQTT | — | [mqtt-source-zh.md](mqtt-source-zh.md) | +| HTTP | — | [http-source-zh.md](http-source-zh.md) | +| ROS (ROS1) | — | [ros-source-zh.md](ros-source-zh.md) | +| ROS2 | — | [ros2-source-zh.md](ros2-source-zh.md) | +| Robot bag | [robot-bag-source.md](robot-bag-source.md) | [robot-bag-source-zh.md](robot-bag-source-zh.md) | + +中文总览：[README-zh.md](README-zh.md) diff --git a/docs/streaming-sql/Source/http-source-zh.md b/docs/streaming-sql/Source/http-source-zh.md new file mode 100644 index 00000000..2ca18b5a --- /dev/null +++ b/docs/streaming-sql/Source/http-source-zh.md @@ -0,0 +1,87 @@ +# HTTP Source（机器人 / REST） + +`http` source 面向机器人遥测与 REST 接口，配置命名对齐 Flink HTTP / `rest-lookup` 连接器习惯。 + +## 模式 + +| 模式 | 场景 | 必填参数 | +|------|------|----------| +| **poll**（默认，有 `url` 时） | 定时拉取机器人 HTTP API | `url`、`format` | +| **webhook**（有 `http.listen.port` 或 `mode='webhook'`） | 机器人/边缘网关 **POST** 上报 | `http.listen.port`、`format` | + +## 支持格式 + +- `json` +- `raw_string` +- `raw_bytes` + +## Flink 对齐参数 + +| 参数 | Flink 对应 | 说明 | +|------|------------|------| +| `connector='http'` | `connector` | 固定为 `http` | +| `url` | `url` | Poll 模式请求地址 | +| `format` | `format` | 如 `json` | +| `lookup-method` / `http.method` | `lookup-method` | `GET`（默认）/ `POST` / `PUT` | +| `scan.interval` | — | 轮询周期，如 `INTERVAL '1' SECOND` | +| `scan.interval.ms` | — | 轮询周期毫秒（默认 1000） | +| `http.request.body` / `body` | `http.request.body-template` | POST 请求体 | +| `header.` | 自定义 Header | 如 `'header.Authorization'='Bearer xxx'` | +| `http.request.timeout.ms` | `gid.connector.http.source.lookup.request.timeout` | 请求超时（默认 30000） | +| `http.response.split` | — | `single`（默认）/ `ndjson` / `json-array` | +| `rate_limit.messages_per_second` | — | 默认不限速 | + +## Webhook 参数（机器人上报） + +| 参数 | 说明 | 默认 | +|------|------|------| +| `http.listen.host` | 监听地址 | `0.0.0.0` | +| `http.listen.port` | 监听端口 | 必填 | +| `http.webhook.path` | POST 路径 | `/` | +| `mode` | `poll` / `webhook` | 按 `url` / 端口自动推断 | + +## 示例：轮询机器人状态 API + +```sql +CREATE TABLE robot_status ( + robot_id STRING, + battery DOUBLE, + pose STRING, + ts TIMESTAMP +) WITH ( + connector='http', + type='source', + url='http://127.0.0.1:8080/api/v1/robots/status', + 'lookup-method'='GET', + 'scan.interval'='INTERVAL ''2'' SECOND', + format='json' +); +``` + +## 示例：Webhook 接收机器人上报 + +```sql +CREATE TABLE robot_telemetry ( + robot_id STRING, + joint_states STRING, + ts TIMESTAMP +) WITH ( + connector='http', + type='source', + mode='webhook', + 'http.listen.host'='0.0.0.0', + 'http.listen.port'='18080', + 'http.webhook.path'='/ingest/telemetry', + format='json' +); +``` + +机器人向 `POST http://:18080/ingest/telemetry` 发送 JSON 即可。 + +## 并行度 + +Poll / Webhook 均建议 **`parallelism = 1`**。Webhook 多 subtask 会重复绑定端口。 + +## Checkpoint + +v1 不持久化 HTTP 偏移；Poll 模式按 `scan.interval` 重复拉取；Webhook 依赖上游重试。 diff --git a/docs/streaming-sql/Source/kafka-source-zh.md b/docs/streaming-sql/Source/kafka-source-zh.md index e3da7597..ed83185f 100644 --- a/docs/streaming-sql/Source/kafka-source-zh.md +++ b/docs/streaming-sql/Source/kafka-source-zh.md @@ -14,10 +14,24 @@ - `type='source'`（默认） - `topic='topic_in'` - `bootstrap.servers='host:9092'` -- `group.id='consumer_group'` +- `group.id='consumer_group'`（可选；未指定时自动生成为 `fs-<表名>-consumer` 并写入 catalog） - `format='json'|'raw_string'|'raw_bytes'` - `scan.startup.mode='earliest'|'latest'|'group-offsets'`（可选） +## Consumer Group + +创建 Kafka Source 表时若未指定 `group.id` 且未指定 `group.id.prefix`，系统会在保存 catalog 前自动分配： + +```text +group.id = fs-<表名>-consumer +``` + +`SHOW CREATE TABLE` 与重启后 catalog 恢复均使用该值，保证 offset 按固定 consumer group 提交。若需每个 subtask 独立 group，可设置 `group.id.prefix`，由 runtime 追加 job/subtask 后缀。 + +## 吞吐说明 + +Kafka Source **不限速**；默认 batch=4096、linger=20ms、fetch 高吞吐参数已内置。 + ## 示例 ```sql @@ -30,7 +44,8 @@ CREATE TABLE src_kafka_json ( type='source', topic='topic_in', 'bootstrap.servers'='127.0.0.1:9092', - 'group.id'='fs_demo', format='json' ); ``` + +上例未写 `group.id`，catalog 中会自动保存 `'group.id'='fs-src_kafka_json-consumer'`。 diff --git a/docs/streaming-sql/Source/kafka-source.md b/docs/streaming-sql/Source/kafka-source.md index 505a6f3e..b3dc5cf2 100644 --- a/docs/streaming-sql/Source/kafka-source.md +++ b/docs/streaming-sql/Source/kafka-source.md @@ -13,7 +13,35 @@ - `connector='kafka'` - `topic='topic_name'` - `bootstrap.servers='host:9092'` +- `group.id='consumer_group'` (optional; if omitted, defaults to `fs--consumer` and is persisted in the catalog) - `format='json'|'raw_string'|'raw_bytes'` -- `scan.startup.mode='earliest'|'latest'` (optional) -- `group.id='consumer_group'` (optional) +- `scan.startup.mode='earliest'|'latest'|'group-offsets'` (optional) + +## Consumer group + +When you create a Kafka source table without `group.id` and without `group.id.prefix`, the planner assigns before catalog save: + +```text +group.id = fs--consumer +``` + +`SHOW CREATE TABLE` and catalog reload after restart use this value so offsets commit under a stable consumer group. For per-subtask groups, set `group.id.prefix` instead; the runtime appends job/subtask suffixes. + +## Example + +```sql +CREATE TABLE src_kafka_json ( + user_id BIGINT, + event STRING, + ts TIMESTAMP +) WITH ( + connector='kafka', + type='source', + topic='topic_in', + 'bootstrap.servers'='127.0.0.1:9092', + format='json' +); +``` + +If `group.id` is omitted, the catalog stores `'group.id'='fs-src_kafka_json-consumer'`. diff --git a/docs/streaming-sql/Source/mqtt-source-zh.md b/docs/streaming-sql/Source/mqtt-source-zh.md new file mode 100644 index 00000000..a6d48a19 --- /dev/null +++ b/docs/streaming-sql/Source/mqtt-source-zh.md @@ -0,0 +1,49 @@ +# MQTT Source + +`mqtt` source 用于从 MQTT broker 订阅 topic 并持续消费消息。 + +## 支持格式 + +- `json` +- `raw_string` +- `raw_bytes` + +## 常用 WITH 参数 + +- `connector='mqtt'` +- `type='source'`（默认） +- `topic='sensors/#'` +- `mqtt.host='127.0.0.1'`（或 `host`） +- `mqtt.port='1883'`（或 `port`，默认 1883） +- `format='json'|'raw_string'|'raw_bytes'` +- `mqtt.client.id`（可选；未设置时运行时生成 `fs-{job_id}-{subtask}-mqtt`） +- `mqtt.username` / `mqtt.password`（可选） +- `mqtt.qos`（可选，0/1/2，默认 1） +- `mqtt.clean.session`（可选，默认 `true`） +- `mqtt.keep.alive.secs`（可选，默认 60） +- `rate_limit.messages_per_second`（可选）：**默认不配置 = 不限速** + +## 并行度 + +MQTT 订阅在多个 subtask 上会重复消费同一 topic，**建议 pipeline `parallelism = 1`**。 + +## Checkpoint + +v1 不向 catalog 持久化 MQTT 偏移；故障恢复依赖 broker 会话与 `client_id` / `clean_session` 配置（至少一次语义）。 + +## 示例 + +```sql +CREATE TABLE src_mqtt_json ( + device_id STRING, + value DOUBLE, + ts TIMESTAMP +) WITH ( + connector='mqtt', + type='source', + topic='sensors/data', + 'mqtt.host'='127.0.0.1', + 'mqtt.port'='1883', + format='json' +); +``` diff --git a/docs/streaming-sql/Source/robot-bag-source-zh.md b/docs/streaming-sql/Source/robot-bag-source-zh.md new file mode 100644 index 00000000..22c6a7d1 --- /dev/null +++ b/docs/streaming-sql/Source/robot-bag-source-zh.md @@ -0,0 +1,151 @@ +# Robot Bag Source（机器人录制文件） + +`robot-bag` source 用于回放机器人离线数据，支持多种工业常见文件格式。 + +## 支持格式 + +| `bag.format` | 文件类型 | 说明 | +|--------------|----------|------| +| `auto` | 按扩展名/魔数推断 | 默认 | +| `mcap` | `.mcap` | Foxglove / MCAP 录制 | +| `ros1` | `.bag` | ROS1 rosbag（v1.02 / v2.0，含 chunk/index） | +| `ros2` | `.db3` 或目录 | ROS2 rosbag2（SQLite） | +| `pcd` | `.pcd` 或目录 | PCL PCD v0.7（ascii / binary / binary_compressed） | +| `jsonl` | `.jsonl` / `.ndjson` | 每行一条 JSON | +| `csv` | `.csv` | 跳过表头，每行一条记录 | + +消息体解析格式由 `format` 指定（通常 `json` 或 `raw_bytes`）。 + +## 常用 WITH 参数 + +| 参数 | 说明 | 默认 | +|------|------|------| +| `connector='robot-bag'` | 固定 | — | +| `path` | 文件或目录 | 必填 | +| `bag.format` | 见上表 | `auto` | +| `topic` | MCAP/ROS1/ROS2 按 topic 过滤 | 全部 | +| `pcd.emit.mode` | PCD：`point`（每点一条）或 `cloud`（每文件一条） | `point` | +| `replay.interval.ms` | 两条消息间隔（0=尽快） | `0` | +| `loop` | 播完后是否循环 | `false` | +| `format` | 消息解码格式 | 必填 | + +## 自动识别（`bag.format='auto'`） + +- **目录**含 `.pcd` → PCD +- **目录**无 `.pcd` → ROS2 bag +- **文件**头为 `#ROSBAG V2.0` / `#ROSBAG V1.02` → ROS1 +- **扩展名**：`.mcap`、`.bag`、`.db3`、`.pcd`、`.jsonl`、`.csv` + +## 示例：ROS1 bag + +```sql +CREATE TABLE robot_ros1_bag ( + payload BYTES, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/data/lidar_run.bag', + 'bag.format'='ros1', + topic='/velodyne_points', + format='raw_bytes' +); +``` + +ROS1 bag 通过标准 `#ROSBAG V2.0` 头解析 chunk/index，按时间戳排序回放。`sensor_msgs/PointCloud2` 等二进制消息请使用 `format='raw_bytes'`。 + +## 示例：PCD 点云（逐点） + +```sql +CREATE TABLE lidar_points ( + x DOUBLE, + y DOUBLE, + z DOUBLE, + intensity DOUBLE, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/data/scans/', + 'bag.format'='pcd', + 'pcd.emit.mode'='point', + format='json' +); +``` + +每个点输出一条 JSON，字段名来自 PCD header（`x`、`y`、`z`、`intensity` 等），并附带 `_source_file`、`_point_index`。PCD 解析遵循 PCL v0.7，支持 `ascii`、`binary`、`binary_compressed`（LZF + SoA→AoS 重排）。目录模式下按文件名排序依次加载所有 `.pcd`。 + +## 示例：PCD 整云（单条 JSON） + +```sql +CREATE TABLE lidar_cloud ( + payload STRING, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/data/scan.pcd', + 'bag.format'='pcd', + 'pcd.emit.mode'='cloud', + format='json' +); +``` + +## 示例：MCAP + +```sql +CREATE TABLE robot_mcap ( + robot_id STRING, + pose STRING, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/data/robot_run.mcap', + 'bag.format'='mcap', + topic='/odom', + format='json' +); +``` + +## 示例：ROS2 bag 目录 + +```sql +CREATE TABLE robot_ros2_bag ( + joint_name STRING, + position DOUBLE, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/data/rosbag2_2024', + 'bag.format'='ros2', + topic='/joint_states', + format='raw_bytes' +); +``` + +## 示例：JSONL 循环回放 + +```sql +CREATE TABLE robot_jsonl ( + event STRING, + payload STRING, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/logs/robot_events.jsonl', + 'bag.format'='jsonl', + format='json', + loop='true' +); +``` + +## 回放完成后 + +`loop=false` 时读完文件先 flush 缓冲区，再发送 `EndOfStream`；`loop=true` 时重新加载文件循环播放。可用 `replay.interval.ms` 模拟实时播放间隔。 + +## 并行度 + +建议 **`parallelism = 1`**。 diff --git a/docs/streaming-sql/Source/robot-bag-source.md b/docs/streaming-sql/Source/robot-bag-source.md new file mode 100644 index 00000000..d2cf36b2 --- /dev/null +++ b/docs/streaming-sql/Source/robot-bag-source.md @@ -0,0 +1,153 @@ +# Robot Bag Source + +The `robot-bag` source replays offline robot recordings from industrial file formats. + +## Supported file formats + +| `bag.format` | File type | Description | +|--------------|-----------|-------------| +| `auto` | Extension / magic bytes | Default | +| `mcap` | `.mcap` | Foxglove / MCAP recordings | +| `ros1` | `.bag` | ROS1 rosbag (v1.02 / v2.0, chunk + index) | +| `ros2` | `.db3` or directory | ROS2 rosbag2 (SQLite) | +| `pcd` | `.pcd` or directory | PCL PCD v0.7 (`ascii` / `binary` / `binary_compressed`) | +| `jsonl` | `.jsonl` / `.ndjson` | One JSON object per line | +| `csv` | `.csv` | Header row skipped, one record per line | + +Payload decoding is controlled by `format` (typically `json` or `raw_bytes`). + +## Common `WITH` options + +| Option | Description | Default | +|--------|-------------|---------| +| `connector='robot-bag'` | Required | — | +| `path` | File or directory | Required | +| `bag.format` | See table above | `auto` | +| `topic` | Filter by topic (MCAP / ROS1 / ROS2) | All topics | +| `pcd.emit.mode` | PCD: `point` (one row per point) or `cloud` (one row per file) | `point` | +| `replay.interval.ms` | Delay between messages (`0` = as fast as possible) | `0` | +| `loop` | Reload file after EOF | `false` | +| `format` | Payload format (`json` / `raw_string` / `raw_bytes`) | Required | + +## Auto-detection + +When `bag.format='auto'`: + +- **Directory** with `.pcd` files → PCD +- **Directory** without `.pcd` → ROS2 bag +- **File** starting with `#ROSBAG V2.0` / `#ROSBAG V1.02` → ROS1 +- **Extension**: `.mcap`, `.bag`, `.db3`, `.pcd`, `.jsonl`, `.csv` + +## Example: ROS1 bag + +```sql +CREATE TABLE robot_ros1_bag ( + payload BYTES, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/data/lidar_run.bag', + 'bag.format'='ros1', + topic='/velodyne_points', + format='raw_bytes' +); +``` + +ROS1 bags are parsed via chunk/index records and replayed in timestamp order. Use `format='raw_bytes'` for binary messages such as `sensor_msgs/PointCloud2`. + +## Example: PCD point cloud (per point) + +```sql +CREATE TABLE lidar_points ( + x DOUBLE, + y DOUBLE, + z DOUBLE, + intensity DOUBLE, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/data/scans/', + 'bag.format'='pcd', + 'pcd.emit.mode'='point', + format='json' +); +``` + +Each point is emitted as a JSON object with field names from the PCD header (`x`, `y`, `z`, `intensity`, etc.) plus `_source_file` and `_point_index`. PCL v0.7 `binary_compressed` uses LZF decompression and SoA→AoS reordering. In directory mode, all `.pcd` files are loaded in sorted filename order. + +## Example: PCD whole cloud (single JSON row) + +```sql +CREATE TABLE lidar_cloud ( + payload STRING, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/data/scan.pcd', + 'bag.format'='pcd', + 'pcd.emit.mode'='cloud', + format='json' +); +``` + +## Example: MCAP + +```sql +CREATE TABLE robot_mcap ( + robot_id STRING, + pose STRING, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/data/robot_run.mcap', + 'bag.format'='mcap', + topic='/odom', + format='json' +); +``` + +## Example: ROS2 bag directory + +```sql +CREATE TABLE robot_ros2_bag ( + joint_name STRING, + position DOUBLE, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/data/rosbag2_2024', + 'bag.format'='ros2', + topic='/joint_states', + format='raw_bytes' +); +``` + +## Example: JSONL log with loop replay + +```sql +CREATE TABLE robot_jsonl ( + event STRING, + payload STRING, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/logs/robot_events.jsonl', + 'bag.format'='jsonl', + format='json', + loop='true' +); +``` + +## End of replay + +When `loop=false`, the source emits `EndOfStream` after the last message (buffered batches are flushed first). When `loop=true`, the file is reloaded and replay starts again. + +## Parallelism + +Use **`parallelism = 1`**. File replay is single-threaded and not partitionable. diff --git a/docs/streaming-sql/Source/robot-sources-zh.md b/docs/streaming-sql/Source/robot-sources-zh.md new file mode 100644 index 00000000..0c50934f --- /dev/null +++ b/docs/streaming-sql/Source/robot-sources-zh.md @@ -0,0 +1,98 @@ +# 机器人场景 Source 指南 + +面向移动机器人、机械臂、AGV 等设备的流式接入说明。配置风格参考 **Apache Flink** 连接器（`url`、`format`、`scan.interval`、`lookup-method` 等）。 + +## 已支持 Source + +| Connector | 典型场景 | 文档 | +|-----------|----------|------| +| **kafka** | 经 ROS2/网关桥接后的汇聚 topic、平台总线 | [kafka-source-zh.md](kafka-source-zh.md) | +| **mqtt** | 设备直连、边缘 MQTT Broker、IoT 规则引擎 | [mqtt-source-zh.md](mqtt-source-zh.md) | +| **http** | REST 轮询机器人 API；Webhook 接收 POST 遥测 | [http-source-zh.md](http-source-zh.md) | +| **ros** | ROS1 topic（rosbridge WebSocket） | [ros-source-zh.md](ros-source-zh.md) | +| **ros2** | ROS2 topic（rosbridge_server） | [ros2-source-zh.md](ros2-source-zh.md) | +| **robot-bag** | MCAP / ROS1 bag / ROS2 bag / PCD / JSONL / CSV 离线回放 | [robot-bag-source-zh.md](robot-bag-source-zh.md) | + +## 推荐选型 + +```text +机器人 ──MQTT──► Broker ──► function-stream (mqtt source) +机器人 ──HTTP POST──► function-stream (http webhook) +调度系统 ──HTTP GET API──► function-stream (http poll) +ROS/ROS2 ──rosbridge──► function-stream (ros / ros2 source) +离线录制 ──bag/MCAP/PCD──► function-stream (robot-bag source) +``` + +| 数据形态 | 推荐 Connector | 说明 | +|----------|----------------|------| +| 高频遥测、弱网重传 | `mqtt` | QoS 1/2、`client_id` 稳定 | +| 云平台/第三方 REST 回调 | `http` webhook | `http.listen.port` + `http.webhook.path` | +| 定时查询机器人状态 | `http` poll | `url` + `scan.interval` | +| ROS/ROS2 实时 topic | `ros` / `ros2` | rosbridge WebSocket | +| 离线 bag / MCAP / PCD / 日志 | `robot-bag` | `path` + `bag.format` | +| 已有 Kafka 管线 | `kafka` | `group.id`、`scan.startup.mode` | + +## robot-bag 格式对照 + +| `bag.format` | 输入 | topic 过滤 | 推荐 `format` | +|--------------|------|------------|---------------| +| `ros1` | `.bag` 文件 | 支持 | 二进制 ROS 消息用 `raw_bytes` | +| `ros2` | `.db3` 或目录 | 支持 | `raw_bytes` | +| `mcap` | `.mcap` 文件 | 支持 | `json` 或 `raw_bytes` | +| `pcd` | `.pcd` 或目录 | 不支持 | `json`（`pcd.emit.mode=point` 或 `cloud`） | +| `jsonl` | `.jsonl` | 不支持 | `json` | +| `csv` | `.csv` | 不支持 | `json` 或 `raw_string` | + +英文文档：[robot-sources.md](robot-sources.md) · [robot-bag-source.md](robot-bag-source.md) + +## 规划中（常量已预留，待实现） + +| Connector | 机器人场景 | Flink / 业界参考 | +|-----------|------------|------------------| +| `websocket` | 实时关节流、仿真器推送 | WebSocket Table Source | +| `nats` | 边缘集群、JetStream | NATS connector | +| `pulsar` | 多租户机器人云平台 | Pulsar source | +| `redis` | 设备在线状态、Stream | Redis Streams | +| `sse` | 浏览器/轻量网关 Server-Sent Events | — | + +工业现场 **Modbus / OPC-UA** 建议经边缘网关转为 MQTT/HTTP 后接入，不在引擎内直接实现现场总线。 + +## 通用约定 + +1. **`format='json'`** 最常用；单字段二进制可用 `raw_bytes`。 +2. **`parallelism = 1`**：MQTT 订阅、HTTP Webhook 端口、HTTP 单 URL 轮询均避免重复消费。 +3. **吞吐**：Source 默认 batch=4096、linger=20ms；Kafka **无限速**；Delta 缓冲默认 256MB 并合并小 batch 写 Parquet。 +4. **Checkpoint**：Kafka 支持分区 offset；MQTT/HTTP v1 为至少一次，依赖 broker 会话或轮询间隔。 + +## 端到端示例（MQTT + Delta） + +```sql +-- 1. 机器人经 MQTT 上报 +CREATE TABLE robot_events ( + robot_id STRING, + event_type STRING, + payload STRING, + ts TIMESTAMP +) WITH ( + connector='mqtt', + type='source', + topic='factory/robot/+/telemetry', + 'mqtt.host'='127.0.0.1', + 'mqtt.port'='1883', + format='json' +); + +-- 2. 流式作业（示意） +CREATE STREAMING TABLE robot_sink +WITH ( + connector='delta', + type='sink', + path='/data/robot', + format='parquet' +) AS +SELECT * FROM robot_events; +``` + +## 添加新 Source 的开发清单 + +见项目内 MQTT/HTTP 实现路径：Proto `XxxSourceConfig` → Planner `SourceProvider` → Runtime `SourceOperator` → `ConnectorSourceDispatcher` 注册。 diff --git a/docs/streaming-sql/Source/robot-sources.md b/docs/streaming-sql/Source/robot-sources.md new file mode 100644 index 00000000..9b3ef633 --- /dev/null +++ b/docs/streaming-sql/Source/robot-sources.md @@ -0,0 +1,79 @@ +# Robot & Industrial Data Sources + +Guide for mobile robots, manipulators, AGV, and LiDAR pipelines. Connector options follow a Flink-style `WITH` clause (`url`, `format`, `scan.interval`, etc.). + +## Supported sources + +| Connector | Typical use | Doc | +|-----------|-------------|-----| +| **kafka** | Aggregated topics via ROS2 bridge / platform bus | [kafka-source.md](kafka-source.md) | +| **mqtt** | Device MQTT, edge broker, IoT rules | [mqtt-source-zh.md](mqtt-source-zh.md) | +| **http** | REST poll or Webhook telemetry | [http-source-zh.md](http-source-zh.md) | +| **ros** | ROS1 topics via rosbridge WebSocket | [ros-source-zh.md](ros-source-zh.md) | +| **ros2** | ROS2 topics via rosbridge_server | [ros2-source-zh.md](ros2-source-zh.md) | +| **robot-bag** | Offline MCAP / ROS1 bag / ROS2 bag / PCD / JSONL / CSV | [robot-bag-source.md](robot-bag-source.md) | + +## Selection guide + +```text +Robot ──MQTT──► Broker ──► function-stream (mqtt source) +Robot ──HTTP POST──► function-stream (http webhook) +Scheduler ──HTTP GET API──► function-stream (http poll) +ROS/ROS2 ──rosbridge──► function-stream (ros / ros2 source) +Offline recording ──bag/MCAP/PCD──► function-stream (robot-bag source) +``` + +| Data shape | Connector | Notes | +|------------|-----------|-------| +| High-frequency telemetry, weak network | `mqtt` | QoS 1/2, stable `client_id` | +| Cloud REST callback | `http` webhook | `http.listen.port` + `http.webhook.path` | +| Periodic robot status poll | `http` poll | `url` + `scan.interval` | +| Live ROS/ROS2 topic | `ros` / `ros2` | rosbridge WebSocket | +| Offline bag / MCAP / PCD / logs | `robot-bag` | `path` + `bag.format` | +| Existing Kafka pipeline | `kafka` | `group.id`, `scan.startup.mode` | + +## robot-bag format matrix + +| `bag.format` | Input | Topic filter | Typical `format` | +|--------------|-------|--------------|------------------| +| `ros1` | `.bag` file | Yes | `raw_bytes` for CDR/binary ROS messages | +| `ros2` | `.db3` or directory | Yes | `raw_bytes` | +| `mcap` | `.mcap` file | Yes | `json` or `raw_bytes` | +| `pcd` | `.pcd` or directory | No | `json` (`pcd.emit.mode=point` or `cloud`) | +| `jsonl` | `.jsonl` | No | `json` | +| `csv` | `.csv` | No | `json` or `raw_string` | + +## Conventions + +1. Prefer **`format='json'`** for structured telemetry; use **`raw_bytes`** for ROS binary payloads. +2. Set **`parallelism = 1`** for MQTT, HTTP webhook, HTTP poll, rosbridge, and robot-bag sources. +3. Kafka source has **no rate limit**; default batch=4096, linger=20ms. +4. robot-bag replays files once unless `loop=true`; use `replay.interval.ms` to simulate real-time playback. + +## End-to-end example (robot-bag PCD → Delta) + +```sql +CREATE TABLE lidar_scan ( + x DOUBLE, + y DOUBLE, + z DOUBLE, + intensity DOUBLE, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/data/lidar_frames/', + 'bag.format'='pcd', + 'pcd.emit.mode'='point', + format='json' +); + +CREATE STREAMING TABLE lidar_delta +WITH ( + connector='delta', + type='sink', + path='/data/lidar_out', + format='parquet' +) AS +SELECT x, y, z, intensity, ts FROM lidar_scan; +``` diff --git a/docs/streaming-sql/Source/ros-source-zh.md b/docs/streaming-sql/Source/ros-source-zh.md new file mode 100644 index 00000000..8d62b5ec --- /dev/null +++ b/docs/streaming-sql/Source/ros-source-zh.md @@ -0,0 +1,41 @@ +# ROS Source（rosbridge / ROS1） + +`ros` source 通过 [rosbridge_suite](https://github.com/RobotWebTools/rosbridge_suite) WebSocket 订阅 ROS topic。 + +ROS2 请使用 [`ros2`](ros2-source-zh.md) connector 或设置 `'ros.version'='ros2'`。 + +## 支持格式 + +- `json` +- `raw_string` / `raw_bytes` + +## 常用 WITH 参数 + +| 参数 | 说明 | 默认 | +|------|------|------| +| `connector='ros'` | 固定 | — | +| `topic` | ROS topic | 必填 | +| `url` | rosbridge WebSocket | `ws://127.0.0.1:9090` | +| `ros.message.field` | JSON 载荷字段 | `msg` | +| `format` | 如 `json` | 必填 | + +## 示例 + +```sql +CREATE TABLE ros1_chatter ( + data STRING, + ts TIMESTAMP +) WITH ( + connector='ros', + type='source', + topic='/chatter', + url='ws://127.0.0.1:9090', + format='json' +); +``` + +## 相关文档 + +- [ROS2 Source](ros2-source-zh.md) +- [Robot Bag 文件 Source](robot-bag-source-zh.md) +- [机器人 Source 总览](robot-sources-zh.md) diff --git a/docs/streaming-sql/Source/ros2-source-zh.md b/docs/streaming-sql/Source/ros2-source-zh.md new file mode 100644 index 00000000..39e567e7 --- /dev/null +++ b/docs/streaming-sql/Source/ros2-source-zh.md @@ -0,0 +1,58 @@ +# ROS2 Source（rosbridge） + +`ros2` source 通过 **rosbridge_suite** WebSocket 订阅 ROS2 topic，协议与 `ros` source 相同，默认按 ROS2 版本注册。 + +## 前置条件 + +ROS2 环境安装并启动 rosbridge： + +```bash +ros2 launch rosbridge_server rosbridge_websocket_launch.xml +``` + +默认 WebSocket：`ws://127.0.0.1:9090` + +## 支持格式 + +- `json`（推荐） +- `raw_string` / `raw_bytes` + +## 常用 WITH 参数 + +| 参数 | 说明 | 默认 | +|------|------|------| +| `connector='ros2'` | 固定 | — | +| `topic` | ROS2 topic，如 `/joint_states` | 必填 | +| `url` / `ros.url` | rosbridge 地址 | `ws://127.0.0.1:9090` | +| `ros.message.field` | publish 帧中的 JSON 字段 | `msg` | +| `format` | 如 `json` | 必填 | + +## 示例 + +```sql +CREATE TABLE ros2_joints ( + name STRING, + position STRING, + ts TIMESTAMP +) WITH ( + connector='ros2', + type='source', + topic='/joint_states', + url='ws://127.0.0.1:9090', + format='json' +); +``` + +## ROS1 与 ROS2 区别 + +| 项 | `ros` | `ros2` | +|----|-------|--------| +| 默认 `ros.version` | ros1 | ros2 | +| 桥接服务 | rosbridge (ROS1) | rosbridge_server (ROS2) | +| WebSocket 协议 | 相同 | 相同 | + +也可在 `connector='ros'` 时显式设置 `'ros.version'='ros2'`。 + +## 并行度 + +建议 **`parallelism = 1`**。 diff --git a/protocol/proto/function_stream_graph.proto b/protocol/proto/function_stream_graph.proto index 18903e4f..b726af6f 100644 --- a/protocol/proto/function_stream_graph.proto +++ b/protocol/proto/function_stream_graph.proto @@ -75,9 +75,103 @@ message ConnectorOp { IcebergSinkConfig iceberg_sink = 11; S3SinkConfig s3_sink = 12; LanceDbSinkConfig lancedb_sink = 13; + MqttSourceConfig mqtt_source = 14; + HttpSourceConfig http_source = 15; + RosSourceConfig ros_source = 16; + RobotBagSourceConfig robot_bag_source = 17; } } +// ─────────────────────── ROS / rosbridge Source ─────────────────────── + +enum RosVersion { + ROS_VERSION_ROS1 = 0; + ROS_VERSION_ROS2 = 1; +} + +message RosSourceConfig { + string url = 1; + string topic = 2; + string message_field = 3; + FormatConfig format = 4; + BadDataPolicy bad_data_policy = 5; + map client_configs = 6; + RosVersion ros_version = 7; +} + +// ─────────────────────── Robot bag / file Source ─────────────────────── + +enum RobotBagFormat { + ROBOT_BAG_AUTO = 0; + ROBOT_BAG_MCAP = 1; + ROBOT_BAG_ROS2 = 2; + ROBOT_BAG_JSONL = 3; + ROBOT_BAG_CSV = 4; + ROBOT_BAG_ROS1 = 5; + ROBOT_BAG_PCD = 6; +} + +message RobotBagSourceConfig { + string path = 1; + RobotBagFormat bag_format = 2; + optional string topic = 3; + uint64 replay_interval_ms = 4; + bool loop_replay = 5; + FormatConfig format = 6; + BadDataPolicy bad_data_policy = 7; + map client_configs = 8; + string pcd_emit_mode = 9; +} + +// ─────────────────────── HTTP Connector Configs (robot / REST) ─────────────────────── + +enum HttpSourceMode { + HTTP_SOURCE_POLL = 0; + HTTP_SOURCE_WEBHOOK = 1; +} + +enum HttpResponseSplit { + HTTP_RESPONSE_SINGLE = 0; + HTTP_RESPONSE_NDJSON = 1; + HTTP_RESPONSE_JSON_ARRAY = 2; +} + +message HttpSourceConfig { + HttpSourceMode mode = 1; + string url = 2; + string method = 3; + uint64 scan_interval_ms = 4; + optional string request_body = 5; + map headers = 6; + string listen_host = 7; + uint32 listen_port = 8; + string webhook_path = 9; + FormatConfig format = 10; + BadDataPolicy bad_data_policy = 11; + uint32 rate_limit_msgs_per_sec = 12; + map client_configs = 13; + uint32 request_timeout_ms = 14; + HttpResponseSplit response_split = 15; +} + +// ─────────────────────── MQTT Connector Configs ─────────────────────── + +message MqttSourceConfig { + string topic = 1; + string host = 2; + uint32 port = 3; + optional string client_id = 4; + optional string username = 5; + optional string password = 6; + uint32 qos = 7; + bool clean_session = 8; + uint32 keep_alive_secs = 9; + FormatConfig format = 10; + BadDataPolicy bad_data_policy = 11; + uint32 rate_limit_msgs_per_sec = 12; + map client_configs = 13; +} + // ─────────────────────── Kafka Connector Configs ─────────────────────── message KafkaSourceConfig { diff --git a/src/streaming_planner/src/common/constants.rs b/src/streaming_planner/src/common/constants.rs index 8cdb68e3..467247b1 100644 --- a/src/streaming_planner/src/common/constants.rs +++ b/src/streaming_planner/src/common/constants.rs @@ -257,6 +257,10 @@ pub mod connector_type { pub const NATS: &str = "nats"; pub const REDIS: &str = "redis"; pub const MQTT: &str = "mqtt"; + pub const HTTP: &str = "http"; + pub const ROS: &str = "ros"; + pub const ROS2: &str = "ros2"; + pub const ROBOT_BAG: &str = "robot-bag"; pub const WEBSOCKET: &str = "websocket"; pub const SSE: &str = "sse"; pub const NEXMARK: &str = "nexmark"; @@ -273,6 +277,11 @@ pub mod connection_table_role { pub const SUPPORTED_CONNECTOR_ADAPTERS: &[&str] = &[ connector_type::KAFKA, + connector_type::MQTT, + connector_type::HTTP, + connector_type::ROS, + connector_type::ROS2, + connector_type::ROBOT_BAG, connector_type::FILESYSTEM, connector_type::S3, connector_type::DELTA, diff --git a/src/streaming_planner/src/common/with_option_keys.rs b/src/streaming_planner/src/common/with_option_keys.rs index 8a8ebd27..dbb21f67 100644 --- a/src/streaming_planner/src/common/with_option_keys.rs +++ b/src/streaming_planner/src/common/with_option_keys.rs @@ -50,6 +50,64 @@ pub const KAFKA_KEY_FIELD_LEGACY: &str = "key.field"; pub const KAFKA_SINK_TIMESTAMP_FIELD: &str = "sink.timestamp.field"; pub const KAFKA_TIMESTAMP_FIELD_LEGACY: &str = "timestamp.field"; +// ── MQTT ────────────────────────────────────────────────────────────────── + +pub const MQTT_HOST: &str = "mqtt.host"; +pub const MQTT_HOST_LEGACY: &str = "host"; +pub const MQTT_PORT: &str = "mqtt.port"; +pub const MQTT_PORT_LEGACY: &str = "port"; +pub const MQTT_TOPIC: &str = "topic"; +pub const MQTT_CLIENT_ID: &str = "mqtt.client.id"; +pub const MQTT_CLIENT_ID_LEGACY: &str = "client.id"; +pub const MQTT_USERNAME: &str = "mqtt.username"; +pub const MQTT_USERNAME_LEGACY: &str = "username"; +pub const MQTT_PASSWORD: &str = "mqtt.password"; +pub const MQTT_PASSWORD_LEGACY: &str = "password"; +pub const MQTT_QOS: &str = "mqtt.qos"; +pub const MQTT_QOS_LEGACY: &str = "qos"; +pub const MQTT_CLEAN_SESSION: &str = "mqtt.clean.session"; +pub const MQTT_CLEAN_SESSION_LEGACY: &str = "clean.session"; +pub const MQTT_KEEP_ALIVE_SECS: &str = "mqtt.keep.alive.secs"; +pub const MQTT_KEEP_ALIVE_SECS_LEGACY: &str = "keep.alive.secs"; +pub const MQTT_RATE_LIMIT_MESSAGES_PER_SECOND: &str = "rate_limit.messages_per_second"; + +// ── HTTP (Flink rest-lookup / robot REST aligned) ───────────────────────── + +pub const HTTP_MODE: &str = "mode"; +pub const HTTP_URL: &str = "url"; +pub const HTTP_METHOD: &str = "http.method"; +pub const HTTP_LOOKUP_METHOD: &str = "lookup-method"; +pub const HTTP_SCAN_INTERVAL: &str = "scan.interval"; +pub const HTTP_SCAN_INTERVAL_MS: &str = "scan.interval.ms"; +pub const HTTP_POLL_INTERVAL_MS: &str = "http.poll.interval.ms"; +pub const HTTP_REQUEST_BODY: &str = "http.request.body"; +pub const HTTP_REQUEST_BODY_LEGACY: &str = "body"; +pub const HTTP_LISTEN_HOST: &str = "http.listen.host"; +pub const HTTP_LISTEN_PORT: &str = "http.listen.port"; +pub const HTTP_WEBHOOK_PATH: &str = "http.webhook.path"; +pub const HTTP_REQUEST_TIMEOUT_MS: &str = "http.request.timeout.ms"; +pub const HTTP_RESPONSE_SPLIT: &str = "http.response.split"; +pub const HTTP_HEADER_PREFIX: &str = "header."; + +// ── ROS / rosbridge ─────────────────────────────────────────────────────── + +pub const ROS_URL: &str = "url"; +pub const ROS_URL_LEGACY: &str = "ros.url"; +pub const ROS_TOPIC: &str = "topic"; +pub const ROS_TOPIC_LEGACY: &str = "ros.topic"; +pub const ROS_MESSAGE_FIELD: &str = "ros.message.field"; +pub const ROS_MESSAGE_FIELD_LEGACY: &str = "message.field"; +pub const ROS_VERSION: &str = "ros.version"; + +// ── Robot bag / file formats ────────────────────────────────────────────── + +pub const ROBOT_BAG_PATH: &str = "path"; +pub const ROBOT_BAG_FORMAT: &str = "bag.format"; +pub const ROBOT_BAG_TOPIC: &str = "topic"; +pub const ROBOT_BAG_REPLAY_INTERVAL_MS: &str = "replay.interval.ms"; +pub const ROBOT_BAG_LOOP: &str = "loop"; +pub const ROBOT_BAG_PCD_EMIT_MODE: &str = "pcd.emit.mode"; + // ── JSON format ─────────────────────────────────────────────────────────── pub const JSON_CONFLUENT_SCHEMA_REGISTRY: &str = "json.confluent_schema_registry"; diff --git a/src/streaming_planner/src/connector/config.rs b/src/streaming_planner/src/connector/config.rs index c3eaf00f..4d4c473b 100644 --- a/src/streaming_planner/src/connector/config.rs +++ b/src/streaming_planner/src/connector/config.rs @@ -12,12 +12,17 @@ use protocol::function_stream_graph::{ DeltaSinkConfig, FilesystemSinkConfig, IcebergSinkConfig, KafkaSinkConfig, KafkaSourceConfig, - LanceDbSinkConfig, S3SinkConfig, connector_op, + HttpSourceConfig, LanceDbSinkConfig, MqttSourceConfig, RobotBagSourceConfig, + RosSourceConfig, S3SinkConfig, connector_op, }; #[derive(Debug, Clone)] pub enum ConnectorConfig { KafkaSource(KafkaSourceConfig), + MqttSource(MqttSourceConfig), + HttpSource(HttpSourceConfig), + RosSource(RosSourceConfig), + RobotBagSource(RobotBagSourceConfig), KafkaSink(KafkaSinkConfig), FilesystemSink(FilesystemSinkConfig), DeltaSink(DeltaSinkConfig), @@ -30,6 +35,12 @@ impl ConnectorConfig { pub fn to_proto_config(&self) -> connector_op::Config { match self { ConnectorConfig::KafkaSource(cfg) => connector_op::Config::KafkaSource(cfg.clone()), + ConnectorConfig::MqttSource(cfg) => connector_op::Config::MqttSource(cfg.clone()), + ConnectorConfig::HttpSource(cfg) => connector_op::Config::HttpSource(cfg.clone()), + ConnectorConfig::RosSource(cfg) => connector_op::Config::RosSource(cfg.clone()), + ConnectorConfig::RobotBagSource(cfg) => { + connector_op::Config::RobotBagSource(cfg.clone()) + } ConnectorConfig::KafkaSink(cfg) => connector_op::Config::KafkaSink(cfg.clone()), ConnectorConfig::FilesystemSink(cfg) => { connector_op::Config::FilesystemSink(cfg.clone()) @@ -49,6 +60,18 @@ impl PartialEq for ConnectorConfig { (ConnectorConfig::KafkaSource(a), ConnectorConfig::KafkaSource(b)) => { a.encode_to_vec() == b.encode_to_vec() } + (ConnectorConfig::MqttSource(a), ConnectorConfig::MqttSource(b)) => { + a.encode_to_vec() == b.encode_to_vec() + } + (ConnectorConfig::HttpSource(a), ConnectorConfig::HttpSource(b)) => { + a.encode_to_vec() == b.encode_to_vec() + } + (ConnectorConfig::RosSource(a), ConnectorConfig::RosSource(b)) => { + a.encode_to_vec() == b.encode_to_vec() + } + (ConnectorConfig::RobotBagSource(a), ConnectorConfig::RobotBagSource(b)) => { + a.encode_to_vec() == b.encode_to_vec() + } (ConnectorConfig::KafkaSink(a), ConnectorConfig::KafkaSink(b)) => { a.encode_to_vec() == b.encode_to_vec() } @@ -80,6 +103,10 @@ impl std::hash::Hash for ConnectorConfig { std::mem::discriminant(self).hash(state); match self { ConnectorConfig::KafkaSource(cfg) => cfg.encode_to_vec().hash(state), + ConnectorConfig::MqttSource(cfg) => cfg.encode_to_vec().hash(state), + ConnectorConfig::HttpSource(cfg) => cfg.encode_to_vec().hash(state), + ConnectorConfig::RosSource(cfg) => cfg.encode_to_vec().hash(state), + ConnectorConfig::RobotBagSource(cfg) => cfg.encode_to_vec().hash(state), ConnectorConfig::KafkaSink(cfg) => cfg.encode_to_vec().hash(state), ConnectorConfig::FilesystemSink(cfg) => cfg.encode_to_vec().hash(state), ConnectorConfig::DeltaSink(cfg) => cfg.encode_to_vec().hash(state), diff --git a/src/streaming_planner/src/connector/registry.rs b/src/streaming_planner/src/connector/registry.rs index 4a8a8c1c..b8e2daff 100644 --- a/src/streaming_planner/src/connector/registry.rs +++ b/src/streaming_planner/src/connector/registry.rs @@ -22,7 +22,12 @@ use super::sink::iceberg::IcebergSinkConnector; use super::sink::kafka::KafkaSinkConnector; use super::sink::lancedb::LanceDbSinkConnector; use super::sink::s3::S3SinkConnector; +use super::source::http::HttpSourceConnector; use super::source::kafka::KafkaSourceConnector; +use super::source::mqtt::MqttSourceConnector; +use super::source::robot_bag::RobotBagSourceConnector; +use super::source::ros::RosSourceConnector; +use super::source::ros2::Ros2SourceConnector; pub struct ConnectorRegistry { sources: HashMap>, @@ -37,6 +42,11 @@ impl ConnectorRegistry { }; registry.register_source(Arc::new(KafkaSourceConnector)); + registry.register_source(Arc::new(MqttSourceConnector)); + registry.register_source(Arc::new(HttpSourceConnector)); + registry.register_source(Arc::new(RosSourceConnector)); + registry.register_source(Arc::new(Ros2SourceConnector)); + registry.register_source(Arc::new(RobotBagSourceConnector)); registry.register_sink(Arc::new(KafkaSinkConnector)); registry.register_sink(Arc::new(S3SinkConnector)); diff --git a/src/streaming_planner/src/connector/source/http.rs b/src/streaming_planner/src/connector/source/http.rs new file mode 100644 index 00000000..eef29123 --- /dev/null +++ b/src/streaming_planner/src/connector/source/http.rs @@ -0,0 +1,258 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; + +use datafusion::common::{Result, plan_datafusion_err, plan_err}; +use protocol::function_stream_graph::{ + BadDataPolicy, DecimalEncodingProto, FormatConfig, HttpResponseSplit, HttpSourceConfig, + HttpSourceMode, JsonFormatConfig, RawBytesFormatConfig, RawStringFormatConfig, + TimestampFormatProto, format_config, +}; + +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::connector_type; +use crate::common::formats::{ + BadData, DecimalEncoding as SqlDecimalEncoding, Format as SqlFormat, + TimestampFormat as SqlTimestampFormat, +}; +use crate::common::with_option_keys as opt; +use crate::connector::config::ConnectorConfig; +use crate::connector::provider::SourceProvider; + +const DEFAULT_SCAN_INTERVAL_MS: u64 = 1000; +const DEFAULT_REQUEST_TIMEOUT_MS: u32 = 30_000; +const DEFAULT_WEBHOOK_HOST: &str = "0.0.0.0"; +const DEFAULT_WEBHOOK_PATH: &str = "/"; + +pub struct HttpSourceConnector; + +impl HttpSourceConnector { + fn sql_format_to_proto(fmt: &SqlFormat) -> Result { + match fmt { + SqlFormat::Json(j) => Ok(FormatConfig { + format: Some(format_config::Format::Json(JsonFormatConfig { + timestamp_format: match j.timestamp_format { + SqlTimestampFormat::RFC3339 => { + TimestampFormatProto::TimestampRfc3339 as i32 + } + SqlTimestampFormat::UnixMillis => { + TimestampFormatProto::TimestampUnixMillis as i32 + } + }, + decimal_encoding: match j.decimal_encoding { + SqlDecimalEncoding::Number => DecimalEncodingProto::DecimalNumber as i32, + SqlDecimalEncoding::String => DecimalEncodingProto::DecimalString as i32, + SqlDecimalEncoding::Bytes => DecimalEncodingProto::DecimalBytes as i32, + }, + include_schema: j.include_schema, + confluent_schema_registry: j.confluent_schema_registry, + schema_id: j.schema_id, + debezium: j.debezium, + unstructured: j.unstructured, + })), + }), + SqlFormat::RawString(_) => Ok(FormatConfig { + format: Some(format_config::Format::RawString(RawStringFormatConfig {})), + }), + SqlFormat::RawBytes(_) => Ok(FormatConfig { + format: Some(format_config::Format::RawBytes(RawBytesFormatConfig {})), + }), + other => plan_err!( + "HTTP source connector: format '{}' is not supported", + other.name() + ), + } + } + + fn bad_data_to_proto(bad: &BadData) -> i32 { + match bad { + BadData::Fail {} => BadDataPolicy::BadDataFail as i32, + BadData::Drop {} => BadDataPolicy::BadDataDrop as i32, + } + } + + fn pull_scan_interval_ms(options: &mut ConnectorOptions) -> Result { + if let Some(d) = options.pull_opt_duration(opt::HTTP_SCAN_INTERVAL)? { + return Ok(d.as_millis() as u64); + } + if let Some(ms) = options.pull_opt_u64(opt::HTTP_SCAN_INTERVAL_MS)? { + return Ok(ms.max(1)); + } + if let Some(ms) = options.pull_opt_u64(opt::HTTP_POLL_INTERVAL_MS)? { + return Ok(ms.max(1)); + } + Ok(DEFAULT_SCAN_INTERVAL_MS) + } + + fn pull_method(options: &mut ConnectorOptions) -> Result { + if let Some(m) = options.pull_opt_str(opt::HTTP_METHOD)? { + return Ok(m.to_ascii_uppercase()); + } + if let Some(m) = options.pull_opt_str(opt::HTTP_LOOKUP_METHOD)? { + return Ok(m.to_ascii_uppercase()); + } + Ok("GET".to_string()) + } + + fn pull_response_split(options: &mut ConnectorOptions) -> Result { + let split = options + .pull_opt_str(opt::HTTP_RESPONSE_SPLIT)? + .map(|s| s.to_ascii_lowercase()); + Ok(match split.as_deref() { + Some("ndjson") | Some("newline") => HttpResponseSplit::HttpResponseNdjson as i32, + Some("json-array") | Some("array") => HttpResponseSplit::HttpResponseJsonArray as i32, + Some("single") | None => HttpResponseSplit::HttpResponseSingle as i32, + Some(other) => { + return plan_err!( + "invalid http.response.split '{other}'; expected single, ndjson, or json-array" + ); + } + }) + } + + fn extract_headers(client_configs: &mut HashMap) -> HashMap { + let mut headers = HashMap::new(); + let keys: Vec = client_configs + .keys() + .filter(|k| k.starts_with(opt::HTTP_HEADER_PREFIX)) + .cloned() + .collect(); + for key in keys { + if let Some(v) = client_configs.remove(&key) { + let name = key[opt::HTTP_HEADER_PREFIX.len()..].to_string(); + headers.insert(name, v); + } + } + headers + } + + fn resolve_mode( + options: &mut ConnectorOptions, + listen_port: Option, + url: Option, + ) -> Result { + if let Some(mode) = options.pull_opt_str(opt::HTTP_MODE)? { + return match mode.to_ascii_lowercase().as_str() { + "poll" | "polling" => Ok(HttpSourceMode::HttpSourcePoll), + "webhook" | "push" => Ok(HttpSourceMode::HttpSourceWebhook), + other => plan_err!("invalid http mode '{other}'; expected poll or webhook"), + }; + } + if listen_port.is_some() { + return Ok(HttpSourceMode::HttpSourceWebhook); + } + if url.is_some() { + return Ok(HttpSourceMode::HttpSourcePoll); + } + Err(plan_datafusion_err!( + "HTTP source requires 'url' (poll) or 'http.listen.port' (webhook)" + )) + } +} + +impl SourceProvider for HttpSourceConnector { + fn name(&self) -> &'static str { + connector_type::HTTP + } + + fn build_source_config( + &self, + options: &mut ConnectorOptions, + format: &Option, + bad_data: BadData, + ) -> Result { + let url = options.pull_opt_str(opt::HTTP_URL)?; + let listen_port = options.pull_opt_u64(opt::HTTP_LISTEN_PORT)?.map(|p| p as u32); + let mode = Self::resolve_mode(options, listen_port, url.clone())?; + + let sql_format = format.as_ref().ok_or_else(|| { + plan_datafusion_err!( + "HTTP source requires 'format' in the WITH clause (e.g. format = 'json')" + ) + })?; + let proto_format = Self::sql_format_to_proto(sql_format)?; + + let rate_limit = options + .pull_opt_u64(opt::MQTT_RATE_LIMIT_MESSAGES_PER_SECOND)? + .map(|v| v.clamp(0, u32::MAX as u64) as u32) + .unwrap_or(0); + + let scan_interval_ms = Self::pull_scan_interval_ms(options)?; + let method = Self::pull_method(options)?; + let response_split = Self::pull_response_split(options)?; + + let request_body = match options.pull_opt_str(opt::HTTP_REQUEST_BODY)? { + Some(s) => Some(s), + None => options.pull_opt_str(opt::HTTP_REQUEST_BODY_LEGACY)?, + }; + + let listen_host = options + .pull_opt_str(opt::HTTP_LISTEN_HOST)? + .unwrap_or_else(|| DEFAULT_WEBHOOK_HOST.to_string()); + + let webhook_path = options + .pull_opt_str(opt::HTTP_WEBHOOK_PATH)? + .unwrap_or_else(|| DEFAULT_WEBHOOK_PATH.to_string()); + + let request_timeout_ms = options + .pull_opt_u64(opt::HTTP_REQUEST_TIMEOUT_MS)? + .map(|v| v.clamp(1, u32::MAX as u64) as u32) + .unwrap_or(DEFAULT_REQUEST_TIMEOUT_MS); + + let _ = options.pull_opt_str(opt::TYPE)?; + let _ = options.pull_opt_str(opt::CONNECTOR)?; + + let mut client_configs = options.drain_remaining_string_values()?; + client_configs.remove(opt::CHECKPOINT_INTERVAL_MS); + client_configs.remove(opt::PIPELINE_PARALLELISM); + client_configs.remove(opt::KEY_BY_PARALLELISM); + client_configs.remove(opt::FORMAT); + + let headers = Self::extract_headers(&mut client_configs); + + let (url, listen_port) = match mode { + HttpSourceMode::HttpSourcePoll => { + let url = url.ok_or_else(|| { + plan_datafusion_err!("HTTP poll mode requires 'url' in the WITH clause") + })?; + (url, listen_port.unwrap_or(0)) + } + HttpSourceMode::HttpSourceWebhook => { + let port = listen_port.ok_or_else(|| { + plan_datafusion_err!( + "HTTP webhook mode requires 'http.listen.port' in the WITH clause" + ) + })?; + (String::new(), port) + } + }; + + Ok(ConnectorConfig::HttpSource(HttpSourceConfig { + mode: mode as i32, + url, + method, + scan_interval_ms, + request_body, + headers, + listen_host, + listen_port, + webhook_path, + format: Some(proto_format), + bad_data_policy: Self::bad_data_to_proto(&bad_data), + rate_limit_msgs_per_sec: rate_limit, + client_configs, + request_timeout_ms, + response_split, + })) + } +} diff --git a/src/streaming_planner/src/connector/source/kafka.rs b/src/streaming_planner/src/connector/source/kafka.rs index 71672279..f735e56d 100644 --- a/src/streaming_planner/src/connector/source/kafka.rs +++ b/src/streaming_planner/src/connector/source/kafka.rs @@ -29,6 +29,37 @@ use crate::connector::provider::SourceProvider; pub struct KafkaSourceConnector; +pub fn default_kafka_consumer_group_id(table_name: &str) -> String { + let sanitized: String = table_name + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '.' || c == '_' || c == '-' { + c + } else { + '-' + } + }) + .collect(); + format!("fs-{sanitized}-consumer") +} + +pub fn ensure_default_consumer_group( + table_name: &str, + options: &mut ConnectorOptions, +) -> Result<()> { + let has_group = options.peek_opt_str(opt::KAFKA_GROUP_ID)?.is_some() + || options.peek_opt_str(opt::KAFKA_GROUP_ID_LEGACY)?.is_some(); + let has_prefix = options.peek_opt_str(opt::KAFKA_GROUP_ID_PREFIX)?.is_some(); + if has_group || has_prefix { + return Ok(()); + } + options.insert_str( + opt::KAFKA_GROUP_ID, + default_kafka_consumer_group_id(table_name), + )?; + Ok(()) +} + impl KafkaSourceConnector { fn sql_format_to_proto(fmt: &SqlFormat) -> Result { match fmt { @@ -108,11 +139,6 @@ impl SourceProvider for KafkaSourceConnector { })?; let proto_format = Self::sql_format_to_proto(sql_format)?; - let rate_limit = options - .pull_opt_u64(opt::KAFKA_RATE_LIMIT_MESSAGES_PER_SECOND)? - .map(|v| v.clamp(1, u32::MAX as u64) as u32) - .unwrap_or(0); - let value_subject = options.pull_opt_str(opt::KAFKA_VALUE_SUBJECT)?; let offset_mode = match options @@ -178,8 +204,68 @@ impl SourceProvider for KafkaSourceConnector { client_configs, format: Some(proto_format), bad_data_policy: Self::bad_data_to_proto(&bad_data), - rate_limit_msgs_per_sec: rate_limit, + rate_limit_msgs_per_sec: 0, value_subject, })) } } + +#[cfg(test)] +mod tests { + use super::*; + use datafusion::sql::sqlparser::ast::{Expr, Ident, SqlOption, Value as SqlValue}; + + fn options_from_map(pairs: &[(&str, &str)]) -> ConnectorOptions { + let opts: Vec = pairs + .iter() + .map(|(k, v)| SqlOption::KeyValue { + key: Ident::new(*k), + value: Expr::Value( + SqlValue::SingleQuotedString((*v).to_string()).with_empty_span(), + ), + }) + .collect(); + ConnectorOptions::new(&opts, &None).expect("options") + } + + #[test] + fn assigns_default_group_id_when_missing() { + let mut options = options_from_map(&[ + ("connector", "kafka"), + ("topic", "events"), + ("bootstrap.servers", "localhost:9092"), + ]); + ensure_default_consumer_group("my_events", &mut options).expect("ensure"); + assert_eq!( + options.peek_opt_str(opt::KAFKA_GROUP_ID).expect("peek"), + Some("fs-my_events-consumer".to_string()) + ); + } + + #[test] + fn keeps_user_group_id() { + let mut options = options_from_map(&[ + ("connector", "kafka"), + ("topic", "events"), + ("bootstrap.servers", "localhost:9092"), + ("group.id", "custom-group"), + ]); + ensure_default_consumer_group("my_events", &mut options).expect("ensure"); + assert_eq!( + options.peek_opt_str(opt::KAFKA_GROUP_ID).expect("peek"), + Some("custom-group".to_string()) + ); + } + + #[test] + fn skips_when_group_id_prefix_set() { + let mut options = options_from_map(&[ + ("connector", "kafka"), + ("topic", "events"), + ("bootstrap.servers", "localhost:9092"), + ("group.id.prefix", "prefix"), + ]); + ensure_default_consumer_group("my_events", &mut options).expect("ensure"); + assert!(options.peek_opt_str(opt::KAFKA_GROUP_ID).expect("peek").is_none()); + } +} diff --git a/src/streaming_planner/src/connector/source/mod.rs b/src/streaming_planner/src/connector/source/mod.rs index b9574391..4f20c39c 100644 --- a/src/streaming_planner/src/connector/source/mod.rs +++ b/src/streaming_planner/src/connector/source/mod.rs @@ -10,4 +10,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod http; pub mod kafka; +pub mod mqtt; +pub mod robot_bag; +pub mod ros; +pub mod ros2; +pub mod ros_common; diff --git a/src/streaming_planner/src/connector/source/mqtt.rs b/src/streaming_planner/src/connector/source/mqtt.rs new file mode 100644 index 00000000..6dd21349 --- /dev/null +++ b/src/streaming_planner/src/connector/source/mqtt.rs @@ -0,0 +1,206 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use datafusion::common::{Result, plan_datafusion_err, plan_err}; +use protocol::function_stream_graph::{ + BadDataPolicy, DecimalEncodingProto, FormatConfig, JsonFormatConfig, MqttSourceConfig, + RawBytesFormatConfig, RawStringFormatConfig, TimestampFormatProto, format_config, +}; + +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::connector_type; +use crate::common::formats::{ + BadData, DecimalEncoding as SqlDecimalEncoding, Format as SqlFormat, + TimestampFormat as SqlTimestampFormat, +}; +use crate::common::with_option_keys as opt; +use crate::connector::config::ConnectorConfig; +use crate::connector::provider::SourceProvider; + +const DEFAULT_MQTT_PORT: u32 = 1883; +const DEFAULT_MQTT_QOS: u32 = 1; +const DEFAULT_MQTT_KEEP_ALIVE_SECS: u32 = 60; + +pub struct MqttSourceConnector; + +impl MqttSourceConnector { + fn sql_format_to_proto(fmt: &SqlFormat) -> Result { + match fmt { + SqlFormat::Json(j) => Ok(FormatConfig { + format: Some(format_config::Format::Json(JsonFormatConfig { + timestamp_format: match j.timestamp_format { + SqlTimestampFormat::RFC3339 => { + TimestampFormatProto::TimestampRfc3339 as i32 + } + SqlTimestampFormat::UnixMillis => { + TimestampFormatProto::TimestampUnixMillis as i32 + } + }, + decimal_encoding: match j.decimal_encoding { + SqlDecimalEncoding::Number => DecimalEncodingProto::DecimalNumber as i32, + SqlDecimalEncoding::String => DecimalEncodingProto::DecimalString as i32, + SqlDecimalEncoding::Bytes => DecimalEncodingProto::DecimalBytes as i32, + }, + include_schema: j.include_schema, + confluent_schema_registry: j.confluent_schema_registry, + schema_id: j.schema_id, + debezium: j.debezium, + unstructured: j.unstructured, + })), + }), + SqlFormat::RawString(_) => Ok(FormatConfig { + format: Some(format_config::Format::RawString(RawStringFormatConfig {})), + }), + SqlFormat::RawBytes(_) => Ok(FormatConfig { + format: Some(format_config::Format::RawBytes(RawBytesFormatConfig {})), + }), + other => plan_err!( + "MQTT source connector: format '{}' is not supported", + other.name() + ), + } + } + + fn bad_data_to_proto(bad: &BadData) -> i32 { + match bad { + BadData::Fail {} => BadDataPolicy::BadDataFail as i32, + BadData::Drop {} => BadDataPolicy::BadDataDrop as i32, + } + } + + fn pull_host(options: &mut ConnectorOptions) -> Result { + match options.pull_opt_str(opt::MQTT_HOST)? { + Some(s) => Ok(s), + None => options.pull_opt_str(opt::MQTT_HOST_LEGACY)?.ok_or_else(|| { + plan_datafusion_err!( + "MQTT connector requires 'mqtt.host' (or 'host') in the WITH clause" + ) + }), + } + } + + fn pull_port(options: &mut ConnectorOptions) -> Result { + if let Some(p) = options.pull_opt_u64(opt::MQTT_PORT)? { + return Ok(p.clamp(1, u16::MAX as u64) as u32); + } + if let Some(p) = options.pull_opt_u64(opt::MQTT_PORT_LEGACY)? { + return Ok(p.clamp(1, u16::MAX as u64) as u32); + } + Ok(DEFAULT_MQTT_PORT) + } + + fn pull_qos(options: &mut ConnectorOptions) -> Result { + let qos = match options.pull_opt_u64(opt::MQTT_QOS)? { + Some(v) => v, + None => options + .pull_opt_u64(opt::MQTT_QOS_LEGACY)? + .unwrap_or(DEFAULT_MQTT_QOS as u64), + }; + Ok(qos.clamp(0, 2) as u32) + } + + fn pull_clean_session(options: &mut ConnectorOptions) -> Result { + if let Some(v) = options.pull_opt_bool(opt::MQTT_CLEAN_SESSION)? { + return Ok(v); + } + if let Some(v) = options.pull_opt_bool(opt::MQTT_CLEAN_SESSION_LEGACY)? { + return Ok(v); + } + Ok(true) + } + + fn pull_keep_alive_secs(options: &mut ConnectorOptions) -> Result { + if let Some(v) = options.pull_opt_u64(opt::MQTT_KEEP_ALIVE_SECS)? { + return Ok(v.max(1) as u32); + } + if let Some(v) = options.pull_opt_u64(opt::MQTT_KEEP_ALIVE_SECS_LEGACY)? { + return Ok(v.max(1) as u32); + } + Ok(DEFAULT_MQTT_KEEP_ALIVE_SECS) + } +} + +impl SourceProvider for MqttSourceConnector { + fn name(&self) -> &'static str { + connector_type::MQTT + } + + fn build_source_config( + &self, + options: &mut ConnectorOptions, + format: &Option, + bad_data: BadData, + ) -> Result { + let host = Self::pull_host(options)?; + let port = Self::pull_port(options)?; + + let topic = options.pull_opt_str(opt::MQTT_TOPIC)?.ok_or_else(|| { + plan_datafusion_err!("MQTT connector requires 'topic' in the WITH clause") + })?; + + let sql_format = format.as_ref().ok_or_else(|| { + plan_datafusion_err!( + "MQTT source requires 'format' in the WITH clause (e.g. format = 'json')" + ) + })?; + let proto_format = Self::sql_format_to_proto(sql_format)?; + + let rate_limit = options + .pull_opt_u64(opt::MQTT_RATE_LIMIT_MESSAGES_PER_SECOND)? + .map(|v| v.clamp(0, u32::MAX as u64) as u32) + .unwrap_or(0); + + let client_id = match options.pull_opt_str(opt::MQTT_CLIENT_ID)? { + Some(s) => Some(s), + None => options.pull_opt_str(opt::MQTT_CLIENT_ID_LEGACY)?, + }; + + let username = match options.pull_opt_str(opt::MQTT_USERNAME)? { + Some(s) => Some(s), + None => options.pull_opt_str(opt::MQTT_USERNAME_LEGACY)?, + }; + + let password = match options.pull_opt_str(opt::MQTT_PASSWORD)? { + Some(s) => Some(s), + None => options.pull_opt_str(opt::MQTT_PASSWORD_LEGACY)?, + }; + + let qos = Self::pull_qos(options)?; + let clean_session = Self::pull_clean_session(options)?; + let keep_alive_secs = Self::pull_keep_alive_secs(options)?; + + let _ = options.pull_opt_str(opt::TYPE)?; + let _ = options.pull_opt_str(opt::CONNECTOR)?; + + let mut client_configs = options.drain_remaining_string_values()?; + client_configs.remove(opt::CHECKPOINT_INTERVAL_MS); + client_configs.remove(opt::PIPELINE_PARALLELISM); + client_configs.remove(opt::KEY_BY_PARALLELISM); + client_configs.remove(opt::FORMAT); + + Ok(ConnectorConfig::MqttSource(MqttSourceConfig { + topic, + host, + port, + client_id, + username, + password, + qos, + clean_session, + keep_alive_secs, + format: Some(proto_format), + bad_data_policy: Self::bad_data_to_proto(&bad_data), + rate_limit_msgs_per_sec: rate_limit, + client_configs, + })) + } +} diff --git a/src/streaming_planner/src/connector/source/robot_bag.rs b/src/streaming_planner/src/connector/source/robot_bag.rs new file mode 100644 index 00000000..0f4c7eac --- /dev/null +++ b/src/streaming_planner/src/connector/source/robot_bag.rs @@ -0,0 +1,184 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io::Read; +use std::path::Path; + +use datafusion::common::{Result, plan_datafusion_err, plan_err}; +use protocol::function_stream_graph::{ + RobotBagFormat, RobotBagSourceConfig, +}; + +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::connector_type; +use crate::common::formats::{BadData, Format as SqlFormat}; +use crate::common::with_option_keys as opt; +use crate::connector::config::ConnectorConfig; +use crate::connector::provider::SourceProvider; + +use super::ros_common::{bad_data_to_proto, sql_format_to_proto}; + +const DEFAULT_REPLAY_INTERVAL_MS: u64 = 0; + +pub struct RobotBagSourceConnector; + +impl RobotBagSourceConnector { + fn directory_has_pcd_files(dir: &Path) -> bool { + std::fs::read_dir(dir) + .ok() + .into_iter() + .flatten() + .filter_map(|e| e.ok()) + .any(|e| { + e.path() + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("pcd")) + }) + } + + fn is_ros1_bag_file(path: &Path) -> bool { + let Ok(mut file) = std::fs::File::open(path) else { + return false; + }; + let mut buf = [0u8; 13]; + let Ok(n) = file.read(&mut buf) else { + return false; + }; + if n < 13 { + return false; + } + std::str::from_utf8(&buf).ok().is_some_and(|s| { + s.starts_with("#ROSBAG V2.0") || s.starts_with("#ROSBAG V1.02") || s.starts_with("#ROSBAG V1.0") + }) + } + + fn detect_format(path: &str) -> i32 { + let p = Path::new(path); + if p.is_dir() { + if Self::directory_has_pcd_files(p) { + return RobotBagFormat::RobotBagPcd as i32; + } + return RobotBagFormat::RobotBagRos2 as i32; + } + if Self::is_ros1_bag_file(p) { + return RobotBagFormat::RobotBagRos1 as i32; + } + match p.extension().and_then(|e| e.to_str()).map(|s| s.to_ascii_lowercase()) { + Some(ext) if ext == "mcap" => RobotBagFormat::RobotBagMcap as i32, + Some(ext) if ext == "db3" => RobotBagFormat::RobotBagRos2 as i32, + Some(ext) if ext == "bag" => RobotBagFormat::RobotBagRos1 as i32, + Some(ext) if ext == "pcd" => RobotBagFormat::RobotBagPcd as i32, + Some(ext) if ext == "jsonl" || ext == "ndjson" => RobotBagFormat::RobotBagJsonl as i32, + Some(ext) if ext == "csv" => RobotBagFormat::RobotBagCsv as i32, + _ => RobotBagFormat::RobotBagAuto as i32, + } + } + + fn parse_bag_format(options: &mut ConnectorOptions, path: &str) -> Result { + let fmt = options + .pull_opt_str(opt::ROBOT_BAG_FORMAT)? + .map(|s| s.to_ascii_lowercase()); + Ok(match fmt.as_deref() { + None => Self::detect_format(path), + Some("auto") => Self::detect_format(path), + Some("mcap") => RobotBagFormat::RobotBagMcap as i32, + Some("ros2" | "db3" | "rosbag2") => RobotBagFormat::RobotBagRos2 as i32, + Some("ros1" | "bag" | "rosbag") => RobotBagFormat::RobotBagRos1 as i32, + Some("pcd" | "pointcloud") => RobotBagFormat::RobotBagPcd as i32, + Some("jsonl" | "ndjson" | "json") => RobotBagFormat::RobotBagJsonl as i32, + Some("csv") => RobotBagFormat::RobotBagCsv as i32, + Some(other) => { + return plan_err!( + "invalid bag.format '{other}'; expected auto, mcap, ros1, ros2, pcd, jsonl, or csv" + ); + } + }) + } +} + +impl SourceProvider for RobotBagSourceConnector { + fn name(&self) -> &'static str { + connector_type::ROBOT_BAG + } + + fn build_source_config( + &self, + options: &mut ConnectorOptions, + format: &Option, + bad_data: BadData, + ) -> Result { + let path = options.pull_opt_str(opt::ROBOT_BAG_PATH)?; + let path = if let Some(p) = path { + p + } else { + options.pull_opt_str(opt::PATH)?.ok_or_else(|| { + plan_datafusion_err!("robot-bag source requires 'path' in the WITH clause") + })? + }; + + let bag_format = Self::parse_bag_format(options, &path)?; + + let topic = if let Some(t) = options.pull_opt_str(opt::ROBOT_BAG_TOPIC)? { + Some(t) + } else { + options.pull_opt_str(opt::ROS_TOPIC)? + }; + + let replay_interval_ms = options + .pull_opt_u64(opt::ROBOT_BAG_REPLAY_INTERVAL_MS)? + .unwrap_or(DEFAULT_REPLAY_INTERVAL_MS); + + let loop_replay = options + .pull_opt_bool(opt::ROBOT_BAG_LOOP)? + .unwrap_or(false); + + let pcd_emit_mode = options + .pull_opt_str(opt::ROBOT_BAG_PCD_EMIT_MODE)? + .unwrap_or_else(|| "point".to_string()); + match pcd_emit_mode.to_ascii_lowercase().as_str() { + "point" | "points" | "cloud" | "file" => {} + other => { + return plan_err!( + "invalid pcd.emit.mode '{other}'; expected 'point' or 'cloud'" + ); + } + } + + let sql_format = format.as_ref().ok_or_else(|| { + plan_datafusion_err!( + "robot-bag source requires 'format' (json / raw_bytes / raw_string)" + ) + })?; + let proto_format = sql_format_to_proto(sql_format)?; + + let _ = options.pull_opt_str(opt::TYPE)?; + let _ = options.pull_opt_str(opt::CONNECTOR)?; + + let mut client_configs = options.drain_remaining_string_values()?; + client_configs.remove(opt::CHECKPOINT_INTERVAL_MS); + client_configs.remove(opt::PIPELINE_PARALLELISM); + client_configs.remove(opt::KEY_BY_PARALLELISM); + client_configs.remove(opt::FORMAT); + + Ok(ConnectorConfig::RobotBagSource(RobotBagSourceConfig { + path, + bag_format, + topic, + replay_interval_ms, + loop_replay, + format: Some(proto_format), + bad_data_policy: bad_data_to_proto(&bad_data), + client_configs, + pcd_emit_mode, + })) + } +} diff --git a/src/streaming_planner/src/connector/source/ros.rs b/src/streaming_planner/src/connector/source/ros.rs new file mode 100644 index 00000000..dd7363cc --- /dev/null +++ b/src/streaming_planner/src/connector/source/ros.rs @@ -0,0 +1,44 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use datafusion::common::Result; +use protocol::function_stream_graph::RosVersion; + +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::connector_type; +use crate::common::formats::{BadData, Format}; +use crate::connector::provider::SourceProvider; + +use super::ros_common::{DEFAULT_ROS_URL, build_ros_source_config}; + +pub struct RosSourceConnector; + +impl SourceProvider for RosSourceConnector { + fn name(&self) -> &'static str { + connector_type::ROS + } + + fn build_source_config( + &self, + options: &mut ConnectorOptions, + format: &Option, + bad_data: BadData, + ) -> Result { + build_ros_source_config( + options, + format, + bad_data, + RosVersion::Ros1, + DEFAULT_ROS_URL, + ) + } +} diff --git a/src/streaming_planner/src/connector/source/ros2.rs b/src/streaming_planner/src/connector/source/ros2.rs new file mode 100644 index 00000000..0a4aa62b --- /dev/null +++ b/src/streaming_planner/src/connector/source/ros2.rs @@ -0,0 +1,45 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use datafusion::common::Result; +use protocol::function_stream_graph::RosVersion; + +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::connector_type; +use crate::common::formats::{BadData, Format}; +use crate::connector::provider::SourceProvider; + +use super::ros_common::{DEFAULT_ROS2_URL, build_ros_source_config}; + +/// ROS2 live source via `rosbridge_server` (same WebSocket protocol as ROS1). +pub struct Ros2SourceConnector; + +impl SourceProvider for Ros2SourceConnector { + fn name(&self) -> &'static str { + connector_type::ROS2 + } + + fn build_source_config( + &self, + options: &mut ConnectorOptions, + format: &Option, + bad_data: BadData, + ) -> Result { + build_ros_source_config( + options, + format, + bad_data, + RosVersion::Ros2, + DEFAULT_ROS2_URL, + ) + } +} diff --git a/src/streaming_planner/src/connector/source/ros_common.rs b/src/streaming_planner/src/connector/source/ros_common.rs new file mode 100644 index 00000000..b470e6e6 --- /dev/null +++ b/src/streaming_planner/src/connector/source/ros_common.rs @@ -0,0 +1,133 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use datafusion::common::{Result, plan_datafusion_err, plan_err}; +use protocol::function_stream_graph::{ + BadDataPolicy, DecimalEncodingProto, FormatConfig, JsonFormatConfig, RawBytesFormatConfig, + RawStringFormatConfig, RosSourceConfig, RosVersion, TimestampFormatProto, format_config, +}; + +use crate::common::connector_options::ConnectorOptions; +use crate::common::formats::{ + BadData, DecimalEncoding as SqlDecimalEncoding, Format as SqlFormat, + TimestampFormat as SqlTimestampFormat, +}; +use crate::common::with_option_keys as opt; +use crate::connector::config::ConnectorConfig; + +pub(crate) const DEFAULT_ROS_URL: &str = "ws://127.0.0.1:9090"; +pub(crate) const DEFAULT_ROS2_URL: &str = "ws://127.0.0.1:9090"; +pub(crate) const DEFAULT_ROS_MESSAGE_FIELD: &str = "msg"; + +pub(crate) fn sql_format_to_proto(fmt: &SqlFormat) -> Result { + match fmt { + SqlFormat::Json(j) => Ok(FormatConfig { + format: Some(format_config::Format::Json(JsonFormatConfig { + timestamp_format: match j.timestamp_format { + SqlTimestampFormat::RFC3339 => TimestampFormatProto::TimestampRfc3339 as i32, + SqlTimestampFormat::UnixMillis => TimestampFormatProto::TimestampUnixMillis as i32, + }, + decimal_encoding: match j.decimal_encoding { + SqlDecimalEncoding::Number => DecimalEncodingProto::DecimalNumber as i32, + SqlDecimalEncoding::String => DecimalEncodingProto::DecimalString as i32, + SqlDecimalEncoding::Bytes => DecimalEncodingProto::DecimalBytes as i32, + }, + include_schema: j.include_schema, + confluent_schema_registry: j.confluent_schema_registry, + schema_id: j.schema_id, + debezium: j.debezium, + unstructured: j.unstructured, + })), + }), + SqlFormat::RawString(_) => Ok(FormatConfig { + format: Some(format_config::Format::RawString(RawStringFormatConfig {})), + }), + SqlFormat::RawBytes(_) => Ok(FormatConfig { + format: Some(format_config::Format::RawBytes(RawBytesFormatConfig {})), + }), + other => plan_err!( + "ROS source connector: format '{}' is not supported", + other.name() + ), + } +} + +pub(crate) fn bad_data_to_proto(bad: &BadData) -> i32 { + match bad { + BadData::Fail {} => BadDataPolicy::BadDataFail as i32, + BadData::Drop {} => BadDataPolicy::BadDataDrop as i32, + } +} + +pub(crate) fn build_ros_source_config( + options: &mut ConnectorOptions, + format: &Option, + bad_data: BadData, + default_version: RosVersion, + default_url: &str, +) -> Result { + let url = match options.pull_opt_str(opt::ROS_URL)? { + Some(s) => s, + None => options + .pull_opt_str(opt::ROS_URL_LEGACY)? + .unwrap_or_else(|| default_url.to_string()), + }; + + let topic = match options.pull_opt_str(opt::ROS_TOPIC)? { + Some(s) => s, + None => options.pull_opt_str(opt::ROS_TOPIC_LEGACY)?.ok_or_else(|| { + plan_datafusion_err!("ROS source requires 'topic' (ROS topic name) in the WITH clause") + })?, + }; + + let message_field = match options.pull_opt_str(opt::ROS_MESSAGE_FIELD)? { + Some(s) => s, + None => options + .pull_opt_str(opt::ROS_MESSAGE_FIELD_LEGACY)? + .unwrap_or_else(|| DEFAULT_ROS_MESSAGE_FIELD.to_string()), + }; + + let ros_version = match options.pull_opt_str(opt::ROS_VERSION)?.as_deref() { + Some("ros1" | "1") => RosVersion::Ros1 as i32, + Some("ros2" | "2") => RosVersion::Ros2 as i32, + None => default_version as i32, + Some(other) => { + return plan_err!("invalid ros.version '{other}'; expected ros1 or ros2"); + } + }; + + let sql_format = format.as_ref().ok_or_else(|| { + plan_datafusion_err!( + "ROS source requires 'format' in the WITH clause (e.g. format = 'json')" + ) + })?; + let proto_format = sql_format_to_proto(sql_format)?; + + let _ = options.pull_opt_str(opt::TYPE)?; + let _ = options.pull_opt_str(opt::CONNECTOR)?; + + let mut client_configs = options.drain_remaining_string_values()?; + client_configs.remove(opt::CHECKPOINT_INTERVAL_MS); + client_configs.remove(opt::PIPELINE_PARALLELISM); + client_configs.remove(opt::KEY_BY_PARALLELISM); + client_configs.remove(opt::FORMAT); + + Ok(ConnectorConfig::RosSource(RosSourceConfig { + url, + topic, + message_field, + format: Some(proto_format), + bad_data_policy: bad_data_to_proto(&bad_data), + client_configs, + ros_version, + })) +} diff --git a/src/streaming_planner/src/schema/introspection/ddl_compiler.rs b/src/streaming_planner/src/schema/introspection/ddl_compiler.rs index 3a0dde74..ba92da06 100644 --- a/src/streaming_planner/src/schema/introspection/ddl_compiler.rs +++ b/src/streaming_planner/src/schema/introspection/ddl_compiler.rs @@ -31,6 +31,7 @@ use crate::common::constants::{connection_table_role, connector_type, sql_field} use crate::common::with_option_keys as opt; use crate::common::{BadData, ConnectorOptions, Format, Framing, JsonCompression, JsonFormat}; use crate::connector::registry::REGISTRY; +use crate::connector::source::kafka::ensure_default_consumer_group; use crate::schema::ColumnDescriptor; use crate::schema::catalog::{ExternalTable, LookupTable, SourceTable}; use crate::schema::data_encoding_format::DataEncodingFormat; @@ -84,6 +85,9 @@ impl<'a> DdlCompiler<'a> { validate_adapter_availability(&adapter_type)?; let pk_constraints = AstUtils::parse_primary_keys(&stmt.constraints)?; + if adapter_type == connector_type::KAFKA { + ensure_default_consumer_group(&target_name, &mut options)?; + } let catalog_with_options = options.snapshot_for_catalog(); let format = Format::from_opts(&mut options)?; @@ -167,6 +171,9 @@ impl<'a> DdlCompiler<'a> { validate_adapter_availability(&adapter_type)?; let pk_constraints = AstUtils::parse_primary_keys(&stmt.constraints)?; + if adapter_type == connector_type::KAFKA { + ensure_default_consumer_group(&target_name, &mut options)?; + } let catalog_with_options = options.snapshot_for_catalog(); let connection_format = Format::from_opts(&mut options)?; diff --git a/src/streaming_runtime_core/src/api/context.rs b/src/streaming_runtime_core/src/api/context.rs index 48d7f337..a2db010c 100644 --- a/src/streaming_runtime_core/src/api/context.rs +++ b/src/streaming_runtime_core/src/api/context.rs @@ -33,7 +33,7 @@ pub struct TaskContextConfig { impl Default for TaskContextConfig { fn default() -> Self { Self { - source_idle_timeout: Duration::from_millis(50), + source_idle_timeout: Duration::from_millis(5), } } } diff --git a/src/streaming_runtime_operators/Cargo.toml b/src/streaming_runtime_operators/Cargo.toml index 53f6fbba..9445d64b 100644 --- a/src/streaming_runtime_operators/Cargo.toml +++ b/src/streaming_runtime_operators/Cargo.toml @@ -42,6 +42,15 @@ parquet = "55" prost = "0.13" protocol = { path = "../../protocol" } rdkafka = { version = "0.38", features = ["cmake-build", "ssl", "gssapi", "curl"] } +rumqttc = "0.24" +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } +axum = "0.7" +tokio-tungstenite = { version = "0.24", features = ["rustls-tls-webpki-roots"] } +mcap = "0.19" +rosbag = "0.6.3" +lzf = "1.0.0" +byteorder = "1.5" +rusqlite = { version = "0.32", features = ["bundled"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = "2" diff --git a/src/streaming_runtime_operators/src/factory/connector/dispatchers.rs b/src/streaming_runtime_operators/src/factory/connector/dispatchers.rs index ca9f98aa..c8743355 100644 --- a/src/streaming_runtime_operators/src/factory/connector/dispatchers.rs +++ b/src/streaming_runtime_operators/src/factory/connector/dispatchers.rs @@ -23,7 +23,9 @@ use crate::sql::common::constants::connector_type; use super::{ DeltaSinkDispatcher, FilesystemSinkDispatcher, IcebergSinkDispatcher, LanceDbSinkDispatcher, - S3SinkDispatcher, kafka::KafkaConnectorDispatcher, + S3SinkDispatcher, http::HttpSourceDispatcher, kafka::KafkaConnectorDispatcher, + mqtt::MqttSourceDispatcher, robot_bag::RobotBagSourceDispatcher, + ros::RosSourceDispatcher, }; pub struct ConnectorSourceDispatcher; @@ -34,6 +36,11 @@ impl OperatorConstructor for ConnectorSourceDispatcher { .context("failed decoding connector op for source dispatch")?; match op.connector.to_ascii_lowercase().as_str() { connector_type::KAFKA => KafkaConnectorDispatcher.with_config(config, registry), + connector_type::MQTT => MqttSourceDispatcher.with_config(config, registry), + connector_type::HTTP => HttpSourceDispatcher.with_config(config, registry), + connector_type::ROS => RosSourceDispatcher.with_config(config, registry), + connector_type::ROS2 => RosSourceDispatcher.with_config(config, registry), + connector_type::ROBOT_BAG => RobotBagSourceDispatcher.with_config(config, registry), _ => bail!("unsupported source connector '{}'", op.connector), } } diff --git a/src/streaming_runtime_operators/src/factory/connector/http.rs b/src/streaming_runtime_operators/src/factory/connector/http.rs new file mode 100644 index 00000000..37d29347 --- /dev/null +++ b/src/streaming_runtime_operators/src/factory/connector/http.rs @@ -0,0 +1,153 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use anyhow::{Context, Result, bail}; +use prost::Message; +use reqwest::Method; +use std::num::NonZeroU32; +use std::sync::Arc; +use std::time::Duration; +use tracing::info; + +use protocol::function_stream_graph::connector_op::Config; +use protocol::function_stream_graph::{ + BadDataPolicy, ConnectorOp, DecimalEncodingProto, FormatConfig, HttpSourceConfig, + TimestampFormatProto, +}; + +use crate::core::api::operator::ConstructedOperator; +use crate::factory::global::Registry; +use crate::factory::operator_constructor::OperatorConstructor; +use crate::format::{ + BadDataPolicy as RtBadDataPolicy, DecimalEncoding as RtDecimalEncoding, + Format as RuntimeFormat, JsonFormat as RuntimeJsonFormat, TimestampFormat as RtTimestampFormat, +}; +use crate::operators::source::http::{ + HttpSourceOperator, proto_mode_to_runtime, proto_split_to_runtime, +}; +use crate::operators::source::kafka::BufferedDeserializer; +use crate::sql::common::FsSchema; + +const DEFAULT_SOURCE_BATCH_SIZE: usize = 1024; + +fn proto_format_to_runtime(fmt: &Option) -> Result { + let cfg = fmt.as_ref().context("FormatConfig is required")?; + match &cfg.format { + Some(protocol::function_stream_graph::format_config::Format::Json(j)) => { + Ok(RuntimeFormat::Json(RuntimeJsonFormat { + timestamp_format: match j.timestamp_format() { + TimestampFormatProto::TimestampRfc3339 => RtTimestampFormat::RFC3339, + TimestampFormatProto::TimestampUnixMillis => RtTimestampFormat::UnixMillis, + }, + decimal_encoding: match j.decimal_encoding() { + DecimalEncodingProto::DecimalNumber => RtDecimalEncoding::Number, + DecimalEncodingProto::DecimalString => RtDecimalEncoding::String, + DecimalEncodingProto::DecimalBytes => RtDecimalEncoding::Bytes, + }, + include_schema: j.include_schema, + })) + } + Some(protocol::function_stream_graph::format_config::Format::RawString(_)) => { + Ok(RuntimeFormat::RawString) + } + Some(protocol::function_stream_graph::format_config::Format::RawBytes(_)) => { + Ok(RuntimeFormat::RawBytes) + } + None => bail!("FormatConfig has no format variant set"), + } +} + +fn proto_bad_data_to_runtime(policy: i32) -> RtBadDataPolicy { + match BadDataPolicy::try_from(policy) { + Ok(BadDataPolicy::BadDataDrop) => RtBadDataPolicy::Drop, + _ => RtBadDataPolicy::Fail, + } +} + +fn parse_method(method: &str) -> Result { + Method::from_bytes(method.as_bytes()) + .with_context(|| format!("unsupported HTTP method '{method}'")) +} + +pub struct HttpSourceDispatcher; + +impl OperatorConstructor for HttpSourceDispatcher { + fn with_config(&self, payload: &[u8], _registry: Arc) -> Result { + let op = ConnectorOp::decode(payload).context("Failed to decode ConnectorOp protobuf")?; + + let fs_schema = op + .fs_schema + .as_ref() + .map(|fs| FsSchema::try_from(fs.clone())) + .transpose() + .map_err(|e| anyhow::anyhow!("{e}"))?; + + match op.config { + Some(Config::HttpSource(ref cfg)) => { + Self::build_http_source(&op.name, cfg, fs_schema) + } + Some(_) => bail!( + "ConnectorOp '{}': received non-http connector config for HTTP dispatcher", + op.name + ), + None => bail!("ConnectorOp '{}' has no configuration payload", op.name), + } + } +} + +impl HttpSourceDispatcher { + fn build_http_source( + _name: &str, + cfg: &HttpSourceConfig, + fs_schema: Option, + ) -> Result { + let mode = proto_mode_to_runtime(cfg.mode); + info!(?mode, "Constructing HTTP Source"); + + let fs = fs_schema.context("fs_schema is required for HTTP Source")?; + let runtime_format = proto_format_to_runtime(&cfg.format)?; + let bad_data = proto_bad_data_to_runtime(cfg.bad_data_policy); + + let deserializer = Box::new(BufferedDeserializer::new( + runtime_format, + fs.schema.clone(), + bad_data, + DEFAULT_SOURCE_BATCH_SIZE, + )); + + let rate_limit = (cfg.rate_limit_msgs_per_sec > 0) + .then(|| NonZeroU32::new(cfg.rate_limit_msgs_per_sec).expect("positive rate limit")); + + let scan_interval = + Duration::from_millis(cfg.scan_interval_ms.max(1)); + let request_timeout = + Duration::from_millis(cfg.request_timeout_ms.max(1) as u64); + + let source_op = HttpSourceOperator::new( + mode, + cfg.url.clone(), + parse_method(&cfg.method)?, + scan_interval, + cfg.request_body.clone(), + cfg.headers.clone(), + cfg.listen_host.clone(), + cfg.listen_port.clamp(1, u16::MAX as u32) as u16, + cfg.webhook_path.clone(), + request_timeout, + proto_split_to_runtime(cfg.response_split), + rate_limit, + deserializer, + ); + + Ok(ConstructedOperator::Source(Box::new(source_op))) + } +} diff --git a/src/streaming_runtime_operators/src/factory/connector/kafka.rs b/src/streaming_runtime_operators/src/factory/connector/kafka.rs index 4e967f6d..a9747665 100644 --- a/src/streaming_runtime_operators/src/factory/connector/kafka.rs +++ b/src/streaming_runtime_operators/src/factory/connector/kafka.rs @@ -13,7 +13,6 @@ use anyhow::{Context, Result, bail}; use prost::Message; use std::collections::HashMap; -use std::num::NonZeroU32; use std::sync::Arc; use protocol::function_stream_graph::connector_op::Config; @@ -33,11 +32,10 @@ use crate::format::{ Format as RuntimeFormat, JsonFormat as RuntimeJsonFormat, TimestampFormat as RtTimestampFormat, }; use crate::operators::sink::kafka::{ConsistencyMode, KafkaSinkOperator}; +use crate::operators::source::batch_buffer::DEFAULT_SOURCE_BATCH_SIZE; use crate::operators::source::kafka::{BufferedDeserializer, KafkaSourceOperator}; use crate::sql::common::FsSchema; -const DEFAULT_SOURCE_BATCH_SIZE: usize = 1024; - // ─────────────── Proto → Runtime type conversions ─────────────── fn proto_format_to_runtime(fmt: &Option) -> Result { @@ -169,9 +167,6 @@ impl KafkaConnectorDispatcher { DEFAULT_SOURCE_BATCH_SIZE, )); - let rate = NonZeroU32::new(cfg.rate_limit_msgs_per_sec.max(1)) - .unwrap_or_else(|| NonZeroU32::new(1_000_000).expect("nonzero")); - let source_op = KafkaSourceOperator::new( cfg.topic.clone(), cfg.bootstrap_servers.clone(), @@ -179,7 +174,6 @@ impl KafkaConnectorDispatcher { cfg.group_id_prefix.clone(), proto_offset_to_runtime(cfg.offset_mode), final_configs, - rate, vec![], deserializer, ); diff --git a/src/streaming_runtime_operators/src/factory/connector/mod.rs b/src/streaming_runtime_operators/src/factory/connector/mod.rs index d1a6d7f3..cddb9344 100644 --- a/src/streaming_runtime_operators/src/factory/connector/mod.rs +++ b/src/streaming_runtime_operators/src/factory/connector/mod.rs @@ -14,7 +14,11 @@ pub mod delta; mod dispatchers; pub mod filesystem; pub mod iceberg; +pub mod http; pub mod kafka; +pub mod mqtt; +pub mod robot_bag; +pub mod ros; pub mod lancedb; pub mod s3; pub mod sink_props_codec; @@ -24,5 +28,9 @@ pub use dispatchers::{ConnectorSinkDispatcher, ConnectorSourceDispatcher}; pub use filesystem::FilesystemSinkDispatcher; pub use iceberg::IcebergSinkDispatcher; pub use kafka::KafkaConnectorDispatcher; +pub use http::HttpSourceDispatcher; +pub use mqtt::MqttSourceDispatcher; +pub use robot_bag::RobotBagSourceDispatcher; +pub use ros::RosSourceDispatcher; pub use lancedb::LanceDbSinkDispatcher; pub use s3::S3SinkDispatcher; diff --git a/src/streaming_runtime_operators/src/factory/connector/mqtt.rs b/src/streaming_runtime_operators/src/factory/connector/mqtt.rs new file mode 100644 index 00000000..e5fc3107 --- /dev/null +++ b/src/streaming_runtime_operators/src/factory/connector/mqtt.rs @@ -0,0 +1,138 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use anyhow::{Context, Result, bail}; +use prost::Message; +use std::sync::Arc; +use tracing::info; + +use protocol::function_stream_graph::connector_op::Config; +use protocol::function_stream_graph::{ + BadDataPolicy, ConnectorOp, DecimalEncodingProto, FormatConfig, MqttSourceConfig, + TimestampFormatProto, +}; + +use crate::core::api::operator::ConstructedOperator; +use crate::factory::global::Registry; +use crate::factory::operator_constructor::OperatorConstructor; +use crate::format::{ + BadDataPolicy as RtBadDataPolicy, DecimalEncoding as RtDecimalEncoding, + Format as RuntimeFormat, JsonFormat as RuntimeJsonFormat, TimestampFormat as RtTimestampFormat, +}; +use crate::operators::source::batch_buffer::DEFAULT_SOURCE_BATCH_SIZE; +use crate::operators::source::kafka::BufferedDeserializer; +use crate::operators::source::mqtt::{MqttSourceOperator, proto_qos_to_rumqtt}; +use crate::sql::common::FsSchema; + +fn proto_format_to_runtime(fmt: &Option) -> Result { + let cfg = fmt.as_ref().context("FormatConfig is required")?; + match &cfg.format { + Some(protocol::function_stream_graph::format_config::Format::Json(j)) => { + Ok(RuntimeFormat::Json(RuntimeJsonFormat { + timestamp_format: match j.timestamp_format() { + TimestampFormatProto::TimestampRfc3339 => RtTimestampFormat::RFC3339, + TimestampFormatProto::TimestampUnixMillis => RtTimestampFormat::UnixMillis, + }, + decimal_encoding: match j.decimal_encoding() { + DecimalEncodingProto::DecimalNumber => RtDecimalEncoding::Number, + DecimalEncodingProto::DecimalString => RtDecimalEncoding::String, + DecimalEncodingProto::DecimalBytes => RtDecimalEncoding::Bytes, + }, + include_schema: j.include_schema, + })) + } + Some(protocol::function_stream_graph::format_config::Format::RawString(_)) => { + Ok(RuntimeFormat::RawString) + } + Some(protocol::function_stream_graph::format_config::Format::RawBytes(_)) => { + Ok(RuntimeFormat::RawBytes) + } + None => bail!("FormatConfig has no format variant set"), + } +} + +fn proto_bad_data_to_runtime(policy: i32) -> RtBadDataPolicy { + match BadDataPolicy::try_from(policy) { + Ok(BadDataPolicy::BadDataDrop) => RtBadDataPolicy::Drop, + _ => RtBadDataPolicy::Fail, + } +} + +pub struct MqttSourceDispatcher; + +impl OperatorConstructor for MqttSourceDispatcher { + fn with_config(&self, payload: &[u8], _registry: Arc) -> Result { + let op = ConnectorOp::decode(payload).context("Failed to decode ConnectorOp protobuf")?; + + let fs_schema = op + .fs_schema + .as_ref() + .map(|fs| FsSchema::try_from(fs.clone())) + .transpose() + .map_err(|e| anyhow::anyhow!("{e}"))?; + + match op.config { + Some(Config::MqttSource(ref cfg)) => { + Self::build_mqtt_source(&op.name, cfg, fs_schema) + } + Some(_) => bail!( + "ConnectorOp '{}': received non-mqtt connector config for MQTT dispatcher", + op.name + ), + None => bail!("ConnectorOp '{}' has no configuration payload", op.name), + } + } +} + +impl MqttSourceDispatcher { + fn build_mqtt_source( + _name: &str, + cfg: &MqttSourceConfig, + fs_schema: Option, + ) -> Result { + info!( + topic = %cfg.topic, + host = %cfg.host, + port = cfg.port, + "Constructing MQTT Source" + ); + + let fs = fs_schema.context("fs_schema is required for MQTT Source")?; + let runtime_format = proto_format_to_runtime(&cfg.format)?; + let bad_data = proto_bad_data_to_runtime(cfg.bad_data_policy); + + let deserializer = Box::new(BufferedDeserializer::new( + runtime_format, + fs.schema.clone(), + bad_data, + DEFAULT_SOURCE_BATCH_SIZE, + )); + + let port = cfg.port.clamp(1, u16::MAX as u32) as u16; + + let source_op = MqttSourceOperator::new( + cfg.topic.clone(), + cfg.host.clone(), + port, + cfg.client_id.clone(), + cfg.username.clone(), + cfg.password.clone(), + proto_qos_to_rumqtt(cfg.qos), + cfg.clean_session, + cfg.keep_alive_secs as u64, + vec![], + deserializer, + ); + + Ok(ConstructedOperator::Source(Box::new(source_op))) + } +} diff --git a/src/streaming_runtime_operators/src/factory/connector/robot_bag.rs b/src/streaming_runtime_operators/src/factory/connector/robot_bag.rs new file mode 100644 index 00000000..85c78894 --- /dev/null +++ b/src/streaming_runtime_operators/src/factory/connector/robot_bag.rs @@ -0,0 +1,134 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use anyhow::{Context, Result, bail}; +use prost::Message; +use std::sync::Arc; +use std::time::Duration; +use tracing::info; + +use protocol::function_stream_graph::connector_op::Config; +use protocol::function_stream_graph::{ + BadDataPolicy, ConnectorOp, DecimalEncodingProto, FormatConfig, RobotBagSourceConfig, + TimestampFormatProto, +}; + +use crate::core::api::operator::ConstructedOperator; +use crate::factory::global::Registry; +use crate::factory::operator_constructor::OperatorConstructor; +use crate::format::{ + BadDataPolicy as RtBadDataPolicy, DecimalEncoding as RtDecimalEncoding, + Format as RuntimeFormat, JsonFormat as RuntimeJsonFormat, TimestampFormat as RtTimestampFormat, +}; +use crate::operators::source::batch_buffer::DEFAULT_SOURCE_BATCH_SIZE; +use crate::operators::source::kafka::BufferedDeserializer; +use crate::operators::source::robot_bag::{ + RobotBagSourceOperator, parse_pcd_emit_mode, proto_bag_format_to_runtime, +}; +use crate::sql::common::FsSchema; + +fn proto_format_to_runtime(fmt: &Option) -> Result { + let cfg = fmt.as_ref().context("FormatConfig is required")?; + match &cfg.format { + Some(protocol::function_stream_graph::format_config::Format::Json(j)) => { + Ok(RuntimeFormat::Json(RuntimeJsonFormat { + timestamp_format: match j.timestamp_format() { + TimestampFormatProto::TimestampRfc3339 => RtTimestampFormat::RFC3339, + TimestampFormatProto::TimestampUnixMillis => RtTimestampFormat::UnixMillis, + }, + decimal_encoding: match j.decimal_encoding() { + DecimalEncodingProto::DecimalNumber => RtDecimalEncoding::Number, + DecimalEncodingProto::DecimalString => RtDecimalEncoding::String, + DecimalEncodingProto::DecimalBytes => RtDecimalEncoding::Bytes, + }, + include_schema: j.include_schema, + })) + } + Some(protocol::function_stream_graph::format_config::Format::RawString(_)) => { + Ok(RuntimeFormat::RawString) + } + Some(protocol::function_stream_graph::format_config::Format::RawBytes(_)) => { + Ok(RuntimeFormat::RawBytes) + } + None => bail!("FormatConfig has no format variant set"), + } +} + +fn proto_bad_data_to_runtime(policy: i32) -> RtBadDataPolicy { + match BadDataPolicy::try_from(policy) { + Ok(BadDataPolicy::BadDataDrop) => RtBadDataPolicy::Drop, + _ => RtBadDataPolicy::Fail, + } +} + +pub struct RobotBagSourceDispatcher; + +impl OperatorConstructor for RobotBagSourceDispatcher { + fn with_config(&self, payload: &[u8], _registry: Arc) -> Result { + let op = ConnectorOp::decode(payload).context("Failed to decode ConnectorOp protobuf")?; + + let fs_schema = op + .fs_schema + .as_ref() + .map(|fs| FsSchema::try_from(fs.clone())) + .transpose() + .map_err(|e| anyhow::anyhow!("{e}"))?; + + match op.config { + Some(Config::RobotBagSource(ref cfg)) => { + Self::build_robot_bag_source(&op.name, cfg, fs_schema) + } + Some(_) => bail!( + "ConnectorOp '{}': received non-robot-bag config for robot-bag dispatcher", + op.name + ), + None => bail!("ConnectorOp '{}' has no configuration payload", op.name), + } + } +} + +impl RobotBagSourceDispatcher { + fn build_robot_bag_source( + _name: &str, + cfg: &RobotBagSourceConfig, + fs_schema: Option, + ) -> Result { + info!(path = %cfg.path, "Constructing robot-bag Source"); + + let fs = fs_schema.context("fs_schema is required for robot-bag Source")?; + let runtime_format = proto_format_to_runtime(&cfg.format)?; + let bad_data = proto_bad_data_to_runtime(cfg.bad_data_policy); + + let deserializer = Box::new(BufferedDeserializer::new( + runtime_format, + fs.schema.clone(), + bad_data, + DEFAULT_SOURCE_BATCH_SIZE, + )); + + let source_op = RobotBagSourceOperator::new( + cfg.path.clone(), + proto_bag_format_to_runtime(cfg.bag_format), + cfg.topic.clone(), + parse_pcd_emit_mode(if cfg.pcd_emit_mode.is_empty() { + None + } else { + Some(cfg.pcd_emit_mode.as_str()) + })?, + Duration::from_millis(cfg.replay_interval_ms), + cfg.loop_replay, + deserializer, + ); + + Ok(ConstructedOperator::Source(Box::new(source_op))) + } +} diff --git a/src/streaming_runtime_operators/src/factory/connector/ros.rs b/src/streaming_runtime_operators/src/factory/connector/ros.rs new file mode 100644 index 00000000..ab4725bb --- /dev/null +++ b/src/streaming_runtime_operators/src/factory/connector/ros.rs @@ -0,0 +1,126 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use anyhow::{Context, Result, bail}; +use prost::Message; +use std::sync::Arc; +use tracing::info; + +use protocol::function_stream_graph::connector_op::Config; +use protocol::function_stream_graph::{ + BadDataPolicy, ConnectorOp, DecimalEncodingProto, FormatConfig, RosSourceConfig, + TimestampFormatProto, +}; + +use crate::core::api::operator::ConstructedOperator; +use crate::factory::global::Registry; +use crate::factory::operator_constructor::OperatorConstructor; +use crate::format::{ + BadDataPolicy as RtBadDataPolicy, DecimalEncoding as RtDecimalEncoding, + Format as RuntimeFormat, JsonFormat as RuntimeJsonFormat, TimestampFormat as RtTimestampFormat, +}; +use crate::operators::source::batch_buffer::DEFAULT_SOURCE_BATCH_SIZE; +use crate::operators::source::kafka::BufferedDeserializer; +use crate::operators::source::ros::RosSourceOperator; +use crate::sql::common::FsSchema; + +fn proto_format_to_runtime(fmt: &Option) -> Result { + let cfg = fmt.as_ref().context("FormatConfig is required")?; + match &cfg.format { + Some(protocol::function_stream_graph::format_config::Format::Json(j)) => { + Ok(RuntimeFormat::Json(RuntimeJsonFormat { + timestamp_format: match j.timestamp_format() { + TimestampFormatProto::TimestampRfc3339 => RtTimestampFormat::RFC3339, + TimestampFormatProto::TimestampUnixMillis => RtTimestampFormat::UnixMillis, + }, + decimal_encoding: match j.decimal_encoding() { + DecimalEncodingProto::DecimalNumber => RtDecimalEncoding::Number, + DecimalEncodingProto::DecimalString => RtDecimalEncoding::String, + DecimalEncodingProto::DecimalBytes => RtDecimalEncoding::Bytes, + }, + include_schema: j.include_schema, + })) + } + Some(protocol::function_stream_graph::format_config::Format::RawString(_)) => { + Ok(RuntimeFormat::RawString) + } + Some(protocol::function_stream_graph::format_config::Format::RawBytes(_)) => { + Ok(RuntimeFormat::RawBytes) + } + None => bail!("FormatConfig has no format variant set"), + } +} + +fn proto_bad_data_to_runtime(policy: i32) -> RtBadDataPolicy { + match BadDataPolicy::try_from(policy) { + Ok(BadDataPolicy::BadDataDrop) => RtBadDataPolicy::Drop, + _ => RtBadDataPolicy::Fail, + } +} + +pub struct RosSourceDispatcher; + +impl OperatorConstructor for RosSourceDispatcher { + fn with_config(&self, payload: &[u8], _registry: Arc) -> Result { + let op = ConnectorOp::decode(payload).context("Failed to decode ConnectorOp protobuf")?; + + let fs_schema = op + .fs_schema + .as_ref() + .map(|fs| FsSchema::try_from(fs.clone())) + .transpose() + .map_err(|e| anyhow::anyhow!("{e}"))?; + + match op.config { + Some(Config::RosSource(ref cfg)) => Self::build_ros_source(&op.name, cfg, fs_schema), + Some(_) => bail!( + "ConnectorOp '{}': received non-ros connector config for ROS dispatcher", + op.name + ), + None => bail!("ConnectorOp '{}' has no configuration payload", op.name), + } + } +} + +impl RosSourceDispatcher { + fn build_ros_source( + _name: &str, + cfg: &RosSourceConfig, + fs_schema: Option, + ) -> Result { + info!( + url = %cfg.url, + topic = %cfg.topic, + "Constructing ROS Source (rosbridge)" + ); + + let fs = fs_schema.context("fs_schema is required for ROS Source")?; + let runtime_format = proto_format_to_runtime(&cfg.format)?; + let bad_data = proto_bad_data_to_runtime(cfg.bad_data_policy); + + let deserializer = Box::new(BufferedDeserializer::new( + runtime_format, + fs.schema.clone(), + bad_data, + DEFAULT_SOURCE_BATCH_SIZE, + )); + + let source_op = RosSourceOperator::new( + cfg.url.clone(), + cfg.topic.clone(), + cfg.message_field.clone(), + deserializer, + ); + + Ok(ConstructedOperator::Source(Box::new(source_op))) + } +} diff --git a/src/streaming_runtime_operators/src/format/encoder.rs b/src/streaming_runtime_operators/src/format/encoder.rs index 198b06b2..5e6eb780 100644 --- a/src/streaming_runtime_operators/src/format/encoder.rs +++ b/src/streaming_runtime_operators/src/format/encoder.rs @@ -57,10 +57,22 @@ impl FormatEncoder { if batches.is_empty() { return Ok(Vec::new()); } + Self::encode_parquet_with_schema(batches, batches[0].schema(), compression) + } + + /// Encode with an explicit schema and larger row groups for throughput. + pub fn encode_parquet_with_schema( + batches: &[RecordBatch], + schema: std::sync::Arc, + compression: Compression, + ) -> Result> { + if batches.is_empty() { + return Ok(Vec::new()); + } - let schema = batches[0].schema(); let props = WriterProperties::builder() .set_compression(compression) + .set_max_row_group_size(128 * 1024 * 1024) .build(); let mut cursor = Cursor::new(Vec::new()); let mut writer = ArrowWriter::try_new(&mut cursor, schema, Some(props)) diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/delta_writer.rs b/src/streaming_runtime_operators/src/operators/sink/delta/delta_writer.rs index 05a6313f..cefe08cd 100644 --- a/src/streaming_runtime_operators/src/operators/sink/delta/delta_writer.rs +++ b/src/streaming_runtime_operators/src/operators/sink/delta/delta_writer.rs @@ -103,6 +103,9 @@ impl DeltaSink { } let record_count: u64 = batches.iter().map(|b| b.num_rows() as u64).sum(); + if record_count == 0 { + return Ok(None); + } let encoded = encode_in_background(batches.to_vec(), format, compression, parquet_write_schema) @@ -186,6 +189,9 @@ impl DeltaLocalSink { } let record_count: u64 = batches.iter().map(|b| b.num_rows() as u64).sum(); + if record_count == 0 { + return Ok(None); + } let encoded = encode_in_background(batches.to_vec(), format, compression, parquet_write_schema) @@ -244,7 +250,7 @@ async fn encode_in_background( let start_cpu = Instant::now(); let encoded = tokio::task::spawn_blocking(move || { let batches = if format == DeltaFormat::Parquet { - if let Some(schema) = parquet_write_schema { + if let Some(ref schema) = parquet_write_schema { cast_batches_for_delta_write(&owned_batches, schema.as_ref()) .map_err(|e| e.to_string())? } else { @@ -257,7 +263,16 @@ async fn encode_in_background( match format { DeltaFormat::Csv => FormatEncoder::encode_csv(&batches).map_err(|e| e.to_string()), DeltaFormat::Parquet => { - FormatEncoder::encode_parquet(&batches, compression).map_err(|e| e.to_string()) + if let Some(ref schema) = parquet_write_schema { + FormatEncoder::encode_parquet_with_schema( + &batches, + Arc::clone(schema), + compression, + ) + .map_err(|e| e.to_string()) + } else { + FormatEncoder::encode_parquet(&batches, compression).map_err(|e| e.to_string()) + } } DeltaFormat::JsonL => FormatEncoder::encode_jsonl(&batches).map_err(|e| e.to_string()), DeltaFormat::Avro => FormatEncoder::encode_avro(&batches).map_err(|e| e.to_string()), @@ -286,12 +301,12 @@ pub fn build_s3_object_store( let retry_config = RetryConfig { backoff: BackoffConfig { - init_backoff: Duration::from_millis(200), - max_backoff: Duration::from_secs(5), + init_backoff: Duration::from_millis(100), + max_backoff: Duration::from_secs(3), base: 2.0, }, - max_retries: 5, - retry_timeout: Duration::from_secs(30), + max_retries: 3, + retry_timeout: Duration::from_secs(120), }; let mut builder = AmazonS3Builder::new() diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs index 2e7baba8..d4f2fcdd 100644 --- a/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs @@ -29,11 +29,12 @@ use std::sync::Once; use arrow_array::RecordBatch; use arrow_schema::Schema as ArrowSchema; use async_trait::async_trait; +use arrow::compute::concat_batches; use delta_commit::{ DeltaCommitStrategy, DeltaTableCommitter, build_delta_storage_options, resolve_delta_table_uri, }; use parquet::basic::Compression; -use tracing::{debug, error, info, instrument, warn}; +use tracing::{error, info, instrument, warn}; use url::Url; use crate::core::StreamOutput; @@ -46,7 +47,22 @@ use crate::sql::common::{CheckpointBarrier, Watermark}; use delta_writer::{DeltaLocalSink, DeltaSink, preflight_s3_bucket}; -const DEFAULT_MAX_BUFFER_BYTES: usize = 64 * 1024 * 1024; +const DEFAULT_MAX_BUFFER_BYTES: usize = 256 * 1024 * 1024; + +fn pending_row_count(batches: &[RecordBatch]) -> usize { + batches.iter().map(|b| b.num_rows()).sum() +} + +/// Merge many small stream batches into one Parquet row group where possible. +fn merge_pending_for_write(batches: Vec) -> Result, DeltaSinkError> { + if batches.len() <= 1 { + return Ok(batches); + } + let schema = batches[0].schema(); + let merged = concat_batches(&schema, &batches) + .map_err(|e| DeltaSinkError::Config(format!("failed to merge pending batches: {e}")))?; + Ok(vec![merged]) +} fn ensure_delta_handlers_registered() { static REGISTER: Once = Once::new(); @@ -172,7 +188,9 @@ impl DeltaSinkOperator { epoch: u64, subtask_idx: usize, ) -> Result<(), DeltaSinkError> { - if self.pending.is_empty() { + if self.pending.is_empty() || pending_row_count(&self.pending) == 0 { + self.pending.clear(); + self.pending_bytes = 0; return Ok(()); } @@ -181,7 +199,7 @@ impl DeltaSinkOperator { .expect("batch has no user columns after removing streaming system columns") }); - let batches = std::mem::take(&mut self.pending); + let batches = merge_pending_for_write(std::mem::take(&mut self.pending))?; self.pending_bytes = 0; let compression = self.parquet_compression; @@ -292,26 +310,14 @@ impl Operator for DeltaSinkOperator { ctx: &mut TaskContext, _collector: &mut dyn Collector, ) -> anyhow::Result<()> { - let batch_rows = batch.num_rows(); - let batch_bytes = batch.get_array_memory_size(); - self.pending_bytes += batch_bytes; - self.pending.push(batch); + if batch.num_rows() == 0 { + return Ok(()); + } - debug!( - table = %self.table_name, - subtask_idx = ctx.subtask_index, - batch_rows, - batch_bytes, - pending_batches = self.pending.len(), - pending_bytes = self.pending_bytes, - "delta sink received data" - ); + self.pending_bytes += batch.get_array_memory_size(); + self.pending.push(batch); if self.pending_bytes > self.early_flush_threshold_bytes { - debug!( - bytes = self.pending_bytes, - "memory high-watermark breached, early flush (commit deferred to checkpoint)" - ); self.flush_data_file(0, ctx.subtask_index as usize).await?; } Ok(()) diff --git a/src/streaming_runtime_operators/src/operators/source/batch_buffer.rs b/src/streaming_runtime_operators/src/operators/source/batch_buffer.rs new file mode 100644 index 00000000..b958ad34 --- /dev/null +++ b/src/streaming_runtime_operators/src/operators/source/batch_buffer.rs @@ -0,0 +1,24 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Shared source batching defaults (Kafka, MQTT, HTTP, ROS, …). + +use std::time::Duration; + +/// Rows buffered before emitting a `RecordBatch`. +pub const DEFAULT_SOURCE_BATCH_SIZE: usize = 4096; + +/// Max wait when the buffer is non-empty but below `DEFAULT_SOURCE_BATCH_SIZE`. +pub const MAX_BATCH_LINGER_TIME: Duration = Duration::from_millis(20); + +/// Poll/read timeout in the hot loop; keep small to avoid idle sleeps between messages. +pub const SOURCE_POLL_TIMEOUT: Duration = Duration::from_millis(10); diff --git a/src/streaming_runtime_operators/src/operators/source/http/mod.rs b/src/streaming_runtime_operators/src/operators/source/http/mod.rs new file mode 100644 index 00000000..c2c701a1 --- /dev/null +++ b/src/streaming_runtime_operators/src/operators/source/http/mod.rs @@ -0,0 +1,449 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! HTTP source for robot / REST telemetry. +//! +//! - **poll** (Flink `url` + `scan.interval`): periodic GET/POST to robot APIs. +//! - **webhook** (`http.listen.port`): robots POST JSON payloads to Function Stream. + +use anyhow::{Context as _, Result, anyhow}; +use async_trait::async_trait; +use axum::{ + Router, + body::Bytes, + extract::State, + http::StatusCode, + routing::post, +}; +use governor::{DefaultDirectRateLimiter, Quota, RateLimiter as GovernorRateLimiter}; +use reqwest::{Client, Method}; +use std::collections::HashMap; +use std::num::NonZeroU32; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use tokio::sync::mpsc; +use tracing::{debug, error, info, warn}; + +use crate::core::api::context::TaskContext; +use crate::core::api::source::{SourceCheckpointReport, SourceEvent, SourceOperator}; +use crate::operators::source::kafka::BatchDeserializer; +use crate::sql::common::CheckpointBarrier; + +const HTTP_WAIT_TIMEOUT: Duration = Duration::from_millis(100); +const MAX_BATCH_LINGER_TIME: Duration = Duration::from_millis(500); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HttpSourceMode { + Poll, + Webhook, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HttpResponseSplit { + Single, + Ndjson, + JsonArray, +} + +pub fn proto_mode_to_runtime(mode: i32) -> HttpSourceMode { + use protocol::function_stream_graph::HttpSourceMode as ProtoMode; + match ProtoMode::try_from(mode) { + Ok(ProtoMode::HttpSourceWebhook) => HttpSourceMode::Webhook, + _ => HttpSourceMode::Poll, + } +} + +pub fn proto_split_to_runtime(split: i32) -> HttpResponseSplit { + use protocol::function_stream_graph::HttpResponseSplit as ProtoSplit; + match ProtoSplit::try_from(split) { + Ok(ProtoSplit::HttpResponseNdjson) => HttpResponseSplit::Ndjson, + Ok(ProtoSplit::HttpResponseJsonArray) => HttpResponseSplit::JsonArray, + _ => HttpResponseSplit::Single, + } +} + +fn wall_clock_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +struct InboundMessage { + payload: Vec, + timestamp_ms: u64, +} + +fn normalize_webhook_path(path: &str) -> String { + if path.is_empty() || path == "/" { + return "/".to_string(); + } + if path.starts_with('/') { + path.to_string() + } else { + format!("/{path}") + } +} + +fn split_response_body(body: &[u8], split: HttpResponseSplit) -> Result>> { + if body.is_empty() { + return Ok(vec![]); + } + match split { + HttpResponseSplit::Single => Ok(vec![body.to_vec()]), + HttpResponseSplit::Ndjson => Ok(body + .split(|&b| b == b'\n') + .filter(|line| !line.is_empty()) + .map(|line| line.to_vec()) + .collect()), + HttpResponseSplit::JsonArray => { + let value: serde_json::Value = + serde_json::from_slice(body).context("HTTP response is not valid JSON")?; + match value { + serde_json::Value::Array(items) => items + .into_iter() + .map(|v| serde_json::to_vec(&v)) + .collect::, _>>() + .context("failed to encode JSON array element"), + _ => Ok(vec![body.to_vec()]), + } + } + } +} + +async fn webhook_handler( + State(tx): State>, + body: Bytes, +) -> StatusCode { + if body.is_empty() { + return StatusCode::OK; + } + let msg = InboundMessage { + payload: body.to_vec(), + timestamp_ms: wall_clock_ms(), + }; + if tx.send(msg).await.is_err() { + warn!("HTTP webhook channel closed; dropping payload"); + } + StatusCode::OK +} + +pub struct HttpSourceOperator { + pub mode: HttpSourceMode, + pub url: String, + pub method: Method, + pub scan_interval: Duration, + pub request_body: Option, + pub headers: HashMap, + pub listen_host: String, + pub listen_port: u16, + pub webhook_path: String, + pub request_timeout: Duration, + pub response_split: HttpResponseSplit, + pub messages_per_second: Option, + + client: Option, + rate_limiter: Option, + deserializer: Box, + last_flush_time: Instant, + last_poll_time: Instant, + + inbound_rx: Option>, + _server_task: Option>, +} + +impl HttpSourceOperator { + #[allow(clippy::too_many_arguments)] + pub fn new( + mode: HttpSourceMode, + url: String, + method: Method, + scan_interval: Duration, + request_body: Option, + headers: HashMap, + listen_host: String, + listen_port: u16, + webhook_path: String, + request_timeout: Duration, + response_split: HttpResponseSplit, + messages_per_second: Option, + deserializer: Box, + ) -> Self { + Self { + mode, + url, + method, + scan_interval, + request_body, + headers, + listen_host, + listen_port, + webhook_path, + request_timeout, + response_split, + messages_per_second, + client: None, + rate_limiter: None, + deserializer, + last_flush_time: Instant::now(), + last_poll_time: Instant::now() + .checked_sub(scan_interval) + .unwrap_or_else(Instant::now), + inbound_rx: None, + _server_task: None, + } + } + + fn try_emit_buffered_batch(&mut self, reason: &str) -> Result> { + let should_flush_by_size = self.deserializer.should_flush(); + let should_flush_by_time = self.last_flush_time.elapsed() > MAX_BATCH_LINGER_TIME; + + if !self.deserializer.is_empty() + && (should_flush_by_size || should_flush_by_time) + && let Some(batch) = self.deserializer.flush_buffer()? + { + self.last_flush_time = Instant::now(); + debug!( + num_rows = batch.num_rows(), + flush_by_size = should_flush_by_size, + flush_by_time = should_flush_by_time, + reason, + "http source emitting record batch" + ); + return Ok(Some(SourceEvent::Data(batch))); + } + Ok(None) + } + + fn ingest_payloads(&mut self, payloads: Vec>, timestamp_ms: u64) -> Result<()> { + for payload in payloads { + if payload.is_empty() { + continue; + } + self.deserializer + .deserialize_slice(&payload, timestamp_ms, None)?; + } + Ok(()) + } + + async fn poll_http_once(&mut self) -> Result<()> { + let client = self + .client + .as_ref() + .ok_or_else(|| anyhow!("HTTP client not initialized"))?; + + let mut req = client.request(self.method.clone(), &self.url); + for (k, v) in &self.headers { + req = req.header(k.as_str(), v.as_str()); + } + if let Some(body) = &self.request_body { + req = req.body(body.clone()); + } + + let response = req + .send() + .await + .with_context(|| format!("HTTP {} {}", self.method, self.url))?; + + let status = response.status(); + let bytes = response + .bytes() + .await + .context("failed to read HTTP response body")?; + + if !status.is_success() { + warn!( + status = %status, + url = %self.url, + body_len = bytes.len(), + "HTTP poll returned non-success status" + ); + return Ok(()); + } + + debug!( + url = %self.url, + status = %status, + body_len = bytes.len(), + "http source poll response" + ); + + let chunks = split_response_body(&bytes, self.response_split)?; + self.ingest_payloads(chunks, wall_clock_ms())?; + Ok(()) + } + + async fn start_webhook_server(&mut self, ctx: &TaskContext) -> Result<()> { + if ctx.parallelism > 1 { + warn!( + job_id = %ctx.job_id, + subtask = ctx.subtask_index, + parallelism = ctx.parallelism, + "HTTP webhook source with parallelism > 1 will bind duplicate ports; use parallelism = 1" + ); + } + + let path = normalize_webhook_path(&self.webhook_path); + let addr = format!("{}:{}", self.listen_host, self.listen_port); + info!( + addr = %addr, + path = %path, + "Starting HTTP webhook server for robot telemetry" + ); + + let (tx, rx) = mpsc::channel::(4096); + let app = Router::new() + .route(&path, post(webhook_handler)) + .with_state(tx); + + let listener = tokio::net::TcpListener::bind(&addr) + .await + .with_context(|| format!("failed to bind HTTP webhook on {addr}"))?; + + let server = tokio::spawn(async move { + if let Err(e) = axum::serve(listener, app).await { + error!("HTTP webhook server error: {e}"); + } + }); + + self.inbound_rx = Some(rx); + self._server_task = Some(server); + Ok(()) + } + + async fn start_poll_client(&mut self) -> Result<()> { + info!( + url = %self.url, + method = %self.method, + interval_ms = self.scan_interval.as_millis(), + "Starting HTTP poll source for robot API" + ); + let client = Client::builder() + .timeout(self.request_timeout) + .build() + .context("failed to build HTTP client")?; + self.client = Some(client); + Ok(()) + } +} + +#[async_trait] +impl SourceOperator for HttpSourceOperator { + fn name(&self) -> &str { + match self.mode { + HttpSourceMode::Poll => &self.url, + HttpSourceMode::Webhook => &self.webhook_path, + } + } + + async fn on_start(&mut self, ctx: &mut TaskContext) -> Result<()> { + match self.mode { + HttpSourceMode::Poll => self.start_poll_client().await?, + HttpSourceMode::Webhook => self.start_webhook_server(ctx).await?, + } + self.rate_limiter = self.messages_per_second.map(|qps| { + GovernorRateLimiter::direct(Quota::per_second(qps)) + }); + Ok(()) + } + + async fn fetch_next(&mut self, _ctx: &mut TaskContext) -> Result { + loop { + if let Some(event) = self.try_emit_buffered_batch("linger")? { + return Ok(event); + } + + match self.mode { + HttpSourceMode::Webhook => { + let rx = self + .inbound_rx + .as_mut() + .ok_or_else(|| anyhow!("HTTP webhook receiver not initialized"))?; + + match tokio::time::timeout(HTTP_WAIT_TIMEOUT, rx.recv()).await { + Ok(Some(msg)) => { + let chunks = + split_response_body(&msg.payload, self.response_split)?; + self.ingest_payloads(chunks, msg.timestamp_ms)?; + + if let Some(rl) = &self.rate_limiter { + rl.until_ready().await; + } + + if let Some(event) = self.try_emit_buffered_batch("webhook")? { + return Ok(event); + } + } + Ok(None) => { + return Err(anyhow!("HTTP webhook channel closed")); + } + Err(_) => { + if let Some(event) = + self.try_emit_buffered_batch("webhook timeout")? + { + return Ok(event); + } + return Ok(SourceEvent::Idle); + } + } + } + HttpSourceMode::Poll => { + if self.last_poll_time.elapsed() >= self.scan_interval { + if let Err(e) = self.poll_http_once().await { + error!("HTTP poll error: {e:#}"); + return Err(e); + } + self.last_poll_time = Instant::now(); + + if let Some(rl) = &self.rate_limiter { + rl.until_ready().await; + } + + if let Some(event) = self.try_emit_buffered_batch("poll")? { + return Ok(event); + } + } else { + if let Some(event) = + self.try_emit_buffered_batch("poll wait")? + { + return Ok(event); + } + return Ok(SourceEvent::Idle); + } + } + } + } + } + + async fn snapshot_state( + &mut self, + barrier: CheckpointBarrier, + ctx: &mut TaskContext, + ) -> Result { + debug!( + subtask = ctx.subtask_index, + epoch = barrier.epoch, + mode = ?self.mode, + "HTTP source checkpoint (no offset state in v1)" + ); + Ok(SourceCheckpointReport::default()) + } + + async fn on_close(&mut self, _ctx: &mut TaskContext) -> Result<()> { + info!("HTTP source shutting down"); + if let Some(task) = self._server_task.take() { + task.abort(); + } + self.client.take(); + self.inbound_rx.take(); + Ok(()) + } +} diff --git a/src/streaming_runtime_operators/src/operators/source/kafka/mod.rs b/src/streaming_runtime_operators/src/operators/source/kafka/mod.rs index 78423540..67fcf647 100644 --- a/src/streaming_runtime_operators/src/operators/source/kafka/mod.rs +++ b/src/streaming_runtime_operators/src/operators/source/kafka/mod.rs @@ -19,7 +19,6 @@ use anyhow::{Context as _, Result, anyhow}; use arrow_array::RecordBatch; use arrow_schema::SchemaRef; use async_trait::async_trait; -use governor::{DefaultDirectRateLimiter, Quota, RateLimiter as GovernorRateLimiter}; use protocol::storage::{ KafkaPartitionOffset, KafkaSourceSubtaskCheckpoint, SourceCheckpointInfo, source_checkpoint_info, @@ -27,10 +26,11 @@ use protocol::storage::{ use rdkafka::consumer::{CommitMode, Consumer, StreamConsumer}; use rdkafka::{ClientConfig, Message as KMessage, Offset, TopicPartitionList}; use std::collections::HashMap; -use std::num::NonZeroU32; use std::time::{Duration, Instant}; use tracing::{debug, error, info, warn}; +use crate::operators::source::batch_buffer::{MAX_BATCH_LINGER_TIME, SOURCE_POLL_TIMEOUT}; + use crate::core::api::context::TaskContext; use crate::core::api::source::{SourceCheckpointReport, SourceEvent, SourceOffset, SourceOperator}; use crate::format::{BadDataPolicy, DataDeserializer, Format}; @@ -131,9 +131,6 @@ impl SourceOffsetExt for SourceOffset { // ============================================================================ // ============================================================================ -const KAFKA_POLL_TIMEOUT: Duration = Duration::from_millis(100); -const MAX_BATCH_LINGER_TIME: Duration = Duration::from_millis(500); - pub struct KafkaSourceOperator { pub topic: String, pub bootstrap_servers: String, @@ -142,11 +139,9 @@ pub struct KafkaSourceOperator { pub offset_mode: SourceOffset, pub client_configs: HashMap, - pub messages_per_second: NonZeroU32, pub metadata_fields: Vec, consumer: Option, - rate_limiter: Option, deserializer: Box, current_offsets: HashMap, @@ -166,7 +161,6 @@ impl KafkaSourceOperator { group_id_prefix: Option, offset_mode: SourceOffset, client_configs: HashMap, - messages_per_second: NonZeroU32, metadata_fields: Vec, deserializer: Box, ) -> Self { @@ -177,10 +171,8 @@ impl KafkaSourceOperator { group_id_prefix, offset_mode, client_configs, - messages_per_second, metadata_fields, consumer: None, - rate_limiter: None, deserializer, current_offsets: HashMap::new(), is_empty_assignment: false, @@ -238,6 +230,19 @@ impl KafkaSourceOperator { client_config.set(key, value); } + // High-throughput fetch defaults (override via client_configs in WITH clause). + for (k, v) in [ + ("fetch.min.bytes", "65536"), + ("fetch.max.bytes", "52428800"), + ("max.partition.fetch.bytes", "1048576"), + ("queued.min.messages", "500000"), + ("queued.max.messages.kbytes", "65536"), + ] { + if !self.client_configs.contains_key(k) { + client_config.set(k, v); + } + } + let consumer: StreamConsumer = client_config .set("bootstrap.servers", &self.bootstrap_servers) .set("enable.partition.eof", "false") @@ -292,6 +297,20 @@ impl KafkaSourceOperator { self.consumer = Some(consumer); Ok(()) } + + fn try_emit_buffered_batch(&mut self, _reason: &str) -> Result> { + let should_flush_by_size = self.deserializer.should_flush(); + let should_flush_by_time = self.last_flush_time.elapsed() > MAX_BATCH_LINGER_TIME; + + if !self.deserializer.is_empty() + && (should_flush_by_size || should_flush_by_time) + && let Some(batch) = self.deserializer.flush_buffer()? + { + self.last_flush_time = Instant::now(); + return Ok(Some(SourceEvent::Data(batch))); + } + Ok(None) + } } // ============================================================================ @@ -309,9 +328,6 @@ impl SourceOperator for KafkaSourceOperator { async fn on_start(&mut self, ctx: &mut TaskContext) -> Result<()> { self.init_and_assign_consumer(ctx).await?; - self.rate_limiter = Some(GovernorRateLimiter::direct(Quota::per_second( - self.messages_per_second, - ))); Ok(()) } @@ -320,107 +336,80 @@ impl SourceOperator for KafkaSourceOperator { return Ok(SourceEvent::Idle); } - let consumer = self - .consumer - .as_ref() - .ok_or_else(|| anyhow!("Kafka consumer not initialized"))?; - let rate_limiter = self - .rate_limiter - .as_ref() - .ok_or_else(|| anyhow!("rate limiter not initialized"))?; - - match tokio::time::timeout(KAFKA_POLL_TIMEOUT, consumer.recv()).await { - Ok(Ok(msg)) => { - let partition = msg.partition(); - let offset = msg.offset(); - let timestamp = msg.timestamp().to_millis().ok_or_else(|| { - anyhow!("Failed to read timestamp from Kafka record: message has no timestamp") - })?; - - self.current_offsets.insert(partition, offset); - - if let Some(payload) = msg.payload() { - let topic = msg.topic(); - debug!( - topic, - partition, - offset, - payload_bytes = payload.len(), - timestamp_ms = timestamp, - "kafka source consumed message" - ); - - let connector_metadata = if !self.metadata_fields.is_empty() { - let mut meta = HashMap::new(); - for f in &self.metadata_fields { - meta.insert( - f.field_name.as_str(), - match f.key.as_str() { - "key" => FieldValueType::Bytes(msg.key()), - "offset_id" => FieldValueType::Int64(Some(msg.offset())), - "partition" => FieldValueType::Int32(Some(msg.partition())), - "topic" => FieldValueType::String(Some(topic)), - "timestamp" => FieldValueType::Int64(Some(timestamp)), - _ => continue, - }, - ); - } - Some(meta) - } else { - None - }; - - self.deserializer.deserialize_slice( - payload, - timestamp.max(0) as u64, - connector_metadata, - )?; - } else { - debug!( - "Received tombstone message at partition {} offset {}", - partition, offset - ); - } + // Keep polling while the in-memory buffer is filling; returning Idle here would make + // SourceDriver sleep `source_idle_timeout` (50ms) between every single message. + loop { + if let Some(event) = self.try_emit_buffered_batch("linger")? { + return Ok(event); + } - rate_limiter.until_ready().await; - - let should_flush_by_size = self.deserializer.should_flush(); - let should_flush_by_time = self.last_flush_time.elapsed() > MAX_BATCH_LINGER_TIME; - - if !self.deserializer.is_empty() - && (should_flush_by_size || should_flush_by_time) - && let Some(batch) = self.deserializer.flush_buffer()? - { - self.last_flush_time = Instant::now(); - debug!( - num_rows = batch.num_rows(), - num_columns = batch.num_columns(), - flush_by_size = should_flush_by_size, - flush_by_time = should_flush_by_time, - "kafka source emitting record batch" - ); - return Ok(SourceEvent::Data(batch)); + let recv_result = { + let consumer = self + .consumer + .as_ref() + .ok_or_else(|| anyhow!("Kafka consumer not initialized"))?; + tokio::time::timeout(SOURCE_POLL_TIMEOUT, consumer.recv()).await + }; + + match recv_result { + Ok(Ok(msg)) => { + let partition = msg.partition(); + let offset = msg.offset(); + let timestamp = msg.timestamp().to_millis().ok_or_else(|| { + anyhow!( + "Failed to read timestamp from Kafka record: message has no timestamp" + ) + })?; + + self.current_offsets.insert(partition, offset); + + if let Some(payload) = msg.payload() { + let topic = msg.topic(); + + let connector_metadata = if !self.metadata_fields.is_empty() { + let mut meta = HashMap::new(); + for f in &self.metadata_fields { + meta.insert( + f.field_name.as_str(), + match f.key.as_str() { + "key" => FieldValueType::Bytes(msg.key()), + "offset_id" => FieldValueType::Int64(Some(msg.offset())), + "partition" => FieldValueType::Int32(Some(msg.partition())), + "topic" => FieldValueType::String(Some(topic)), + "timestamp" => FieldValueType::Int64(Some(timestamp)), + _ => continue, + }, + ); + } + Some(meta) + } else { + None + }; + + self.deserializer.deserialize_slice( + payload, + timestamp.max(0) as u64, + connector_metadata, + )?; + } + + if let Some(event) = self.try_emit_buffered_batch("batch full")? { + return Ok(event); + } + // Buffer not ready: poll again immediately (do not return Idle). } - - Ok(SourceEvent::Idle) - } - Ok(Err(e)) => { - error!("Kafka recv error: {}", e); - Err(anyhow!("Kafka error: {}", e)) - } - Err(_) => { - if !self.deserializer.is_empty() - && let Some(batch) = self.deserializer.flush_buffer()? - { - self.last_flush_time = Instant::now(); - debug!( - num_rows = batch.num_rows(), - num_columns = batch.num_columns(), - "kafka source emitting record batch (poll timeout flush)" - ); - return Ok(SourceEvent::Data(batch)); + Ok(Err(e)) => { + error!("Kafka recv error: {}", e); + return Err(anyhow!("Kafka error: {}", e)); + } + Err(_) => { + if let Some(event) = + self.try_emit_buffered_batch("poll timeout")? + { + return Ok(event); + } + return Ok(SourceEvent::Idle); } - Ok(SourceEvent::Idle) } } } diff --git a/src/streaming_runtime_operators/src/operators/source/mod.rs b/src/streaming_runtime_operators/src/operators/source/mod.rs index b9574391..b9eb1d5f 100644 --- a/src/streaming_runtime_operators/src/operators/source/mod.rs +++ b/src/streaming_runtime_operators/src/operators/source/mod.rs @@ -10,4 +10,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod batch_buffer; +pub mod http; pub mod kafka; +pub mod mqtt; +pub mod robot_bag; +pub mod ros; diff --git a/src/streaming_runtime_operators/src/operators/source/mqtt/mod.rs b/src/streaming_runtime_operators/src/operators/source/mqtt/mod.rs new file mode 100644 index 00000000..36fdbc11 --- /dev/null +++ b/src/streaming_runtime_operators/src/operators/source/mqtt/mod.rs @@ -0,0 +1,243 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! MQTT source: subscribe on connect, batch deserialize like Kafka source. +//! Checkpoint v1 reports no offset state (at-least-once; use stable `client_id` +//! and `clean_session=false` on the broker if you need redelivery after restart). + +use anyhow::{Context as _, Result, anyhow}; +use async_trait::async_trait; +use rumqttc::{AsyncClient, Event, EventLoop, MqttOptions, Packet, QoS}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; +use tracing::{debug, error, info, warn}; + +use crate::core::api::context::TaskContext; +use crate::core::api::source::{SourceCheckpointReport, SourceEvent, SourceOperator}; +use crate::operators::source::batch_buffer::{MAX_BATCH_LINGER_TIME, SOURCE_POLL_TIMEOUT}; +use crate::operators::source::kafka::BatchDeserializer; +use crate::sql::common::{CheckpointBarrier, MetadataField}; + +pub fn proto_qos_to_rumqtt(qos: u32) -> QoS { + match qos { + 0 => QoS::AtMostOnce, + 2 => QoS::ExactlyOnce, + _ => QoS::AtLeastOnce, + } +} + +fn wall_clock_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +pub struct MqttSourceOperator { + pub topic: String, + pub host: String, + pub port: u16, + pub client_id: Option, + pub username: Option, + pub password: Option, + pub qos: QoS, + pub clean_session: bool, + pub keep_alive_secs: u64, + pub metadata_fields: Vec, + + _client: Option, + eventloop: Option, + deserializer: Box, + last_flush_time: Instant, +} + +impl MqttSourceOperator { + #[allow(clippy::too_many_arguments)] + pub fn new( + topic: String, + host: String, + port: u16, + client_id: Option, + username: Option, + password: Option, + qos: QoS, + clean_session: bool, + keep_alive_secs: u64, + metadata_fields: Vec, + deserializer: Box, + ) -> Self { + Self { + topic, + host, + port, + client_id, + username, + password, + qos, + clean_session, + keep_alive_secs, + metadata_fields, + _client: None, + eventloop: None, + deserializer, + last_flush_time: Instant::now(), + } + } + + fn try_emit_buffered_batch(&mut self, reason: &str) -> Result> { + let should_flush_by_size = self.deserializer.should_flush(); + let should_flush_by_time = self.last_flush_time.elapsed() > MAX_BATCH_LINGER_TIME; + + if !self.deserializer.is_empty() + && (should_flush_by_size || should_flush_by_time) + && let Some(batch) = self.deserializer.flush_buffer()? + { + self.last_flush_time = Instant::now(); + debug!( + num_rows = batch.num_rows(), + num_columns = batch.num_columns(), + flush_by_size = should_flush_by_size, + flush_by_time = should_flush_by_time, + reason, + "mqtt source emitting record batch" + ); + return Ok(Some(SourceEvent::Data(batch))); + } + Ok(None) + } + + async fn connect(&mut self, ctx: &mut TaskContext) -> Result<()> { + if ctx.parallelism > 1 { + warn!( + job_id = %ctx.job_id, + subtask = ctx.subtask_index, + parallelism = ctx.parallelism, + "MQTT source with pipeline parallelism > 1 will duplicate messages across subtasks; use parallelism = 1" + ); + } + + let client_id = self.client_id.clone().unwrap_or_else(|| { + format!("fs-{}-{}-mqtt", ctx.job_id, ctx.subtask_index) + }); + + info!( + host = %self.host, + port = self.port, + topic = %self.topic, + client_id = %client_id, + "Creating MQTT client" + ); + + let mut mqttoptions = MqttOptions::new(client_id, &self.host, self.port); + mqttoptions.set_keep_alive(std::time::Duration::from_secs(self.keep_alive_secs.max(1))); + mqttoptions.set_clean_session(self.clean_session); + if let (Some(user), Some(pass)) = (&self.username, &self.password) { + mqttoptions.set_credentials(user, pass); + } + + let (client, eventloop) = AsyncClient::new(mqttoptions, 100); + client + .subscribe(self.topic.clone(), self.qos) + .await + .with_context(|| format!("Failed to subscribe to MQTT topic '{}'", self.topic))?; + + self._client = Some(client); + self.eventloop = Some(eventloop); + Ok(()) + } +} + +#[async_trait] +impl SourceOperator for MqttSourceOperator { + fn name(&self) -> &str { + &self.topic + } + + async fn on_start(&mut self, ctx: &mut TaskContext) -> Result<()> { + self.connect(ctx).await?; + Ok(()) + } + + async fn fetch_next(&mut self, _ctx: &mut TaskContext) -> Result { + loop { + if let Some(event) = self.try_emit_buffered_batch("linger")? { + return Ok(event); + } + + let poll_result = { + let eventloop = self + .eventloop + .as_mut() + .ok_or_else(|| anyhow!("MQTT event loop not initialized"))?; + tokio::time::timeout(SOURCE_POLL_TIMEOUT, eventloop.poll()).await + }; + + match poll_result { + Ok(Ok(Event::Incoming(Packet::Publish(publish)))) => { + let payload = publish.payload.as_ref(); + let timestamp_ms = wall_clock_ms(); + + if !payload.is_empty() { + debug!( + topic = %publish.topic, + payload_bytes = payload.len(), + qos = ?publish.qos, + "mqtt source consumed message" + ); + let _ = &self.metadata_fields; + self.deserializer.deserialize_slice( + payload, + timestamp_ms, + None, + )?; + } + + if let Some(event) = self.try_emit_buffered_batch("batch full")? { + return Ok(event); + } + } + Ok(Ok(_)) => { + // ConnAck, SubAck, PingResp, etc. + } + Ok(Err(e)) => { + error!("MQTT poll error: {}", e); + return Err(anyhow!("MQTT error: {}", e)); + } + Err(_) => { + if let Some(event) = self.try_emit_buffered_batch("poll timeout")? { + return Ok(event); + } + return Ok(SourceEvent::Idle); + } + } + } + } + + async fn snapshot_state( + &mut self, + barrier: CheckpointBarrier, + ctx: &mut TaskContext, + ) -> Result { + debug!( + subtask = ctx.subtask_index, + epoch = barrier.epoch, + "MQTT source checkpoint (no offset state in v1)" + ); + Ok(SourceCheckpointReport::default()) + } + + async fn on_close(&mut self, _ctx: &mut TaskContext) -> Result<()> { + info!("MQTT source shutting down"); + self._client.take(); + self.eventloop.take(); + Ok(()) + } +} diff --git a/src/streaming_runtime_operators/src/operators/source/robot_bag/mod.rs b/src/streaming_runtime_operators/src/operators/source/robot_bag/mod.rs new file mode 100644 index 00000000..91dbe278 --- /dev/null +++ b/src/streaming_runtime_operators/src/operators/source/robot_bag/mod.rs @@ -0,0 +1,460 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Robot recording file source: MCAP, ROS1 `.bag`, ROS2 `.db3`, PCD, JSONL/NDJSON, CSV. + +mod pcd; +mod pcd_body; +mod ros1_bag; + +use anyhow::{Context as _, Result, anyhow}; +use async_trait::async_trait; +use pcd::{PcdEmitMode, load_pcd_paths, parse_pcd_file, pcd_to_messages}; +use ros1_bag::{is_ros1_bag_file, load_ros1_bag}; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use tracing::{debug, info, warn}; + +use crate::core::api::context::TaskContext; +use crate::core::api::source::{SourceCheckpointReport, SourceEvent, SourceOperator}; +use crate::operators::source::batch_buffer::{MAX_BATCH_LINGER_TIME, SOURCE_POLL_TIMEOUT}; +use crate::operators::source::kafka::BatchDeserializer; +use crate::sql::common::CheckpointBarrier; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RobotBagFormat { + Auto, + Mcap, + Ros1, + Ros2, + Pcd, + Jsonl, + Csv, +} + +pub fn proto_bag_format_to_runtime(fmt: i32) -> RobotBagFormat { + use protocol::function_stream_graph::RobotBagFormat as Proto; + match Proto::try_from(fmt) { + Ok(Proto::RobotBagMcap) => RobotBagFormat::Mcap, + Ok(Proto::RobotBagRos1) => RobotBagFormat::Ros1, + Ok(Proto::RobotBagRos2) => RobotBagFormat::Ros2, + Ok(Proto::RobotBagPcd) => RobotBagFormat::Pcd, + Ok(Proto::RobotBagJsonl) => RobotBagFormat::Jsonl, + Ok(Proto::RobotBagCsv) => RobotBagFormat::Csv, + _ => RobotBagFormat::Auto, + } +} + +fn wall_clock_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +#[derive(Clone)] +struct BagMessage { + payload: Vec, + timestamp_ms: u64, +} + +fn directory_has_pcd_files(dir: &Path) -> bool { + std::fs::read_dir(dir) + .ok() + .into_iter() + .flatten() + .filter_map(|e| e.ok()) + .any(|e| { + e.path() + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("pcd")) + }) +} + +fn detect_format(path: &Path) -> RobotBagFormat { + if path.is_dir() { + if directory_has_pcd_files(path) { + return RobotBagFormat::Pcd; + } + return RobotBagFormat::Ros2; + } + if is_ros1_bag_file(path) { + return RobotBagFormat::Ros1; + } + match path.extension().and_then(|e| e.to_str()).map(|s| s.to_ascii_lowercase()) { + Some(ext) if ext == "mcap" => RobotBagFormat::Mcap, + Some(ext) if ext == "bag" => RobotBagFormat::Ros1, + Some(ext) if ext == "db3" => RobotBagFormat::Ros2, + Some(ext) if ext == "pcd" => RobotBagFormat::Pcd, + Some(ext) if ext == "jsonl" || ext == "ndjson" => RobotBagFormat::Jsonl, + Some(ext) if ext == "csv" => RobotBagFormat::Csv, + _ => RobotBagFormat::Jsonl, + } +} + +fn resolve_format(path: &Path, fmt: RobotBagFormat) -> RobotBagFormat { + if fmt == RobotBagFormat::Auto { + detect_format(path) + } else { + fmt + } +} + +fn find_ros2_db3(path: &Path) -> Result { + if path.is_file() { + return Ok(path.to_path_buf()); + } + if path.is_dir() { + for entry in std::fs::read_dir(path).with_context(|| format!("read dir {}", path.display()))? + { + let p = entry?.path(); + if p.extension().is_some_and(|e| e == "db3") { + return Ok(p); + } + } + } + Err(anyhow!( + "no ROS2 bag .db3 file found under '{}'", + path.display() + )) +} + +fn load_mcap(path: &Path, topic_filter: Option<&str>) -> Result> { + let data = std::fs::read(path).with_context(|| format!("read mcap {}", path.display()))?; + + let mut out = Vec::new(); + for msg in mcap::MessageStream::new(&data).context("mcap message stream")? { + let msg = msg.context("mcap message")?; + let topic = msg.channel.topic.as_str(); + if let Some(filter) = topic_filter + && topic != filter + { + continue; + } + out.push(BagMessage { + payload: msg.data.to_vec(), + timestamp_ms: msg.log_time / 1_000_000, + }); + } + out.sort_by_key(|m| m.timestamp_ms); + info!( + path = %path.display(), + messages = out.len(), + topic_filter = ?topic_filter, + "loaded MCAP robot bag" + ); + Ok(out) +} + +fn load_ros1(path: &Path, topic_filter: Option<&str>) -> Result> { + let raw = load_ros1_bag(path, topic_filter)?; + Ok(raw + .into_iter() + .map(|(payload, timestamp_ms)| BagMessage { + payload, + timestamp_ms, + }) + .collect()) +} + +fn load_ros2_db3(path: &Path, topic_filter: Option<&str>) -> Result> { + let db3 = find_ros2_db3(path)?; + let conn = rusqlite::Connection::open(&db3) + .with_context(|| format!("open ros2 bag sqlite {}", db3.display()))?; + + let mut out = Vec::new(); + if let Some(topic) = topic_filter { + let mut stmt = conn.prepare( + "SELECT m.timestamp, m.data FROM messages m \ + JOIN topics t ON m.topic_id = t.id \ + WHERE t.name = ?1 ORDER BY m.timestamp", + )?; + let rows = stmt.query_map([topic], |row| { + let ts: i64 = row.get(0)?; + let data: Vec = row.get(1)?; + Ok((ts, data)) + })?; + for row in rows { + let (ts, data) = row?; + out.push(BagMessage { + payload: data, + timestamp_ms: (ts / 1_000_000).max(0) as u64, + }); + } + } else { + let mut stmt = + conn.prepare("SELECT timestamp, data FROM messages ORDER BY timestamp")?; + let rows = stmt.query_map([], |row| { + let ts: i64 = row.get(0)?; + let data: Vec = row.get(1)?; + Ok((ts, data)) + })?; + for row in rows { + let (ts, data) = row?; + out.push(BagMessage { + payload: data, + timestamp_ms: (ts / 1_000_000).max(0) as u64, + }); + } + } + + info!( + db3 = %db3.display(), + messages = out.len(), + topic_filter = ?topic_filter, + "loaded ROS2 bag" + ); + Ok(out) +} + +fn load_pcd(path: &Path, emit_mode: PcdEmitMode) -> Result> { + let files = load_pcd_paths(path)?; + let mut out = Vec::new(); + let base_ts = wall_clock_ms(); + + for (file_idx, file_path) in files.iter().enumerate() { + let cloud = parse_pcd_file(file_path)?; + let file_ts = base_ts.saturating_add(file_idx as u64); + let msgs = pcd_to_messages(&cloud, emit_mode, file_ts)?; + info!( + path = %file_path.display(), + points = cloud.header.points, + data_mode = ?cloud.header.data_mode, + emit_mode = ?emit_mode, + messages = msgs.len(), + "loaded PCD point cloud" + ); + for (payload, ts) in msgs { + out.push(BagMessage { + payload, + timestamp_ms: ts, + }); + } + } + Ok(out) +} + +fn load_jsonl(path: &Path) -> Result> { + let file = File::open(path).with_context(|| format!("open jsonl {}", path.display()))?; + let reader = BufReader::new(file); + let mut out = Vec::new(); + for line in reader.lines() { + let line = line?; + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + out.push(BagMessage { + payload: trimmed.as_bytes().to_vec(), + timestamp_ms: wall_clock_ms(), + }); + } + info!(path = %path.display(), messages = out.len(), "loaded JSONL robot log"); + Ok(out) +} + +fn load_csv(path: &Path) -> Result> { + let file = File::open(path).with_context(|| format!("open csv {}", path.display()))?; + let reader = BufReader::new(file); + let mut out = Vec::new(); + let mut first = true; + for line in reader.lines() { + let line = line?; + if first { + first = false; + continue; + } + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + out.push(BagMessage { + payload: trimmed.as_bytes().to_vec(), + timestamp_ms: wall_clock_ms(), + }); + } + info!(path = %path.display(), messages = out.len(), "loaded CSV robot log"); + Ok(out) +} + +fn load_bag_messages( + path: &Path, + format: RobotBagFormat, + topic_filter: Option<&str>, + pcd_emit_mode: PcdEmitMode, +) -> Result> { + let fmt = resolve_format(path, format); + match fmt { + RobotBagFormat::Mcap => load_mcap(path, topic_filter), + RobotBagFormat::Ros1 => load_ros1(path, topic_filter), + RobotBagFormat::Ros2 => load_ros2_db3(path, topic_filter), + RobotBagFormat::Pcd => load_pcd(path, pcd_emit_mode), + RobotBagFormat::Jsonl => load_jsonl(path), + RobotBagFormat::Csv => load_csv(path), + RobotBagFormat::Auto => unreachable!("resolved above"), + } +} + +pub struct RobotBagSourceOperator { + path: PathBuf, + bag_format: RobotBagFormat, + topic_filter: Option, + pcd_emit_mode: PcdEmitMode, + replay_interval: Duration, + loop_replay: bool, + deserializer: Box, + messages: Vec, + cursor: usize, + last_flush_time: Instant, + last_emit_time: Instant, +} + +impl RobotBagSourceOperator { + pub fn new( + path: String, + bag_format: RobotBagFormat, + topic_filter: Option, + pcd_emit_mode: PcdEmitMode, + replay_interval: Duration, + loop_replay: bool, + deserializer: Box, + ) -> Self { + Self { + path: PathBuf::from(path), + bag_format, + topic_filter, + pcd_emit_mode, + replay_interval, + loop_replay, + deserializer, + messages: Vec::new(), + cursor: 0, + last_flush_time: Instant::now(), + last_emit_time: Instant::now(), + } + } + + fn try_emit_buffered_batch(&mut self, reason: &str) -> Result> { + let should_flush_by_size = self.deserializer.should_flush(); + let should_flush_by_time = self.last_flush_time.elapsed() > MAX_BATCH_LINGER_TIME; + + if !self.deserializer.is_empty() + && (should_flush_by_size || should_flush_by_time) + && let Some(batch) = self.deserializer.flush_buffer()? + { + self.last_flush_time = Instant::now(); + debug!(num_rows = batch.num_rows(), reason, "robot-bag emitting batch"); + return Ok(Some(SourceEvent::Data(batch))); + } + Ok(None) + } + + fn reload_messages(&mut self) -> Result<()> { + self.messages = load_bag_messages( + &self.path, + self.bag_format, + self.topic_filter.as_deref(), + self.pcd_emit_mode, + )?; + self.cursor = 0; + Ok(()) + } +} + +#[async_trait] +impl SourceOperator for RobotBagSourceOperator { + fn name(&self) -> &str { + self.path.to_str().unwrap_or("robot-bag") + } + + async fn on_start(&mut self, _ctx: &mut TaskContext) -> Result<()> { + self.reload_messages()?; + if self.messages.is_empty() { + warn!(path = %self.path.display(), "robot bag loaded zero messages"); + } + Ok(()) + } + + async fn fetch_next(&mut self, _ctx: &mut TaskContext) -> Result { + loop { + if let Some(event) = self.try_emit_buffered_batch("linger")? { + return Ok(event); + } + + if self.cursor >= self.messages.len() { + if !self.deserializer.is_empty() + && let Some(batch) = self.deserializer.flush_buffer()? + { + self.last_flush_time = Instant::now(); + return Ok(SourceEvent::Data(batch)); + } + if self.loop_replay { + info!(path = %self.path.display(), "robot bag loop replay"); + self.reload_messages()?; + if self.messages.is_empty() { + return Ok(SourceEvent::Idle); + } + } else { + return Ok(SourceEvent::EndOfStream); + } + } + + if !self.replay_interval.is_zero() + && self.last_emit_time.elapsed() < self.replay_interval + { + tokio::time::sleep(SOURCE_POLL_TIMEOUT).await; + continue; + } + + let msg = self.messages[self.cursor].clone(); + self.cursor += 1; + self.deserializer + .deserialize_slice(&msg.payload, msg.timestamp_ms, None)?; + self.last_emit_time = Instant::now(); + + if let Some(event) = self.try_emit_buffered_batch("batch full")? { + return Ok(event); + } + } + } + + async fn snapshot_state( + &mut self, + barrier: CheckpointBarrier, + ctx: &mut TaskContext, + ) -> Result { + debug!( + subtask = ctx.subtask_index, + epoch = barrier.epoch, + cursor = self.cursor, + "robot-bag checkpoint (file offset v1: cursor only in memory)" + ); + Ok(SourceCheckpointReport::default()) + } + + async fn on_close(&mut self, _ctx: &mut TaskContext) -> Result<()> { + info!(path = %self.path.display(), "robot-bag source closed"); + Ok(()) + } +} + +pub fn parse_pcd_emit_mode(value: Option<&str>) -> Result { + match value.map(|s| s.to_ascii_lowercase()).as_deref() { + None | Some("point") | Some("points") => Ok(PcdEmitMode::Point), + Some("cloud") | Some("file") => Ok(PcdEmitMode::Cloud), + Some(other) => Err(anyhow!( + "invalid pcd.emit.mode '{other}'; expected 'point' or 'cloud'" + )), + } +} diff --git a/src/streaming_runtime_operators/src/operators/source/robot_bag/pcd.rs b/src/streaming_runtime_operators/src/operators/source/robot_bag/pcd.rs new file mode 100644 index 00000000..d8c4ba99 --- /dev/null +++ b/src/streaming_runtime_operators/src/operators/source/robot_bag/pcd.rs @@ -0,0 +1,461 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Strict PCL PCD v0.7 parser (ascii / binary / binary_compressed). + +use anyhow::{Context as _, Result, anyhow, bail}; +use super::pcd_body::{ + append_binary_point_fields, decompress_binary_compressed_body, fill_point_object_from_flat_values, + find_data_body_offset, parse_ascii_rows, PcdDataMode, PcdHeader, +}; +use serde_json::{Map, Value, json}; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PcdEmitMode { + Point, + Cloud, +} + +#[derive(Debug, Clone)] +pub struct PcdPointCloud { + pub header: PcdHeader, + pub source_path: PathBuf, + aos_body: Vec, + ascii_rows: Vec>, +} + +impl PcdPointCloud { + pub fn point_to_json(&self, index: usize) -> Result { + if index >= self.header.points { + bail!("PCD point index {index} out of range (points={})", self.header.points); + } + let mut obj = Map::new(); + obj.insert( + "_source_file".into(), + Value::String( + self.source_path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("") + .to_string(), + ), + ); + obj.insert("_point_index".into(), json!(index)); + + match self.header.data_mode { + PcdDataMode::Ascii => { + fill_point_object_from_flat_values( + &self.header, + &self.ascii_rows[index], + &mut obj, + )?; + } + PcdDataMode::Binary | PcdDataMode::BinaryCompressed => { + append_binary_point_fields(&self.aos_body, &self.header, index, &mut obj)?; + } + } + Ok(Value::Object(obj)) + } + + pub fn cloud_to_json(&self) -> Result { + let mut points = Vec::with_capacity(self.header.points); + for i in 0..self.header.points { + let mut point = self.point_to_json(i)?; + if let Some(map) = point.as_object_mut() { + map.remove("_source_file"); + } + points.push(point); + } + Ok(json!({ + "_source_file": self.source_path.file_name().and_then(|s| s.to_str()).unwrap_or(""), + "width": self.header.width, + "height": self.header.height, + "points": self.header.points, + "fields": self.header.fields, + "viewpoint": self.header.viewpoint, + "data": points, + })) + } +} + +pub fn load_pcd_paths(path: &Path) -> Result> { + if path.is_file() { + return Ok(vec![path.to_path_buf()]); + } + if path.is_dir() { + let mut files: Vec = std::fs::read_dir(path) + .with_context(|| format!("read pcd dir {}", path.display()))? + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| { + p.is_file() + && p.extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("pcd")) + }) + .collect(); + files.sort(); + if files.is_empty() { + bail!("no .pcd files found under '{}'", path.display()); + } + return Ok(files); + } + bail!("pcd path '{}' is neither file nor directory", path.display()) +} + +pub fn parse_pcd_file(path: &Path) -> Result { + let data = std::fs::read(path).with_context(|| format!("read pcd {}", path.display()))?; + let body_offset = find_data_body_offset(&data)?; + let header = parse_pcd_header(&data[..body_offset])?; + let body = &data[body_offset..]; + + let (aos_body, ascii_rows) = match header.data_mode { + PcdDataMode::Ascii => { + let text = std::str::from_utf8(body).context("pcd ascii body must be utf-8")?; + (Vec::new(), parse_ascii_rows(text, &header)?) + } + PcdDataMode::Binary => { + let expected = header.points * header.point_step; + if body.len() < expected { + bail!( + "PCD binary body too short: need {expected} bytes, got {}", + body.len() + ); + } + (body[..expected].to_vec(), Vec::new()) + } + PcdDataMode::BinaryCompressed => { + (decompress_binary_compressed_body(body, &header)?, Vec::new()) + } + }; + + Ok(PcdPointCloud { + header, + source_path: path.to_path_buf(), + aos_body, + ascii_rows, + }) +} + +pub fn pcd_to_messages( + cloud: &PcdPointCloud, + emit_mode: PcdEmitMode, + timestamp_ms: u64, +) -> Result, u64)>> { + match emit_mode { + PcdEmitMode::Point => { + let mut out = Vec::with_capacity(cloud.header.points); + for i in 0..cloud.header.points { + let json = cloud.point_to_json(i)?; + out.push((serde_json::to_vec(&json)?, timestamp_ms)); + } + Ok(out) + } + PcdEmitMode::Cloud => { + let json = cloud.cloud_to_json()?; + Ok(vec![(serde_json::to_vec(&json)?, timestamp_ms)]) + } + } +} + +fn parse_pcd_header(data: &[u8]) -> Result { + let text = std::str::from_utf8(data).context("pcd header must be utf-8")?; + let mut version = None; + let mut fields = None; + let mut sizes = None; + let mut types = None; + let mut counts = None; + let mut width = None; + let mut height = None; + let mut viewpoint = None; + let mut points = None; + let mut data_mode = None; + + for line in text.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let Some((key, value)) = line.split_once(' ') else { + continue; + }; + let key = key.trim().to_ascii_uppercase(); + let value = value.trim(); + match key.as_str() { + "VERSION" => version = Some(value.to_string()), + "FIELDS" => { + fields = Some( + value + .split_whitespace() + .map(|s| s.to_string()) + .collect::>(), + ); + } + "SIZE" => sizes = Some(parse_u32_list(value, "SIZE")?), + "TYPE" => types = Some(parse_type_tokens(value)?), + "COUNT" => counts = Some(parse_u32_list(value, "COUNT")?), + "WIDTH" => width = Some(value.parse::().context("WIDTH")?), + "HEIGHT" => height = Some(value.parse::().context("HEIGHT")?), + "VIEWPOINT" => { + let vals = parse_f64_list(value, "VIEWPOINT")?; + if vals.len() != 7 { + bail!("VIEWPOINT must contain 7 values, got {}", vals.len()); + } + let mut vp = [0.0; 7]; + vp.copy_from_slice(&vals); + viewpoint = Some(vp); + } + "POINTS" => points = Some(value.parse::().context("POINTS")?), + "DATA" => { + data_mode = Some(parse_data_mode(value)?); + break; + } + _ => {} + } + } + + version.ok_or_else(|| anyhow!("PCD header missing VERSION"))?; + let fields = fields.ok_or_else(|| anyhow!("PCD header missing FIELDS"))?; + let sizes = sizes.ok_or_else(|| anyhow!("PCD header missing SIZE"))?; + let types = types.ok_or_else(|| anyhow!("PCD header missing TYPE"))?; + let counts = counts.unwrap_or_else(|| vec![1; fields.len()]); + let width = width.ok_or_else(|| anyhow!("PCD header missing WIDTH"))?; + let height = height.ok_or_else(|| anyhow!("PCD header missing HEIGHT"))?; + let viewpoint = viewpoint.unwrap_or([0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0]); + let points = points.ok_or_else(|| anyhow!("PCD header missing POINTS"))?; + let data_mode = data_mode.ok_or_else(|| anyhow!("PCD header missing DATA"))?; + + if fields.len() != sizes.len() || fields.len() != types.len() || fields.len() != counts.len() + { + bail!( + "PCD FIELDS/SIZE/TYPE/COUNT length mismatch: {}, {}, {}, {}", + fields.len(), + sizes.len(), + types.len(), + counts.len() + ); + } + for (i, ((&size, &count), &ty)) in sizes.iter().zip(&counts).zip(&types).enumerate() { + validate_field_type(size, ty) + .with_context(|| format!("invalid field '{}' at index {}", fields[i], i))?; + if count == 0 { + bail!("PCD COUNT for field '{}' must be >= 1", fields[i]); + } + } + if width == 0 || height == 0 { + bail!("PCD WIDTH/HEIGHT must be > 0"); + } + if width * height != points as u32 { + bail!( + "PCD POINTS ({points}) must equal WIDTH * HEIGHT ({} * {} = {})", + width, + height, + width * height + ); + } + + let mut field_offsets = Vec::with_capacity(fields.len()); + let mut point_step = 0usize; + for (&size, &count) in sizes.iter().zip(&counts) { + field_offsets.push(point_step); + point_step += (size * count) as usize; + } + + Ok(PcdHeader { + fields, + sizes, + types, + counts, + width, + height, + viewpoint, + points, + data_mode, + point_step, + field_offsets, + }) +} + +fn parse_type_tokens(value: &str) -> Result> { + let parts: Vec<&str> = value.split_whitespace().collect(); + if parts.is_empty() { + bail!("PCD TYPE is empty"); + } + if parts.len() > 1 { + Ok(parts.iter().filter_map(|p| p.chars().next()).collect()) + } else { + Ok(parts[0].chars().collect()) + } +} + +fn parse_data_mode(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "ascii" => Ok(PcdDataMode::Ascii), + "binary" => Ok(PcdDataMode::Binary), + "binary_compressed" => Ok(PcdDataMode::BinaryCompressed), + other => bail!( + "unsupported PCD DATA mode '{other}'; expected ascii, binary, or binary_compressed" + ), + } +} + +fn parse_u32_list(value: &str, key: &str) -> Result> { + value + .split_whitespace() + .map(|v| v.parse::().with_context(|| format!("{key} value '{v}'"))) + .collect() +} + +fn parse_f64_list(value: &str, key: &str) -> Result> { + value + .split_whitespace() + .map(|v| v.parse::().with_context(|| format!("{key} value '{v}'"))) + .collect() +} + +fn validate_field_type(size: u32, ty: char) -> Result<()> { + match (ty, size) { + ('I' | 'U', 1 | 2 | 4) | ('F', 4 | 8) => Ok(()), + (other, sz) => bail!("unsupported PCD TYPE '{other}' with SIZE {sz}"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const ASCII_PCD: &str = "\ +# .PCD v0.7 - Point Cloud Data file format +VERSION .7 +FIELDS x y z intensity +SIZE 4 4 4 4 +TYPE F F F F +COUNT 1 1 1 1 +WIDTH 2 +HEIGHT 1 +VIEWPOINT 0 0 0 1 0 0 0 +POINTS 2 +DATA ascii +1.0 2.0 3.0 0.5 +4.0 5.0 6.0 0.8 +"; + + fn write_temp(name: &str, content: impl AsRef<[u8]>) -> PathBuf { + let dir = std::env::temp_dir().join("fs_pcd_test"); + let _ = std::fs::create_dir_all(&dir); + let path = dir.join(name); + std::fs::write(&path, content.as_ref()).expect("write"); + path + } + + #[test] + fn parse_ascii_pcd_header_and_points() { + let path = write_temp("ascii.pcd", ASCII_PCD); + let cloud = parse_pcd_file(&path).expect("parse"); + assert_eq!(cloud.header.points, 2); + assert_eq!(cloud.header.data_mode, PcdDataMode::Ascii); + + let p0 = cloud.point_to_json(0).expect("p0"); + assert_eq!(p0["x"], 1.0); + assert_eq!(p0["y"], 2.0); + assert_eq!(p0["z"], 3.0); + assert_eq!(p0["intensity"], 0.5); + let p1 = cloud.point_to_json(1).expect("p1"); + assert_eq!(p1["x"], 4.0); + + let msgs = pcd_to_messages(&cloud, PcdEmitMode::Point, 0).expect("msgs"); + assert_eq!(msgs.len(), 2); + let _ = std::fs::remove_file(path); + } + + #[test] + fn parse_binary_pcd_xyz() { + let mut body: Vec = Vec::new(); + for (x, y, z) in [(1.0_f32, 2.0_f32, 3.0_f32), (4.0_f32, 5.0_f32, 6.0_f32)] { + body.extend_from_slice(&x.to_le_bytes()); + body.extend_from_slice(&y.to_le_bytes()); + body.extend_from_slice(&z.to_le_bytes()); + } + let header = r"# .PCD v0.7 +VERSION .7 +FIELDS x y z +SIZE 4 4 4 +TYPE F F F +COUNT 1 1 1 +WIDTH 2 +HEIGHT 1 +VIEWPOINT 0 0 0 1 0 0 0 +POINTS 2 +DATA binary +"; + let mut file = header.as_bytes().to_vec(); + file.extend(body); + let path = write_temp("binary.pcd", file); + let cloud = parse_pcd_file(&path).expect("parse"); + let p0 = cloud.point_to_json(0).expect("p0"); + assert!((p0["x"].as_f64().unwrap() - 1.0).abs() < 1e-5); + let _ = std::fs::remove_file(path); + } + + #[test] + fn parse_binary_compressed_pcd() { + let points = 64usize; + let mut aos = Vec::with_capacity(points * 12); + for i in 0..points { + let f = i as f32; + aos.extend_from_slice(&f.to_le_bytes()); + aos.extend_from_slice(&(f + 100.0).to_le_bytes()); + aos.extend_from_slice(&(f + 200.0).to_le_bytes()); + } + + let mut soa = vec![0u8; points * 12]; + for field in 0..3usize { + let field_bytes = 4; + let soa_base = field * points * field_bytes; + for pt in 0..points { + let aos_off = pt * 12 + field * field_bytes; + let dst = soa_base + pt * field_bytes; + soa[dst..dst + field_bytes].copy_from_slice(&aos[aos_off..aos_off + field_bytes]); + } + } + + let compressed = lzf::compress(&soa).expect("compress"); + let mut body = Vec::new(); + body.extend_from_slice(&(compressed.len() as u32).to_le_bytes()); + body.extend_from_slice(&(soa.len() as u32).to_le_bytes()); + body.extend_from_slice(&compressed); + + let header = format!( + "# .PCD v0.7\n\ +VERSION .7\n\ +FIELDS x y z\n\ +SIZE 4 4 4\n\ +TYPE F F F\n\ +COUNT 1 1 1\n\ +WIDTH {points}\n\ +HEIGHT 1\n\ +VIEWPOINT 0 0 0 1 0 0 0\n\ +POINTS {points}\n\ +DATA binary_compressed\n" + ); + let mut file = header.into_bytes(); + file.extend(body); + let path = write_temp("compressed.pcd", file); + let cloud = parse_pcd_file(&path).expect("parse"); + let p1 = cloud.point_to_json(1).expect("p1"); + assert!((p1["x"].as_f64().unwrap() - 1.0).abs() < 1e-5); + assert!((p1["z"].as_f64().unwrap() - 201.0).abs() < 1e-5); + + let cloud_msgs = pcd_to_messages(&cloud, PcdEmitMode::Cloud, 0).expect("cloud"); + assert_eq!(cloud_msgs.len(), 1); + let _ = std::fs::remove_file(path); + } +} diff --git a/src/streaming_runtime_operators/src/operators/source/robot_bag/pcd_body.rs b/src/streaming_runtime_operators/src/operators/source/robot_bag/pcd_body.rs new file mode 100644 index 00000000..17ff2b68 --- /dev/null +++ b/src/streaming_runtime_operators/src/operators/source/robot_bag/pcd_body.rs @@ -0,0 +1,230 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use anyhow::{Context as _, Result, bail}; +use byteorder::{ByteOrder, LittleEndian}; +use serde_json::{Map, Value, json}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PcdDataMode { + Ascii, + Binary, + BinaryCompressed, +} + +#[derive(Debug, Clone)] +pub struct PcdHeader { + pub fields: Vec, + pub sizes: Vec, + pub types: Vec, + pub counts: Vec, + pub width: u32, + pub height: u32, + pub viewpoint: [f64; 7], + pub points: usize, + pub data_mode: PcdDataMode, + pub point_step: usize, + pub field_offsets: Vec, +} + +pub(crate) fn find_data_body_offset(data: &[u8]) -> Result { + let mut offset = 0usize; + while offset < data.len() { + let line_start = offset; + let line_end = data[offset..] + .iter() + .position(|&b| b == b'\n') + .map(|pos| line_start + pos) + .unwrap_or(data.len()); + let line_bytes = &data[line_start..line_end]; + let line = std::str::from_utf8(line_bytes).context("pcd header line must be utf-8")?; + let trimmed = line.trim(); + if !trimmed.is_empty() + && !trimmed.starts_with('#') + && trimmed.to_ascii_uppercase().starts_with("DATA ") + { + return Ok(if line_end < data.len() { + line_end + 1 + } else { + line_end + }); + } + if line_end >= data.len() { + break; + } + offset = line_end + 1; + } + bail!("PCD header missing DATA line") +} + +pub(crate) fn decompress_binary_compressed_body(body: &[u8], header: &PcdHeader) -> Result> { + if body.len() < 8 { + bail!("binary_compressed body too short"); + } + let compressed_size = LittleEndian::read_u32(&body[0..4]) as usize; + let uncompressed_size = LittleEndian::read_u32(&body[4..8]) as usize; + if body.len() < 8 + compressed_size { + bail!( + "binary_compressed truncated: need {} bytes, got {}", + 8 + compressed_size, + body.len() + ); + } + let compressed = &body[8..8 + compressed_size]; + let decompressed = lzf::decompress(compressed, uncompressed_size) + .map_err(|e| anyhow::anyhow!("LZF decompress failed: {e:?}"))?; + if decompressed.len() != uncompressed_size { + bail!( + "LZF decompressed size mismatch: expected {uncompressed_size}, got {}", + decompressed.len() + ); + } + + let expected: usize = header + .sizes + .iter() + .zip(&header.counts) + .map(|(&s, &c)| header.points * (s * c) as usize) + .sum(); + if decompressed.len() != expected { + bail!( + "binary_compressed uncompressed payload size mismatch: expected {expected}, got {}", + decompressed.len() + ); + } + + let mut aos = vec![0u8; header.points * header.point_step]; + for (field_idx, (&size, &count)) in header.sizes.iter().zip(&header.counts).enumerate() { + let field_bytes = (size * count) as usize; + let soa_offset: usize = header + .sizes + .iter() + .zip(&header.counts) + .take(field_idx) + .map(|(&s, &c)| header.points * (s * c) as usize) + .sum(); + let point_offset = header.field_offsets[field_idx]; + for pt in 0..header.points { + let src = soa_offset + pt * field_bytes; + let dst = pt * header.point_step + point_offset; + aos[dst..dst + field_bytes].copy_from_slice(&decompressed[src..src + field_bytes]); + } + } + Ok(aos) +} + +pub(crate) fn parse_ascii_rows(body: &str, header: &PcdHeader) -> Result>> { + let values_per_point: usize = header.counts.iter().map(|c| *c as usize).sum(); + let mut rows = Vec::with_capacity(header.points); + let mut tokens = body.split_whitespace(); + for pt in 0..header.points { + let mut row = Vec::with_capacity(values_per_point); + for _ in 0..values_per_point { + let Some(tok) = tokens.next() else { + bail!("PCD ascii data ended early at point {pt}"); + }; + row.push(tok.parse::().with_context(|| format!("ascii token '{tok}'"))?); + } + rows.push(row); + } + Ok(rows) +} + +pub(crate) fn fill_point_object_from_flat_values( + header: &PcdHeader, + values: &[f64], + obj: &mut Map, +) -> Result<()> { + let mut cursor = 0; + for (field_idx, field_name) in header.fields.iter().enumerate() { + let count = header.counts[field_idx] as usize; + if cursor + count > values.len() { + bail!("PCD ascii row too short for field '{field_name}'"); + } + insert_field_values(obj, field_name, &values[cursor..cursor + count]); + cursor += count; + } + Ok(()) +} + +pub(crate) fn append_binary_point_fields( + body: &[u8], + header: &PcdHeader, + point_index: usize, + obj: &mut Map, +) -> Result<()> { + let start = point_index * header.point_step; + let end = start + header.point_step; + if end > body.len() { + bail!( + "PCD binary data too short for point {point_index}: need {end}, got {}", + body.len() + ); + } + fill_point_object_from_bytes(header, &body[start..end], obj) +} + +fn fill_point_object_from_bytes( + header: &PcdHeader, + point_bytes: &[u8], + obj: &mut Map, +) -> Result<()> { + for (field_idx, field_name) in header.fields.iter().enumerate() { + let offset = header.field_offsets[field_idx]; + let size = header.sizes[field_idx] as usize; + let count = header.counts[field_idx] as usize; + let mut values = Vec::with_capacity(count); + for elem in 0..count { + let start = offset + elem * size; + let end = start + size; + if end > point_bytes.len() { + bail!("PCD binary point truncated at field '{field_name}'"); + } + values.push(decode_field_value( + header.types[field_idx], + size, + &point_bytes[start..end], + )?); + } + insert_field_values(obj, field_name, &values); + } + Ok(()) +} + +pub(crate) fn decode_field_value(ty: char, size: usize, bytes: &[u8]) -> Result { + let v = match (ty, size) { + ('I', 1) => bytes[0] as i8 as f64, + ('I', 2) => LittleEndian::read_i16(bytes) as f64, + ('I', 4) => LittleEndian::read_i32(bytes) as f64, + ('U', 1) => bytes[0] as f64, + ('U', 2) => LittleEndian::read_u16(bytes) as f64, + ('U', 4) => LittleEndian::read_u32(bytes) as f64, + ('F', 4) => LittleEndian::read_f32(bytes) as f64, + ('F', 8) => LittleEndian::read_f64(bytes), + _ => bail!("unsupported PCD field TYPE '{ty}' SIZE {size}"), + }; + Ok(v) +} + +pub(crate) fn insert_field_values(obj: &mut Map, field_name: &str, values: &[f64]) { + match values.len() { + 0 => {} + 1 => { + obj.insert(field_name.to_string(), json!(values[0])); + } + _ => { + for (i, v) in values.iter().enumerate() { + obj.insert(format!("{field_name}_{i}"), json!(v)); + } + } + } +} diff --git a/src/streaming_runtime_operators/src/operators/source/robot_bag/ros1_bag.rs b/src/streaming_runtime_operators/src/operators/source/robot_bag/ros1_bag.rs new file mode 100644 index 00000000..f499aa28 --- /dev/null +++ b/src/streaming_runtime_operators/src/operators/source/robot_bag/ros1_bag.rs @@ -0,0 +1,145 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! ROS1 rosbag (v1/v2) loader using the `rosbag` crate. + +use anyhow::{Context as _, Result, anyhow, bail}; +use rosbag::{ChunkRecord, IndexRecord, MessageRecord, RosBag, record_types::MessageData}; +use std::collections::HashMap; +use std::io::Read; +use std::path::Path; +use tracing::info; + +pub fn is_ros1_bag_file(path: &Path) -> bool { + let Ok(mut file) = std::fs::File::open(path) else { + return false; + }; + let mut buf = [0u8; 13]; + let Ok(n) = file.read(&mut buf) else { + return false; + }; + if n < 13 { + return false; + } + std::str::from_utf8(&buf).ok().is_some_and(|s| { + s.starts_with("#ROSBAG V2.0") || s.starts_with("#ROSBAG V1.02") || s.starts_with("#ROSBAG V1.0") + }) +} + +pub fn load_ros1_bag(path: &Path, topic_filter: Option<&str>) -> Result, u64)>> { + if !path.is_file() { + bail!("ROS1 bag path '{}' must be a .bag file", path.display()); + } + if !is_ros1_bag_file(path) { + bail!( + "file '{}' is not a ROS1 rosbag (expected header '#ROSBAG V2.0' or '#ROSBAG V1.02')", + path.display() + ); + } + + let bag = RosBag::new(path).with_context(|| format!("open ros1 bag {}", path.display()))?; + + let mut conn_topics: HashMap = HashMap::new(); + + for record in bag.index_records() { + match record.context("ros1 bag index record")? { + IndexRecord::Connection(conn) => { + conn_topics.insert( + conn.id, + (conn.topic.to_string(), conn.tp.to_string()), + ); + } + IndexRecord::IndexData(_) | IndexRecord::ChunkInfo(_) => {} + } + } + + let mut messages = Vec::new(); + + for record in bag.chunk_records() { + match record.context("ros1 bag chunk record")? { + ChunkRecord::Chunk(chunk) => { + let mut chunk_messages = Vec::new(); + for msg in chunk.messages() { + match msg.context("ros1 bag message record")? { + MessageRecord::Connection(conn) => { + conn_topics.insert( + conn.id, + (conn.topic.to_string(), conn.tp.to_string()), + ); + } + MessageRecord::MessageData(msg_data) => { + chunk_messages.push(msg_data); + } + } + } + for msg_data in chunk_messages { + push_ros1_message(&mut messages, &conn_topics, msg_data, topic_filter); + } + } + ChunkRecord::IndexData(_) => {} + } + } + + messages.sort_by_key(|m| m.1); + + info!( + path = %path.display(), + messages = messages.len(), + topic_filter = ?topic_filter, + connections = conn_topics.len(), + "loaded ROS1 rosbag" + ); + + if messages.is_empty() && topic_filter.is_some() { + return Err(anyhow!( + "ROS1 bag '{}' has no messages for topic '{}'", + path.display(), + topic_filter.unwrap_or("") + )); + } + + Ok(messages) +} + +fn push_ros1_message( + out: &mut Vec<(Vec, u64)>, + conn_topics: &HashMap, + msg_data: MessageData<'_>, + topic_filter: Option<&str>, +) { + let Some((topic, _)) = conn_topics.get(&msg_data.conn_id) else { + return; + }; + if let Some(filter) = topic_filter + && topic != filter + { + return; + } + out.push((msg_data.data.to_vec(), msg_data.time / 1_000_000)); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reject_non_bag_file() { + let dir = std::env::temp_dir().join("fs_ros1_test"); + let _ = std::fs::create_dir_all(&dir); + let path = dir.join("not_bag.txt"); + std::fs::write(&path, b"hello").expect("write"); + assert!(!is_ros1_bag_file(&path)); + let err = load_ros1_bag(&path, None).unwrap_err(); + assert!(err.to_string().contains("not a ROS1 rosbag")); + let _ = std::fs::remove_file(path); + } +} diff --git a/src/streaming_runtime_operators/src/operators/source/ros/mod.rs b/src/streaming_runtime_operators/src/operators/source/ros/mod.rs new file mode 100644 index 00000000..4a88e073 --- /dev/null +++ b/src/streaming_runtime_operators/src/operators/source/ros/mod.rs @@ -0,0 +1,272 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! ROS source via [rosbridge](https://github.com/RobotWebTools/rosbridge_suite) WebSocket protocol. +//! +//! Connects to `rosbridge_websocket`, subscribes to a ROS topic, extracts the `msg` payload +//! (or a custom JSON field), and deserializes with the configured format. + +use anyhow::{Context as _, Result, anyhow}; +use async_trait::async_trait; +use futures::{SinkExt, StreamExt}; +use serde_json::Value; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; +use tokio::time::timeout; +use tokio_tungstenite::{ + connect_async, + tungstenite::{Message, client::IntoClientRequest}, +}; +use tracing::{debug, error, info, warn}; + +use crate::core::api::context::TaskContext; +use crate::core::api::source::{SourceCheckpointReport, SourceEvent, SourceOperator}; +use crate::operators::source::batch_buffer::{MAX_BATCH_LINGER_TIME, SOURCE_POLL_TIMEOUT}; +use crate::operators::source::kafka::BatchDeserializer; +use crate::sql::common::CheckpointBarrier; + +fn wall_clock_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +fn extract_ros_payload(msg: &Value, message_field: &str) -> Option> { + let payload = msg.get(message_field).unwrap_or(msg); + match payload { + Value::String(s) => Some(s.as_bytes().to_vec()), + Value::Object(_) | Value::Array(_) => serde_json::to_vec(payload).ok(), + Value::Null => None, + other => serde_json::to_vec(other).ok(), + } +} + +pub struct RosSourceOperator { + pub url: String, + pub topic: String, + pub message_field: String, + deserializer: Box, + last_flush_time: Instant, + ws_write: Option< + futures::stream::SplitSink< + tokio_tungstenite::WebSocketStream>, + Message, + >, + >, + ws_read: Option< + futures::stream::SplitStream< + tokio_tungstenite::WebSocketStream>, + >, + >, +} + +impl RosSourceOperator { + pub fn new( + url: String, + topic: String, + message_field: String, + deserializer: Box, + ) -> Self { + Self { + url, + topic, + message_field, + deserializer, + last_flush_time: Instant::now(), + ws_write: None, + ws_read: None, + } + } + + fn try_emit_buffered_batch(&mut self, reason: &str) -> Result> { + let should_flush_by_size = self.deserializer.should_flush(); + let should_flush_by_time = self.last_flush_time.elapsed() > MAX_BATCH_LINGER_TIME; + + if !self.deserializer.is_empty() + && (should_flush_by_size || should_flush_by_time) + && let Some(batch) = self.deserializer.flush_buffer()? + { + self.last_flush_time = Instant::now(); + debug!( + num_rows = batch.num_rows(), + reason, + "ros source emitting record batch" + ); + return Ok(Some(SourceEvent::Data(batch))); + } + Ok(None) + } + + async fn connect_and_subscribe(&mut self) -> Result<()> { + info!( + url = %self.url, + topic = %self.topic, + "Connecting to rosbridge WebSocket" + ); + + let request = self + .url + .as_str() + .into_client_request() + .context("invalid rosbridge WebSocket URL")?; + + let (ws, _) = connect_async(request) + .await + .with_context(|| format!("failed to connect rosbridge at {}", self.url))?; + + let (write, read) = ws.split(); + self.ws_write = Some(write); + self.ws_read = Some(read); + + let subscribe = serde_json::json!({ + "op": "subscribe", + "topic": self.topic, + }); + let write = self + .ws_write + .as_mut() + .ok_or_else(|| anyhow!("rosbridge write half not initialized"))?; + write + .send(Message::Text(subscribe.to_string().into())) + .await + .context("failed to send rosbridge subscribe")?; + + info!(topic = %self.topic, "Subscribed to ROS topic via rosbridge"); + Ok(()) + } + + fn handle_rosbridge_text(&mut self, text: &str) -> Result { + let msg: Value = match serde_json::from_str(text) { + Ok(v) => v, + Err(e) => { + debug!(error = %e, "ignoring non-JSON rosbridge frame"); + return Ok(false); + } + }; + + let op = msg.get("op").and_then(|v| v.as_str()).unwrap_or(""); + if op != "publish" { + return Ok(false); + } + + let topic = msg.get("topic").and_then(|v| v.as_str()).unwrap_or(""); + if topic != self.topic { + return Ok(false); + } + + let Some(payload) = extract_ros_payload(&msg, &self.message_field) else { + return Ok(false); + }; + + debug!( + topic, + payload_bytes = payload.len(), + "ros source received message" + ); + self.deserializer + .deserialize_slice(&payload, wall_clock_ms(), None)?; + Ok(true) + } +} + +#[async_trait] +impl SourceOperator for RosSourceOperator { + fn name(&self) -> &str { + &self.topic + } + + async fn on_start(&mut self, ctx: &mut TaskContext) -> Result<()> { + if ctx.parallelism > 1 { + warn!( + subtask = ctx.subtask_index, + parallelism = ctx.parallelism, + "ROS source with parallelism > 1 duplicates rosbridge subscriptions; use parallelism = 1" + ); + } + self.connect_and_subscribe().await + } + + async fn fetch_next(&mut self, _ctx: &mut TaskContext) -> Result { + loop { + if let Some(event) = self.try_emit_buffered_batch("linger")? { + return Ok(event); + } + + let read_result = { + let read = self + .ws_read + .as_mut() + .ok_or_else(|| anyhow!("rosbridge read half not initialized"))?; + timeout(SOURCE_POLL_TIMEOUT, read.next()).await + }; + + match read_result { + Ok(Some(Ok(Message::Text(text)))) => { + if self.handle_rosbridge_text(text.as_ref())? { + if let Some(event) = self.try_emit_buffered_batch("batch full")? { + return Ok(event); + } + } + } + Ok(Some(Ok(Message::Binary(bin)))) => { + if let Ok(text) = std::str::from_utf8(&bin) { + if self.handle_rosbridge_text(text)? { + if let Some(event) = self.try_emit_buffered_batch("batch full")? { + return Ok(event); + } + } + } + } + Ok(Some(Ok(Message::Ping(_))) | Some(Ok(Message::Pong(_)))) => {} + Ok(Some(Ok(Message::Close(_)))) => { + return Err(anyhow!("rosbridge WebSocket closed")); + } + Ok(Some(Ok(_))) => {} + Ok(Some(Err(e))) => { + error!("rosbridge WebSocket error: {e}"); + return Err(anyhow!("rosbridge error: {e}")); + } + Ok(None) => { + return Err(anyhow!("rosbridge WebSocket stream ended")); + } + Err(_) => { + if let Some(event) = self.try_emit_buffered_batch("poll timeout")? { + return Ok(event); + } + return Ok(SourceEvent::Idle); + } + } + } + } + + async fn snapshot_state( + &mut self, + barrier: CheckpointBarrier, + ctx: &mut TaskContext, + ) -> Result { + debug!( + subtask = ctx.subtask_index, + epoch = barrier.epoch, + "ROS source checkpoint (no offset state in v1)" + ); + Ok(SourceCheckpointReport::default()) + } + + async fn on_close(&mut self, _ctx: &mut TaskContext) -> Result<()> { + info!("ROS source shutting down"); + if let Some(mut write) = self.ws_write.take() { + let _ = write.close().await; + } + self.ws_read.take(); + Ok(()) + } +} From 087b9b993fca72423220326af987bbd35c32f711 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 3 Jun 2026 23:16:46 +0800 Subject: [PATCH 16/16] fix(format): decode full Kafka JSON batches beyond Arrow 1024-row limit Loop Arrow JSON decode/flush and concat batches so source batches up to 4096 rows align with valid_indices, fixing SO101 collector DEGRADED state. Co-authored-by: Cursor --- .../src/format/deserializer.rs | 147 ++++++++++++++++-- 1 file changed, 135 insertions(+), 12 deletions(-) diff --git a/src/streaming_runtime_operators/src/format/deserializer.rs b/src/streaming_runtime_operators/src/format/deserializer.rs index 4058908f..5056b4df 100644 --- a/src/streaming_runtime_operators/src/format/deserializer.rs +++ b/src/streaming_runtime_operators/src/format/deserializer.rs @@ -11,6 +11,7 @@ // limitations under the License. use anyhow::{Context, Result, anyhow}; +use arrow::compute::concat_batches; use arrow_array::builder::{BinaryBuilder, StringBuilder, TimestampNanosecondBuilder}; use arrow_array::{ArrayRef, RecordBatch}; use arrow_json::reader::ReaderBuilder; @@ -108,18 +109,8 @@ impl DataDeserializer { buffer.push(b'\n'); } - let mut decoder = ReaderBuilder::new(self.decoder_schema.clone()) - .with_strict_mode(false) - .build_decoder() - .context("Failed to build Arrow JSON decoder")?; - - decoder - .decode(&buffer) - .context("Arrow batch decoding failed")?; - - let decoded_batch = decoder - .flush()? - .ok_or_else(|| anyhow!("Decoder returned empty batch after successful validation"))?; + let decoded_batch = + decode_ndjson_buffer(&self.decoder_schema, &buffer, valid_indices.len())?; // Step 3: Re-inject Event-Time Column self.rebuild_with_timestamp(decoded_batch, kafka_timestamps_ms, &valid_indices) @@ -217,6 +208,53 @@ impl DataDeserializer { } } +/// Decodes NDJSON bytes into a single [`RecordBatch`], looping decode/flush until the +/// buffer is fully consumed. Arrow's JSON decoder defaults to `batch_size = 1024`, so a +/// single decode+flush is insufficient when the input has more rows than that. +fn decode_ndjson_buffer( + decoder_schema: &SchemaRef, + buffer: &[u8], + expected_rows: usize, +) -> Result { + let mut decoder = ReaderBuilder::new(decoder_schema.clone()) + .with_strict_mode(false) + .with_batch_size(expected_rows.max(1)) + .build_decoder() + .context("Failed to build Arrow JSON decoder")?; + + let mut offset = 0; + let mut batches = Vec::new(); + while offset < buffer.len() { + let decoded = decoder + .decode(&buffer[offset..]) + .context("Arrow batch decoding failed")?; + if decoded == 0 { + return Err(anyhow!( + "Arrow JSON decoder stalled at offset {}/{}", + offset, + buffer.len() + )); + } + offset += decoded; + + while let Some(batch) = decoder.flush()? { + batches.push(batch); + } + } + + if batches.is_empty() { + return Err(anyhow!( + "Decoder returned empty batch after successful validation" + )); + } + + if batches.len() == 1 { + Ok(batches.into_iter().next().unwrap()) + } else { + concat_batches(decoder_schema, &batches).context("Failed to concat decoded JSON batches") + } +} + /// Helper: Strips the specialized timestamp field to allow the raw decoder /// to focus only on payload data. fn schema_without_timestamp(schema: &Schema) -> SchemaRef { @@ -228,3 +266,88 @@ fn schema_without_timestamp(schema: &Schema) -> SchemaRef { .collect::>(); Arc::new(Schema::new_with_metadata(fields, schema.metadata().clone())) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::config::{BadDataPolicy, DecimalEncoding, Format, JsonFormat, TimestampFormat}; + use arrow_schema::{DataType, Field, TimeUnit}; + + fn test_deserializer() -> DataDeserializer { + let schema = Arc::new(Schema::new(vec![ + Field::new( + TIMESTAMP_FIELD, + DataType::Timestamp(TimeUnit::Nanosecond, None), + false, + ), + Field::new("id", DataType::Int64, false), + Field::new("value", DataType::Utf8, false), + ])); + DataDeserializer::new( + Format::Json(JsonFormat { + timestamp_format: TimestampFormat::UnixMillis, + decimal_encoding: DecimalEncoding::String, + include_schema: false, + }), + schema, + BadDataPolicy::Fail, + ) + } + + #[test] + fn decode_json_batch_exceeds_arrow_default_batch_size() { + let deserializer = test_deserializer(); + let row_count = 2500; + let messages: Vec> = (0..row_count) + .map(|i| format!(r#"{{"id":{i},"value":"row-{i}"}}"#).into_bytes()) + .collect(); + let msg_refs: Vec<&[u8]> = messages.iter().map(|m| m.as_slice()).collect(); + let kafka_ts: Vec = (0..row_count).map(|i| 1_700_000_000_000 + i as u64).collect(); + + let batch = deserializer + .deserialize_batch_with_kafka_timestamps(&msg_refs, &kafka_ts) + .expect("batch larger than 1024 rows should decode"); + + assert_eq!(batch.num_rows(), row_count); + } + + #[test] + fn decode_json_batch_at_kafka_source_batch_size() { + let deserializer = test_deserializer(); + let row_count = 4096; + let messages: Vec> = (0..row_count) + .map(|i| format!(r#"{{"id":{i},"value":"row-{i}"}}"#).into_bytes()) + .collect(); + let msg_refs: Vec<&[u8]> = messages.iter().map(|m| m.as_slice()).collect(); + let kafka_ts: Vec = (0..row_count).map(|i| i as u64).collect(); + + let batch = deserializer + .deserialize_batch_with_kafka_timestamps(&msg_refs, &kafka_ts) + .expect("4096-row batch should decode without alignment error"); + + assert_eq!(batch.num_rows(), row_count); + } + + #[test] + fn decode_ndjson_buffer_loops_past_default_batch_size() { + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); + let row_count = 3647; + let mut buffer = Vec::new(); + for i in 0..row_count { + buffer.extend_from_slice(format!(r#"{{"id":{i}}}"#).as_bytes()); + buffer.push(b'\n'); + } + + // Pre-fix behavior: one decode + one flush only yields 1024 rows. + let mut decoder = ReaderBuilder::new(schema.clone()) + .with_strict_mode(false) + .build_decoder() + .unwrap(); + decoder.decode(&buffer).unwrap(); + let single_flush = decoder.flush().unwrap().unwrap(); + assert_eq!(single_flush.num_rows(), 1024); + + let batch = decode_ndjson_buffer(&schema, &buffer, row_count).expect("3647 rows"); + assert_eq!(batch.num_rows(), row_count); + } +}