diff --git a/.gitignore b/.gitignore index 5dffeb0f..62f1dfb1 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,7 @@ python/**/target/ tests/integration/target/ tests/integration/.venv/ tests/integration/install + +# Python bytecode (do not commit) +**/__pycache__/ +*.py[cod] diff --git a/Cargo.lock b/Cargo.lock index 18b01c0d..ea8b82c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -140,12 +140,12 @@ checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" dependencies = [ "bigdecimal", "bon", - "digest", + "digest 0.10.7", "log", "miniz_oxide", "num-bigint", "quad-rand", - "rand 0.9.2", + "rand 0.9.4", "regex-lite", "serde", "serde_bytes", @@ -180,6 +180,12 @@ dependencies = [ "rustversion", ] +[[package]] +name = "array-init" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" + [[package]] name = "arrayref" version = "0.3.9" @@ -215,23 +221,44 @@ dependencies = [ [[package]] name = "arrow" -version = "57.3.0" +version = "57.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bd47f2a6ddc39244bd722a27ee5da66c03369d087b9e024eafdb03e98b98ea7" +dependencies = [ + "arrow-arith 57.3.1", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-cast 57.3.1", + "arrow-csv 57.3.1", + "arrow-data 57.3.1", + "arrow-ipc 57.3.1", + "arrow-json 57.3.1", + "arrow-ord 57.3.1", + "arrow-row 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", + "arrow-string 57.3.1", +] + +[[package]] +name = "arrow" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4754a624e5ae42081f464514be454b39711daae0458906dacde5f4c632f33a8" +checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" dependencies = [ - "arrow-arith 57.3.0", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-cast 57.3.0", - "arrow-csv 57.3.0", - "arrow-data 57.3.0", - "arrow-ipc 57.3.0", - "arrow-json 57.3.0", - "arrow-ord 57.3.0", - "arrow-row 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", - "arrow-string 57.3.0", + "arrow-arith 58.3.0", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-cast 58.3.0", + "arrow-csv 58.3.0", + "arrow-data 58.3.0", + "arrow-ipc 58.3.0", + "arrow-json 58.3.0", + "arrow-ord 58.3.0", + "arrow-row 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", + "arrow-string 58.3.0", ] [[package]] @@ -250,14 +277,28 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "57.3.0" +version = "57.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c7bbd679c5418b8639b92be01f361d60013c4906574b578b77b63c78356594c" +dependencies = [ + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-data 57.3.1", + "arrow-schema 57.3.1", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-arith" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7b3141e0ec5145a22d8694ea8b6d6f69305971c4fa1c1a13ef0195aef2d678b" +checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", "chrono", "num-traits", ] @@ -297,14 +338,14 @@ dependencies = [ [[package]] name = "arrow-array" -version = "57.3.0" +version = "57.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8955af33b25f3b175ee10af580577280b4bd01f7e823d94c7cdef7cf8c9aef" +checksum = "c8a4ab47b3f3eac60f7fd31b81e9028fda018607bcc63451aca4f2b755269862" dependencies = [ "ahash", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", + "arrow-buffer 57.3.1", + "arrow-data 57.3.1", + "arrow-schema 57.3.1", "chrono", "chrono-tz", "half", @@ -314,6 +355,25 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-array" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" +dependencies = [ + "ahash", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + [[package]] name = "arrow-buffer" version = "52.2.0" @@ -338,9 +398,21 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "57.3.0" +version = "57.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d18b89b4c4f4811d0858175e79541fe98e33e18db3b011708bc287b1240593f" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c697ddca96183182f35b3a18e50b9110b11e916d7b7799cbfd4d34662f2c56c2" +checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" dependencies = [ "bytes", "half", @@ -391,16 +463,16 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "57.3.0" +version = "57.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "646bbb821e86fd57189c10b4fcdaa941deaf4181924917b0daa92735baa6ada5" +checksum = "722b5c41dd1d14d0a879a1bce92c6fe33f546101bb2acce57a209825edd075b3" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-ord 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-data 57.3.1", + "arrow-ord 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", "atoi", "base64", "chrono", @@ -411,6 +483,27 @@ dependencies = [ "ryu", ] +[[package]] +name = "arrow-cast" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" +dependencies = [ + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-ord 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", + "atoi", + "base64", + "chrono", + "half", + "lexical-core 1.0.6", + "num-traits", + "ryu", +] + [[package]] name = "arrow-csv" version = "55.2.0" @@ -428,13 +521,28 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "57.3.0" +version = "57.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27ddb80a4848e03b1655af496d5ac2563a779e5742fcb48f2ca2e089c9cd2197" +dependencies = [ + "arrow-array 57.3.1", + "arrow-cast 57.3.1", + "arrow-schema 57.3.1", + "chrono", + "csv", + "csv-core", + "regex", +] + +[[package]] +name = "arrow-csv" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da746f4180004e3ce7b83c977daf6394d768332349d3d913998b10a120b790a" +checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" dependencies = [ - "arrow-array 57.3.0", - "arrow-cast 57.3.0", - "arrow-schema 57.3.0", + "arrow-array 58.3.0", + "arrow-cast 58.3.0", + "arrow-schema 58.3.0", "chrono", "csv", "csv-core", @@ -467,12 +575,25 @@ dependencies = [ [[package]] name = "arrow-data" -version = "57.3.0" +version = "57.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fdd994a9d28e6365aa78e15da3f3950c0fdcea6b963a12fa1c391afb637b304" +checksum = "c1683705c63dcf0d18972759eda48489028cbbff67af7d6bef2c6b7b74ab778a" dependencies = [ - "arrow-buffer 57.3.0", - "arrow-schema 57.3.0", + "arrow-buffer 57.3.1", + "arrow-schema 57.3.1", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-data" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" +dependencies = [ + "arrow-buffer 58.3.0", + "arrow-schema 58.3.0", "half", "num-integer", "num-traits", @@ -508,20 +629,34 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "57.3.0" +version = "57.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf7df950701ab528bf7c0cf7eeadc0445d03ef5d6ffc151eaae6b38a58feff1" +checksum = "8cf72d04c07229fbf4dbebe7145cac37d7cf7ec582fe705c6b92cb314af096ab" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-data 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", "flatbuffers 25.12.19", - "lz4_flex 0.12.1", + "lz4_flex 0.12.2", "zstd", ] +[[package]] +name = "arrow-ipc" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" +dependencies = [ + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", + "flatbuffers 25.12.19", +] + [[package]] name = "arrow-json" version = "55.2.0" @@ -535,7 +670,7 @@ dependencies = [ "arrow-schema 55.2.0", "chrono", "half", - "indexmap 2.13.1", + "indexmap 2.14.0", "lexical-core 1.0.6", "memchr", "num", @@ -546,18 +681,43 @@ dependencies = [ [[package]] name = "arrow-json" -version = "57.3.0" +version = "57.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a84a905f41fedfcd7679813c89a61dc369c0f932b27aa8dcc6aa051cc781a97d" +dependencies = [ + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-cast 57.3.1", + "arrow-data 57.3.1", + "arrow-schema 57.3.1", + "chrono", + "half", + "indexmap 2.14.0", + "itoa", + "lexical-core 1.0.6", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + +[[package]] +name = "arrow-json" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ff8357658bedc49792b13e2e862b80df908171275f8e6e075c460da5ee4bf86" +checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-cast 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-cast 58.3.0", + "arrow-ord 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", "chrono", "half", - "indexmap 2.13.1", + "indexmap 2.14.0", "itoa", "lexical-core 1.0.6", "memchr", @@ -583,15 +743,28 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "57.3.0" +version = "57.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d8f1870e03d4cbed632959498bcc84083b5a24bded52905ae1695bd29da45b" +checksum = "082342947d4e5a2bcccf029a0a0397e21cb3bb8421edd9571d34fb5dd2670256" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-data 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", +] + +[[package]] +name = "arrow-ord" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" +dependencies = [ + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", ] [[package]] @@ -609,14 +782,27 @@ dependencies = [ [[package]] name = "arrow-row" -version = "57.3.0" +version = "57.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a931b520a2a5e22033e01a6f2486b4cdc26f9106b759abeebc320f125e94d7" +dependencies = [ + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-data 57.3.1", + "arrow-schema 57.3.1", + "half", +] + +[[package]] +name = "arrow-row" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18228633bad92bff92a95746bbeb16e5fc318e8382b75619dec26db79e4de4c0" +checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", "half", ] @@ -638,15 +824,26 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "57.3.0" +version = "57.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c872d36b7bf2a6a6a2b40de9156265f0242910791db366a2c17476ba8330d68" +checksum = "e4cf0d4a6609679e03002167a61074a21d7b1ad9ea65e462b2c0a97f8a3b2bc6" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "serde_core", "serde_json", ] +[[package]] +name = "arrow-schema" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" +dependencies = [ + "bitflags 2.11.1", + "serde", + "serde_core", +] + [[package]] name = "arrow-select" version = "52.2.0" @@ -677,15 +874,29 @@ dependencies = [ [[package]] name = "arrow-select" -version = "57.3.0" +version = "57.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b320d86a9806923663bb0fd9baa65ecaba81cb0cd77ff8c1768b9716b4ef891" +dependencies = [ + "ahash", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-data 57.3.1", + "arrow-schema 57.3.1", + "num-traits", +] + +[[package]] +name = "arrow-select" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68bf3e3efbd1278f770d67e5dc410257300b161b93baedb3aae836144edcaf4b" +checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" dependencies = [ "ahash", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", "num-traits", ] @@ -708,15 +919,32 @@ dependencies = [ [[package]] name = "arrow-string" -version = "57.3.0" +version = "57.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b493e99162e5764077e7823e50ba284858d365922631c7aaefe9487b1abd02c2" +dependencies = [ + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-data 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "arrow-string" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85e968097061b3c0e9fe3079cf2e703e487890700546b5b0647f60fca1b5a8d8" +checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", "memchr", "num-traits", "regex", @@ -741,7 +969,7 @@ version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" dependencies = [ - "bzip2", + "bzip2 0.5.2", "flate2", "futures-core", "memchr", @@ -839,9 +1067,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-config" -version = "1.8.14" +version = "1.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a8fc176d53d6fe85017f230405e3255cedb4a02221cb55ed6d76dccbbb099b2" +checksum = "50f156acdd2cf55f5aa53ee416c4ac851cf1222694506c0b1f78c85695e9ca9d" dependencies = [ "aws-credential-types", "aws-runtime", @@ -859,7 +1087,7 @@ dependencies = [ "fastrand", "hex", "http 1.4.0", - "ring", + "sha1", "time", "tokio", "tracing", @@ -869,9 +1097,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.13" +version = "1.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d203b0bf2626dcba8665f5cd0871d7c2c0930223d6b6be9097592fea21242d0" +checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -881,9 +1109,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.16.3" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" dependencies = [ "aws-lc-sys", "zeroize", @@ -891,9 +1119,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.40.0" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" dependencies = [ "cc", "cmake", @@ -903,9 +1131,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.7.1" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede2ddc593e6c8acc6ce3358c28d6677a6dc49b65ba4b37a2befe14a11297e75" +checksum = "5dcd93c82209ac7413532388067dce79be5a8780c1786e5fae3df22e4dee2864" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -926,11 +1154,35 @@ dependencies = [ "uuid", ] +[[package]] +name = "aws-sdk-dynamodb" +version = "1.111.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc418346e3cb248c7d59e642acbcb06488b7c7cd2ba6ebc79e8003f618c60099" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + [[package]] name = "aws-sdk-sso" -version = "1.95.0" +version = "1.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00c5ff27c6ba2cbd95e6e26e2e736676fdf6bcf96495b187733f521cfe4ce448" +checksum = "d69c77aafa20460c68b6b3213c84f6423b6e76dbf89accd3e1789a686ffd9489" dependencies = [ "aws-credential-types", "aws-runtime", @@ -952,9 +1204,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.97.0" +version = "1.100.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d186f1e5a3694a188e5a0640b3115ccc6e084d104e16fd6ba968dca072ffef8" +checksum = "1c7e7b09346d5ca22a2a08267555843a6a0127fb20d8964cb6ecfb8fdb190225" dependencies = [ "aws-credential-types", "aws-runtime", @@ -976,9 +1228,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.99.0" +version = "1.103.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9acba7c62f3d4e2408fa998a3a8caacd8b9a5b5549cf36e2372fbdae329d5449" +checksum = "c2249b81a2e73a8027c41c378463a81ec39b8510f184f2caab87de912af0f49b" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1001,9 +1253,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.4.1" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37411f8e0f4bea0c3ca0958ce7f18f6439db24d555dbd809787262cd00926aa9" +checksum = "68dc0b907359b120170613b5c09ccc61304eac3998ff6274b97d93ee6490115a" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -1012,20 +1264,20 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac", + "hmac 0.13.0", "http 0.2.12", "http 1.4.0", "percent-encoding", - "sha2", + "sha2 0.11.0", "time", "tracing", ] [[package]] name = "aws-smithy-async" -version = "1.2.13" +version = "1.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc50d0f63e714784b84223abd7abbc8577de8c35d699e0edd19f0a88a08ae13" +checksum = "2ffcaf626bdda484571968400c326a244598634dc75fd451325a54ad1a59acfc" dependencies = [ "futures-util", "pin-project-lite", @@ -1034,9 +1286,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.63.5" +version = "0.63.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d619373d490ad70966994801bc126846afaa0d1ee920697a031f0cf63f2568e7" +checksum = "ba1ab2dc1c2c3749ead27180d333c42f11be8b0e934058fb4b2258ee8dbe5231" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -1055,9 +1307,9 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.1.11" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00ccbb08c10f6bcf912f398188e42ee2eab5f1767ce215a02a73bc5df1bbdd95" +checksum = "6a2f165a7feee6f263028b899d0a181987f4fa7179a6411a32a439fba7c5f769" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -1068,38 +1320,40 @@ dependencies = [ "hyper-rustls", "hyper-util", "pin-project-lite", - "rustls", - "rustls-native-certs", + "rustls 0.23.40", + "rustls-native-certs 0.8.3", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tower 0.5.3", "tracing", ] [[package]] name = "aws-smithy-json" -version = "0.62.4" +version = "0.62.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b3a779093e18cad88bbae08dc4261e1d95018c4c5b9356a52bcae7c0b6e9bb" +checksum = "517089205f18ab4adc5a3e02888cb139bbbbb2e168eac9f396216925d1fbeaf5" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", ] [[package]] name = "aws-smithy-observability" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d3f39d5bb871aaf461d59144557f16d5927a5248a983a40654d9cf3b9ba183b" +checksum = "a06c2315d173edbf1920da8ba3a7189695827002e4c0fc961973ab1c54abca9c" dependencies = [ "aws-smithy-runtime-api", ] [[package]] name = "aws-smithy-query" -version = "0.60.14" +version = "0.60.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f76a580e3d8f8961e5d48763214025a2af65c2fa4cd1fb7f270a0e107a71b0" +checksum = "1a56d79744fb3edb5d722ef79d86081e121d3b9422cb209eb03aea6aa4f21ebd" dependencies = [ "aws-smithy-types", "urlencoding", @@ -1107,15 +1361,16 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.10.2" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ccf7f6eba8b2dcf8ce9b74806c6c185659c311665c4bf8d6e71ebd454db6bf" +checksum = "b8e6f5caf6fea86f8c2206541ab5857cfcda9013426cdbe8fa0098b9e2d32182" dependencies = [ "aws-smithy-async", "aws-smithy-http", "aws-smithy-http-client", "aws-smithy-observability", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "bytes", "fastrand", @@ -1132,11 +1387,12 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.11.5" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4af6e5def28be846479bbeac55aa4603d6f7986fc5da4601ba324dd5d377516" +checksum = "dc117c179ecf39a62a0a3f49f600e9ac26a7ad7dd172177999f83933af776c32" dependencies = [ "aws-smithy-async", + "aws-smithy-runtime-api-macros", "aws-smithy-types", "bytes", "http 0.2.12", @@ -1147,15 +1403,38 @@ dependencies = [ "zeroize", ] +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "aws-smithy-schema" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.4.0", +] + [[package]] name = "aws-smithy-types" -version = "1.4.5" +version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ca2734c16913a45343b37313605d84e7d8b34a4611598ce1d25b35860a2bed3" +checksum = "056b66dbce2f81cc0c1e2b05bb402eb58f8a3530479d650efadd5bbae9a4050b" dependencies = [ "base64-simd", "bytes", "bytes-utils", + "futures-core", "http 0.2.12", "http 1.4.0", "http-body 0.4.6", @@ -1168,22 +1447,24 @@ dependencies = [ "ryu", "serde", "time", + "tokio", + "tokio-util", ] [[package]] name = "aws-smithy-xml" -version = "0.60.14" +version = "0.60.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b53543b4b86ed43f051644f704a98c7291b3618b67adf057ee77a366fa52fcaa" +checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "1.3.13" +version = "1.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0470cc047657c6e286346bdf10a8719d26efd6a91626992e0e64481e44323e96" +checksum = "2f4bbcaa9304ea40902d3d5f42a0428d1bd895a2b0f6999436fb279ffddc58ac" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -1206,6 +1487,8 @@ dependencies = [ "http 1.4.0", "http-body 1.0.1", "http-body-util", + "hyper", + "hyper-util", "itoa", "matchit", "memchr", @@ -1214,10 +1497,15 @@ dependencies = [ "pin-project-lite", "rustversion", "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", "sync_wrapper", + "tokio", "tower 0.5.3", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -1238,6 +1526,7 @@ dependencies = [ "sync_wrapper", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -1251,6 +1540,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + [[package]] name = "base64" version = "0.22.1" @@ -1281,6 +1576,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bimap" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" + [[package]] name = "bincode" version = "2.0.1" @@ -1328,7 +1629,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1340,6 +1641,30 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "binrw" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1faf7031c34c71da53eec4e070cf90c3b825729e21ca3aab51b20da4a1d1d9" +dependencies = [ + "array-init", + "binrw_derive", + "bytemuck", +] + +[[package]] +name = "binrw_derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c5eb3446e2f5ea7fa9a6f2cb594648c73bf2dbc60eccf3b2fa41834e5449150" +dependencies = [ + "either", + "owo-colors", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -1348,9 +1673,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] name = "bitmaps" @@ -1388,14 +1713,14 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] name = "blake3" -version = "1.8.4" +version = "1.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" dependencies = [ "arrayref", "arrayvec", @@ -1414,6 +1739,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bon" version = "3.9.1" @@ -1430,7 +1764,7 @@ version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" dependencies = [ - "darling", + "darling 0.23.0", "ident_case", "prettyplease", "proc-macro2", @@ -1469,6 +1803,50 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "buoyant_kernel" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d3ca37afa82755db7b4fd51a4eab9e53eeb0aa1898fae15d373bd6df4bdf0f8" +dependencies = [ + "arrow 58.3.0", + "buoyant_kernel_derive", + "bytes", + "chrono", + "crc", + "futures", + "indexmap 2.14.0", + "itertools 0.14.0", + "object_store 0.13.2", + "parquet 58.3.0", + "percent-encoding", + "rand 0.9.4", + "reqwest 0.13.3", + "roaring", + "rustc_version", + "serde", + "serde_json", + "strum 0.27.2", + "thiserror 2.0.18", + "tokio", + "tracing", + "tracing-subscriber", + "url", + "uuid", + "z85", +] + +[[package]] +name = "buoyant_kernel_derive" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3448e05bba811d98c73a466843abd8c16d0416dc083f92b66847693fcb0714f4" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "bytemuck" version = "1.25.0" @@ -1499,7 +1877,17 @@ dependencies = [ [[package]] name = "bzip2" -version = "0.5.2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" dependencies = [ @@ -1565,7 +1953,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8144c22e24bbcf26ade86cb6501a0916c46b7e4787abdb0045a467eb1645a1d" dependencies = [ "ambient-authority", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -1596,9 +1984,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.59" +version = "1.2.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" dependencies = [ "find-msvc-tools", "jobserver", @@ -1639,6 +2027,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.44" @@ -1676,9 +2075,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", "clap_derive", @@ -1698,9 +2097,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ "heck", "proc-macro2", @@ -1732,6 +2131,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746" + [[package]] name = "cobs" version = "0.3.0" @@ -1747,6 +2152,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "comfy-table" version = "7.2.2" @@ -1773,6 +2188,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -1799,6 +2220,25 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +[[package]] +name = "convert_case" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db05ffb6856bf0ecdf6367558a76a0e8a77b1713044eb92845c692100ed50190" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -1903,7 +2343,7 @@ dependencies = [ "rustc-hash 2.1.2", "serde", "serde_derive", - "sha2", + "sha2 0.10.9", "smallvec", "target-lexicon", "wasmtime-internal-math", @@ -1983,6 +2423,21 @@ version = "0.128.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "903adeaf4938e60209a97b53a2e4326cd2d356aab9764a1934630204bae381c9" +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32c" version = "0.6.8" @@ -2060,7 +2515,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "crossterm_winapi", "document-features", "parking_lot", @@ -2093,6 +2548,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.4.0" @@ -2114,11 +2578,37 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctor" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83cf0d42651b16c6dfe68685716d18480d18a9c39c62d76e8cf3eb6ed5d8bcbf" +dependencies = [ + "ctor-proc-macro", + "dtor", + "link-section", +] + +[[package]] +name = "ctor-proc-macro" +version = "0.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a949c44fcacbbbb7ada007dc7acb34603dd97cd47de5d054f2b6493ecebb483" + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "curl-sys" -version = "0.4.87+curl-8.19.0" +version = "0.4.88+curl-8.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a460380f0ef783703dcbe909107f39c162adeac050d73c850055118b5b6327" +checksum = "644816de6547255eff4e491a1dda1c19b7237f00b62a61e6e64859ce4f2906d0" dependencies = [ "cc", "libc", @@ -2126,7 +2616,27 @@ dependencies = [ "openssl-sys", "pkg-config", "vcpkg", - "windows-sys 0.59.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core 0.21.3", + "darling_macro 0.21.3", ] [[package]] @@ -2135,8 +2645,35 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.23.0", + "darling_macro 0.23.0", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", +] + +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "syn 2.0.117", ] [[package]] @@ -2152,22 +2689,44 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core 0.21.3", + "quote", + "syn 2.0.117", +] + [[package]] name = "darling_macro" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core", + "darling_core 0.23.0", "quote", "syn 2.0.117", ] [[package]] name = "dashmap" -version = "6.1.0" +version = "6.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -2177,6 +2736,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + [[package]] name = "datafusion" version = "48.0.1" @@ -2187,7 +2752,7 @@ dependencies = [ "arrow-schema 55.2.0", "async-trait", "bytes", - "bzip2", + "bzip2 0.5.2", "chrono", "datafusion-catalog 48.0.1", "datafusion-catalog-listing 48.0.1", @@ -2216,10 +2781,10 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.12.5", "parking_lot", - "parquet", - "rand 0.9.2", + "parquet 55.2.0", + "rand 0.9.4", "regex", "sqlparser 0.55.0", "tempfile", @@ -2236,8 +2801,8 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7541353e77dc7262b71ca27be07d8393661737e3a73b5d1b1c6f7d814c64fa2a" dependencies = [ - "arrow 57.3.0", - "arrow-schema 57.3.0", + "arrow 57.3.1", + "arrow-schema 57.3.1", "async-trait", "bytes", "chrono", @@ -2268,9 +2833,9 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.12.5", "parking_lot", - "rand 0.9.2", + "rand 0.9.4", "regex", "sqlparser 0.59.0", "tempfile", @@ -2299,7 +2864,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.12.5", "parking_lot", "tokio", ] @@ -2310,7 +2875,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9997731f90fa5398ef831ad0e69600f92c861b79c0d38bd1a29b6f0e3a0ce4c8" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "async-trait", "dashmap", "datafusion-common 52.5.0", @@ -2324,7 +2889,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.12.5", "parking_lot", "tokio", ] @@ -2347,7 +2912,7 @@ dependencies = [ "datafusion-session 48.0.1", "futures", "log", - "object_store", + "object_store 0.12.5", "tokio", ] @@ -2357,7 +2922,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b30a3dd50dec860c9559275c8d97d9de602e611237a6ecfbda0b3b63b872352" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "async-trait", "datafusion-catalog 52.5.0", "datafusion-common 52.5.0", @@ -2371,7 +2936,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.12.5", ] [[package]] @@ -2385,11 +2950,11 @@ dependencies = [ "base64", "half", "hashbrown 0.14.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "libc", "log", - "object_store", - "parquet", + "object_store 0.12.5", + "parquet 55.2.0", "paste", "recursive", "sqlparser 0.55.0", @@ -2404,15 +2969,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d551054acec0398ca604512310b77ce05c46f66e54b54d48200a686e385cca4e" dependencies = [ "ahash", - "arrow 57.3.0", - "arrow-ipc 57.3.0", + "arrow 57.3.1", + "arrow-ipc 57.3.1", "chrono", "half", "hashbrown 0.16.1", - "indexmap 2.13.1", + "indexmap 2.14.0", "libc", "log", - "object_store", + "object_store 0.12.5", "paste", "sqlparser 0.59.0", "tokio", @@ -2449,7 +3014,7 @@ dependencies = [ "async-compression", "async-trait", "bytes", - "bzip2", + "bzip2 0.5.2", "chrono", "datafusion-common 48.0.1", "datafusion-common-runtime 48.0.1", @@ -2464,9 +3029,9 @@ dependencies = [ "glob", "itertools 0.14.0", "log", - "object_store", - "parquet", - "rand 0.9.2", + "object_store 0.12.5", + "parquet 55.2.0", + "rand 0.9.4", "tempfile", "tokio", "tokio-util", @@ -2481,7 +3046,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27d2668f51b3b30befae2207472569e37807fdedd1d14da58acc6f8ca6257eae" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "async-trait", "bytes", "chrono", @@ -2498,8 +3063,8 @@ dependencies = [ "glob", "itertools 0.14.0", "log", - "object_store", - "rand 0.9.2", + "object_store 0.12.5", + "rand 0.9.4", "tokio", "url", ] @@ -2510,8 +3075,8 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e02e1b3e3a8ec55f1f62de4252b0407c8567363d056078769a197e24fc834a0f" dependencies = [ - "arrow 57.3.0", - "arrow-ipc 57.3.0", + "arrow 57.3.1", + "arrow-ipc 57.3.1", "async-trait", "bytes", "datafusion-common 52.5.0", @@ -2524,7 +3089,7 @@ dependencies = [ "datafusion-session 52.5.0", "futures", "itertools 0.14.0", - "object_store", + "object_store 0.12.5", "tokio", ] @@ -2547,7 +3112,7 @@ dependencies = [ "datafusion-physical-plan 48.0.1", "datafusion-session 48.0.1", "futures", - "object_store", + "object_store 0.12.5", "regex", "tokio", ] @@ -2558,7 +3123,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b559d7bf87d4f900f847baba8509634f838d9718695389e903604cdcccdb01f3" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "async-trait", "bytes", "datafusion-common 52.5.0", @@ -2570,7 +3135,7 @@ dependencies = [ "datafusion-physical-plan 52.5.0", "datafusion-session 52.5.0", "futures", - "object_store", + "object_store 0.12.5", "regex", "tokio", ] @@ -2594,7 +3159,7 @@ dependencies = [ "datafusion-physical-plan 48.0.1", "datafusion-session 48.0.1", "futures", - "object_store", + "object_store 0.12.5", "serde_json", "tokio", ] @@ -2605,7 +3170,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "250e2d7591ba8b638f063854650faa40bca4e8bd4059b2ece8836f6388d02db4" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "async-trait", "bytes", "datafusion-common 52.5.0", @@ -2617,7 +3182,7 @@ dependencies = [ "datafusion-physical-plan 52.5.0", "datafusion-session 52.5.0", "futures", - "object_store", + "object_store 0.12.5", "tokio", ] @@ -2644,10 +3209,10 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.12.5", "parking_lot", - "parquet", - "rand 0.9.2", + "parquet 55.2.0", + "rand 0.9.4", "tokio", ] @@ -2673,9 +3238,9 @@ dependencies = [ "datafusion-expr 48.0.1", "futures", "log", - "object_store", + "object_store 0.12.5", "parking_lot", - "rand 0.9.2", + "rand 0.9.4", "tempfile", "url", ] @@ -2686,7 +3251,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc45d23c516ed8d3637751e44e09e21b45b3f58b473c802dddd1f1ad4fe435ff" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "async-trait", "chrono", "dashmap", @@ -2694,9 +3259,9 @@ dependencies = [ "datafusion-expr 52.5.0", "futures", "log", - "object_store", + "object_store 0.12.5", "parking_lot", - "rand 0.9.2", + "rand 0.9.4", "tempfile", "url", ] @@ -2714,7 +3279,7 @@ dependencies = [ "datafusion-functions-aggregate-common 48.0.1", "datafusion-functions-window-common 48.0.1", "datafusion-physical-expr-common 48.0.1", - "indexmap 2.13.1", + "indexmap 2.14.0", "paste", "recursive", "serde_json", @@ -2727,7 +3292,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63dd30526d2db4fda6440806a41e4676334a94bc0596cc9cc2a0efed20ef2c44" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "async-trait", "chrono", "datafusion-common 52.5.0", @@ -2736,7 +3301,7 @@ dependencies = [ "datafusion-functions-aggregate-common 52.5.0", "datafusion-functions-window-common 52.5.0", "datafusion-physical-expr-common 52.5.0", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "paste", "serde_json", @@ -2750,7 +3315,7 @@ source = "git+https://github.com/FunctionStream/datafusion?branch=48.0.1%2Ffs#c3 dependencies = [ "arrow 55.2.0", "datafusion-common 48.0.1", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "paste", ] @@ -2761,9 +3326,9 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b486b5f6255d40976b88bb83813b0d035a8333e0ec39864824e78068cf42fa6" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "datafusion-common 52.5.0", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "paste", ] @@ -2789,9 +3354,9 @@ dependencies = [ "itertools 0.14.0", "log", "md-5", - "rand 0.9.2", + "rand 0.9.4", "regex", - "sha2", + "sha2 0.10.9", "unicode-segmentation", "uuid", ] @@ -2802,8 +3367,8 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07356c94118d881130dd0ffbff127540407d969c8978736e324edcd6c41cd48f" dependencies = [ - "arrow 57.3.0", - "arrow-buffer 57.3.0", + "arrow 57.3.1", + "arrow-buffer 57.3.1", "base64", "blake2", "blake3", @@ -2820,9 +3385,9 @@ dependencies = [ "log", "md-5", "num-traits", - "rand 0.9.2", + "rand 0.9.4", "regex", - "sha2", + "sha2 0.10.9", "unicode-segmentation", "uuid", ] @@ -2854,7 +3419,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b644f9cf696df9233ce6958b9807666d78563b56f923267474dd6c07795f1f8f" dependencies = [ "ahash", - "arrow 57.3.0", + "arrow 57.3.1", "datafusion-common 52.5.0", "datafusion-doc 52.5.0", "datafusion-execution 52.5.0", @@ -2887,7 +3452,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1de2deaaabe8923ce9ea9f29c47bbb4ee14f67ea2fe1ab5398d9bbebcf86e56" dependencies = [ "ahash", - "arrow 57.3.0", + "arrow 57.3.1", "datafusion-common 52.5.0", "datafusion-expr-common 52.5.0", "datafusion-physical-expr-common 52.5.0", @@ -2919,8 +3484,8 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "552f8d92e4331ee91d23c02d12bb6acf32cbfd5215117e01c0fb63cd4b15af1a" dependencies = [ - "arrow 57.3.0", - "arrow-ord 57.3.0", + "arrow 57.3.1", + "arrow-ord 57.3.1", "datafusion-common 52.5.0", "datafusion-doc 52.5.0", "datafusion-execution 52.5.0", @@ -2957,7 +3522,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "970fd0cdd3df8802b9a9975ff600998289ba9d46682a4f7285cba4820c9ada78" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "async-trait", "datafusion-catalog 52.5.0", "datafusion-common 52.5.0", @@ -2990,7 +3555,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40b4c21a7c8a986a1866c0a87ab756d0bbf7b5f41f306009fa2d9af79c52ed31" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "datafusion-common 52.5.0", "datafusion-doc 52.5.0", "datafusion-expr 52.5.0", @@ -3052,7 +3617,7 @@ dependencies = [ "datafusion-common 48.0.1", "datafusion-expr 48.0.1", "datafusion-physical-expr 48.0.1", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "log", "recursive", @@ -3066,13 +3631,13 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff9aa82b240252a88dee118372f9b9757c545ab9e53c0736bebab2e7da0ef1f2" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "chrono", "datafusion-common 52.5.0", "datafusion-expr 52.5.0", "datafusion-expr-common 52.5.0", "datafusion-physical-expr 52.5.0", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "log", "regex", @@ -3093,7 +3658,7 @@ dependencies = [ "datafusion-physical-expr-common 48.0.1", "half", "hashbrown 0.14.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "log", "paste", @@ -3107,7 +3672,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d48022b8af9988c1d852644f9e8b5584c490659769a550c5e8d39457a1da0a5" dependencies = [ "ahash", - "arrow 57.3.0", + "arrow 57.3.1", "datafusion-common 52.5.0", "datafusion-expr 52.5.0", "datafusion-expr-common 52.5.0", @@ -3115,7 +3680,7 @@ dependencies = [ "datafusion-physical-expr-common 52.5.0", "half", "hashbrown 0.16.1", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", "paste", @@ -3129,7 +3694,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae7a8abc0b4fe624000972a9b145b30b7f1b680bffaa950ea53f78d9b21c27c3" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "datafusion-common 52.5.0", "datafusion-expr 52.5.0", "datafusion-functions 52.5.0", @@ -3158,12 +3723,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147253ca3e6b9d59c162de64c02800973018660e13340dd1886dd038d17ac429" dependencies = [ "ahash", - "arrow 57.3.0", + "arrow 57.3.1", "chrono", "datafusion-common 52.5.0", "datafusion-expr-common 52.5.0", "hashbrown 0.16.1", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", ] @@ -3192,7 +3757,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "689156bb2282107b6239db8d7ef44b4dab10a9b33d3491a0c74acac5e4fedd72" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "datafusion-common 52.5.0", "datafusion-execution 52.5.0", "datafusion-expr 52.5.0", @@ -3225,7 +3790,7 @@ dependencies = [ "futures", "half", "hashbrown 0.14.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "log", "parking_lot", @@ -3240,9 +3805,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68253dc0ee5330aa558b2549c9b0da5af9fc17d753ae73022939014ad616fc28" dependencies = [ "ahash", - "arrow 57.3.0", - "arrow-ord 57.3.0", - "arrow-schema 57.3.0", + "arrow 57.3.1", + "arrow-ord 57.3.1", + "arrow-schema 57.3.1", "async-trait", "datafusion-common 52.5.0", "datafusion-common-runtime 52.5.0", @@ -3256,7 +3821,7 @@ dependencies = [ "futures", "half", "hashbrown 0.16.1", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "log", "parking_lot", @@ -3275,7 +3840,7 @@ dependencies = [ "datafusion-common 48.0.1", "datafusion-expr 48.0.1", "datafusion-proto-common", - "object_store", + "object_store 0.12.5", "prost 0.13.5", ] @@ -3295,7 +3860,7 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fcad240a54d0b1d3e8f668398900260a53122d522b2102ab57218590decacd6" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "datafusion-common 52.5.0", "datafusion-datasource 52.5.0", "datafusion-expr-common 52.5.0", @@ -3324,7 +3889,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.12.5", "parking_lot", "tokio", ] @@ -3352,7 +3917,7 @@ dependencies = [ "bigdecimal", "datafusion-common 48.0.1", "datafusion-expr 48.0.1", - "indexmap 2.13.1", + "indexmap 2.14.0", "log", "recursive", "regex", @@ -3365,12 +3930,12 @@ version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be53e9eb55db0fbb8980bb6d87f2435b0524acf4c718ed54a57cabbb299b2ab3" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "bigdecimal", "chrono", "datafusion-common 52.5.0", "datafusion-expr 52.5.0", - "indexmap 2.13.1", + "indexmap 2.14.0", "log", "regex", "sqlparser 0.59.0", @@ -3405,6 +3970,109 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "deltalake" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33449d2bcc676a8a0088ed644c3f7c5bf094ae4b115c1730735ba30e8a1866bb" +dependencies = [ + "buoyant_kernel", + "ctor", + "deltalake-aws", + "deltalake-core", +] + +[[package]] +name = "deltalake-aws" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ef837d5d58bb0cd62dc9ee3bc201805751090f4bab2ca0ff09669661814fd4" +dependencies = [ + "async-trait", + "aws-config", + "aws-credential-types", + "aws-sdk-dynamodb", + "aws-sdk-sts", + "aws-smithy-runtime-api", + "backon", + "bytes", + "chrono", + "deltalake-core", + "futures", + "object_store 0.13.2", + "regex", + "thiserror 2.0.18", + "tokio", + "tracing", + "typed-builder", + "url", + "uuid", +] + +[[package]] +name = "deltalake-core" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8e67c5de0c7cc492a22c6e5bd7f37cfb59c471dd135617058eac609ddcfb878" +dependencies = [ + "arrow 58.3.0", + "arrow-arith 58.3.0", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-cast 58.3.0", + "arrow-ipc 58.3.0", + "arrow-json 58.3.0", + "arrow-ord 58.3.0", + "arrow-row 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", + "async-trait", + "buoyant_kernel", + "bytes", + "cfg-if", + "chrono", + "dashmap", + "deltalake-derive", + "dirs", + "either", + "futures", + "humantime", + "indexmap 2.14.0", + "itertools 0.14.0", + "num_cpus", + "object_store 0.13.2", + "parking_lot", + "parquet 58.3.0", + "percent-encoding", + "percent-encoding-rfc3986", + "pin-project-lite", + "rand 0.10.1", + "regex", + "serde", + "serde_json", + "sqlparser 0.61.0", + "strum 0.27.2", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "uuid", + "validator", +] + +[[package]] +name = "deltalake-derive" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5a646e2b05608db1e5f91e75b6af841d92294ea0864789bb01f3f818e5f7b25" +dependencies = [ + "convert_case", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "deranged" version = "0.5.8" @@ -3421,12 +4089,24 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.0", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "directories-next" version = "2.0.0" @@ -3504,6 +4184,21 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" +[[package]] +name = "dtor" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edf234dd1594d6dd434a8fb8cada51ddbbc593e40e4a01556a0b31c62da2775b" +dependencies = [ + "dtor-proc-macro", +] + +[[package]] +name = "dtor-proc-macro" +version = "0.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2647271c92754afcb174e758003cfd1cbf1e43e5a7853d7b1813e63e19e39a73" + [[package]] name = "duct" version = "0.13.7" @@ -3555,6 +4250,27 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" +[[package]] +name = "enumset" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "839c4174b41e75c8f7306110b2c51996a293b8d1d850edd529011841d9fede7d" +dependencies = [ + "enumset_derive", +] + +[[package]] +name = "enumset_derive" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd536557b58c682b217b8fb199afdff47cd3eff260623f19e77074eb073d63a" +dependencies = [ + "darling 0.21.3", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "env_logger" version = "0.10.2" @@ -3592,9 +4308,9 @@ checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" [[package]] name = "ethnum" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca81e6b4777c89fd810c25a4be2b1bd93ea034fbe58e6a75216a34c6b82c539b" +checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f" [[package]] name = "event-listener" @@ -3623,6 +4339,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fast-float2" version = "0.2.3" @@ -3637,9 +4359,9 @@ checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" [[package]] name = "fastrand" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a043dc74da1e37d6afe657061213aa6f425f855399a11d3463c6ecccc4dfda1f" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "fd-lock" @@ -3686,7 +4408,7 @@ version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "rustc_version", ] @@ -3701,6 +4423,17 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -3757,12 +4490,12 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2195cc7f87e84bd695586137de99605e7e9579b26ec5e01b82960ddb4d0922f2" +checksum = "2b3a6f3550e61b999febd7168d462db953948eff4fc3448276b3d10d10324dbb" dependencies = [ - "arrow-array 57.3.0", - "rand 0.9.2", + "arrow-array 57.3.1", + "rand 0.9.4", ] [[package]] @@ -3783,9 +4516,9 @@ dependencies = [ "apache-avro", "arrow 55.2.0", "arrow-array 55.2.0", - "arrow-array 57.3.0", + "arrow-array 57.3.1", "arrow-ipc 55.2.0", - "arrow-ipc 57.3.0", + "arrow-ipc 57.3.1", "arrow-json 55.2.0", "arrow-schema 55.2.0", "async-trait", @@ -3800,6 +4533,11 @@ dependencies = [ "datafusion-expr 48.0.1", "datafusion-physical-expr 48.0.1", "datafusion-proto", + "function-stream-config", + "function-stream-logger", + "function-stream-runtime-common", + "function-stream-streaming-planner", + "function-stream-streaming-runtime", "futures", "governor", "itertools 0.14.0", @@ -3807,14 +4545,14 @@ dependencies = [ "log", "lru", "num_cpus", - "object_store", + "object_store 0.12.5", "parking_lot", - "parquet", + "parquet 55.2.0", "petgraph 0.7.1", "proctitle", "prost 0.13.5", "protocol", - "rand 0.8.5", + "rand 0.8.6", "rdkafka", "rocksdb", "serde", @@ -3838,9 +4576,26 @@ dependencies = [ "xxhash-rust", ] +[[package]] +name = "function-stream-catalog" +version = "0.6.0" +dependencies = [ + "protocol", +] + +[[package]] +name = "function-stream-catalog-storage" +version = "0.6.0" +dependencies = [ + "anyhow", + "function-stream-catalog", + "parking_lot", + "rocksdb", +] + [[package]] name = "function-stream-cli" -version = "0.1.0" +version = "0.6.0" dependencies = [ "arrow-array 52.2.0", "arrow-ipc 52.2.0", @@ -3853,6 +4608,204 @@ dependencies = [ "tonic", ] +[[package]] +name = "function-stream-common" +version = "0.6.0" + +[[package]] +name = "function-stream-config" +version = "0.6.0" +dependencies = [ + "serde", + "serde_yaml", + "uuid", +] + +[[package]] +name = "function-stream-coordinator" +version = "0.6.0" + +[[package]] +name = "function-stream-logger" +version = "0.6.0" +dependencies = [ + "anyhow", + "function-stream-config", + "tracing", + "tracing-appender", + "tracing-subscriber", +] + +[[package]] +name = "function-stream-runtime-common" +version = "0.6.0" +dependencies = [ + "arrow-array 55.2.0", + "bincode", + "parking_lot", + "serde", + "tokio", + "tracing", +] + +[[package]] +name = "function-stream-servicer" +version = "0.6.0" + +[[package]] +name = "function-stream-sqlparser" +version = "0.6.0" + +[[package]] +name = "function-stream-streaming-planner" +version = "0.6.0" +dependencies = [ + "ahash", + "anyhow", + "apache-avro", + "arrow 55.2.0", + "arrow-array 55.2.0", + "arrow-json 55.2.0", + "arrow-schema 55.2.0", + "async-trait", + "bincode", + "bytes", + "chrono", + "datafusion 48.0.1", + "datafusion-common 48.0.1", + "datafusion-execution 48.0.1", + "datafusion-expr 48.0.1", + "datafusion-physical-expr 48.0.1", + "datafusion-proto", + "function-stream-config", + "function-stream-runtime-common", + "futures", + "itertools 0.14.0", + "petgraph 0.7.1", + "prost 0.13.5", + "protocol", + "rand 0.8.6", + "serde", + "serde_json", + "serde_json_path", + "sqlparser 0.55.0", + "strum 0.26.3", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tracing", + "unicase", + "xxhash-rust", +] + +[[package]] +name = "function-stream-streaming-runtime" +version = "0.6.0" +dependencies = [ + "function-stream-streaming-runtime-core", + "function-stream-streaming-runtime-job-manager", + "function-stream-streaming-runtime-operators", +] + +[[package]] +name = "function-stream-streaming-runtime-core" +version = "0.6.0" +dependencies = [ + "anyhow", + "arrow 55.2.0", + "arrow-array 55.2.0", + "arrow-schema 55.2.0", + "async-trait", + "crossbeam-channel", + "function-stream-runtime-common", + "function-stream-streaming-planner", + "parking_lot", + "parquet 55.2.0", + "protocol", + "serde", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tracing", + "uuid", +] + +[[package]] +name = "function-stream-streaming-runtime-job-manager" +version = "0.6.0" +dependencies = [ + "anyhow", + "function-stream-config", + "function-stream-runtime-common", + "function-stream-streaming-planner", + "function-stream-streaming-runtime-core", + "function-stream-streaming-runtime-operators", + "protocol", + "tokio", + "tokio-stream", + "tracing", +] + +[[package]] +name = "function-stream-streaming-runtime-operators" +version = "0.6.0" +dependencies = [ + "ahash", + "anyhow", + "apache-avro", + "arrow 55.2.0", + "arrow-array 55.2.0", + "arrow-array 57.3.1", + "arrow-ipc 55.2.0", + "arrow-ipc 57.3.1", + "arrow-json 55.2.0", + "arrow-schema 55.2.0", + "async-trait", + "axum", + "base64", + "bincode", + "byteorder", + "bytes", + "datafusion 48.0.1", + "datafusion-common 48.0.1", + "datafusion-execution 48.0.1", + "datafusion-expr 48.0.1", + "datafusion-physical-expr 48.0.1", + "datafusion-proto", + "deltalake", + "function-stream-config", + "function-stream-runtime-common", + "function-stream-streaming-planner", + "function-stream-streaming-runtime-core", + "futures", + "governor", + "itertools 0.14.0", + "lance", + "lzf", + "mcap", + "object_store 0.12.5", + "parquet 55.2.0", + "prost 0.13.5", + "protocol", + "rdkafka", + "reqwest 0.12.28", + "rosbag", + "rumqttc", + "rusqlite", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tokio-tungstenite", + "tracing", + "url", +] + +[[package]] +name = "function-stream-wasm-runtime" +version = "0.6.0" + [[package]] name = "funty" version = "2.0.0" @@ -3932,9 +4885,9 @@ checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-timer" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" [[package]] name = "futures-util" @@ -3959,7 +4912,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25234f20a3ec0a962a61770cfe39ecf03cb529a6e474ad8cff025ed497eda557" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "debugid", "rustc-hash 2.1.2", "serde", @@ -4028,6 +4981,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", ] @@ -4039,7 +4993,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" dependencies = [ "fallible-iterator", - "indexmap 2.13.1", + "indexmap 2.14.0", "stable_deref_trait", ] @@ -4078,7 +5032,7 @@ dependencies = [ "parking_lot", "portable-atomic", "quanta", - "rand 0.9.2", + "rand 0.9.4", "smallvec", "spinning_top", "web-time", @@ -4086,9 +5040,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" dependencies = [ "atomic-waker", "bytes", @@ -4096,7 +5050,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.4.0", - "indexmap 2.13.1", + "indexmap 2.14.0", "slab", "tokio", "tokio-util", @@ -4154,6 +5108,21 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "heck" version = "0.5.0" @@ -4178,7 +5147,16 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", ] [[package]] @@ -4269,6 +5247,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.9.0" @@ -4293,20 +5280,19 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.7" +version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ "http 1.4.0", "hyper", "hyper-util", - "rustls", - "rustls-native-certs", - "rustls-pki-types", + "rustls 0.23.40", + "rustls-native-certs 0.8.3", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tower-service", - "webpki-roots", + "webpki-roots 1.0.7", ] [[package]] @@ -4340,9 +5326,11 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2 0.6.3", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", ] [[package]] @@ -4485,9 +5473,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", @@ -4519,12 +5507,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45a8a2b9cb3e0b0c1803dbb0758ffac5de2f425b23c28f518faabd9d805342ff" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "serde", "serde_core", ] @@ -4563,18 +5551,8 @@ checksum = "06432fb54d3be7964ecd3649233cddf80db2832f47fec34c01f65b3d9d774983" [[package]] name = "ipnet" version = "2.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" - -[[package]] -name = "iri-string" -version = "0.7.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" -dependencies = [ - "memchr", - "serde", -] +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "is-terminal" @@ -4639,9 +5617,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" dependencies = [ "jiff-static", "jiff-tzdb-platform", @@ -4654,9 +5632,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" dependencies = [ "proc-macro2", "quote", @@ -4678,6 +5656,55 @@ dependencies = [ "jiff-tzdb", ] +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -4690,9 +5717,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.94" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" dependencies = [ "cfg-if", "futures-util", @@ -4714,7 +5741,7 @@ dependencies = [ "nom 8.0.0", "num-traits", "ordered-float 5.3.0", - "rand 0.9.2", + "rand 0.9.4", "serde", "serde_json", "zmij", @@ -4722,19 +5749,19 @@ dependencies = [ [[package]] name = "lance" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efe6c3ddd79cdfd2b7e1c23cafae52806906bc40fbd97de9e8cf2f8c7a75fc04" -dependencies = [ - "arrow 57.3.0", - "arrow-arith 57.3.0", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-ipc 57.3.0", - "arrow-ord 57.3.0", - "arrow-row 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63e285ceee2b4ca8eb3a8742266cc1ac8161599767a8ecb4d8c2f9fd43d8b29" +dependencies = [ + "arrow 57.3.1", + "arrow-arith 57.3.1", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-ipc 57.3.1", + "arrow-ord 57.3.1", + "arrow-row 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", "async-recursion", "async-trait", "async_cell", @@ -4767,12 +5794,12 @@ dependencies = [ "lance-table", "log", "moka", - "object_store", + "object_store 0.12.5", "permutation", "pin-project", "prost 0.14.3", "prost-types 0.14.3", - "rand 0.9.2", + "rand 0.9.4", "roaring", "semver", "serde", @@ -4789,31 +5816,31 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d9f5d95bdda2a2b790f1fb8028b5b6dcf661abeb3133a8bca0f3d24b054af87" +checksum = "5c55e62fc04422ef4cd4af6f863ada32641ae23124f9b2e9c567a40d617e8c97" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-cast 57.3.0", - "arrow-data 57.3.0", - "arrow-ord 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-cast 57.3.1", + "arrow-data 57.3.1", + "arrow-ord 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", "bytes", "futures", "getrandom 0.2.17", "half", "jsonb", "num-traits", - "rand 0.9.2", + "rand 0.9.4", ] [[package]] name = "lance-bitpacking" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f827d6ab9f8f337a9509d5ad66a12f3314db8713868260521c344ef6135eb4e4" +checksum = "a48d232a2908645af0040f96c60a6387fea2df75e762d7033e93e17bb420c6a1" dependencies = [ "arrayref", "paste", @@ -4822,13 +5849,13 @@ dependencies = [ [[package]] name = "lance-core" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f1e25df6a79bf72ee6bcde0851f19b1cd36c5848c1b7db83340882d3c9fdecb" +checksum = "ce071baaff88fcdcf67f1dd0af54e17656f52ae75aaeb75f25f9cf4da29241f2" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-schema 57.3.0", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-schema 57.3.1", "async-trait", "byteorder", "bytes", @@ -4844,10 +5871,10 @@ dependencies = [ "mock_instant", "moka", "num_cpus", - "object_store", + "object_store 0.12.5", "pin-project", "prost 0.14.3", - "rand 0.9.2", + "rand 0.9.4", "roaring", "serde_json", "snafu", @@ -4861,16 +5888,16 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93146de8ae720cb90edef81c2f2d0a1b065fc2f23ecff2419546f389b0fa70a4" +checksum = "11ebc97ee94fa8e1af6fd0520066c7e7e0eab38a100e750ba9aabad644c5aa57" dependencies = [ - "arrow 57.3.0", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-ord 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow 57.3.1", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-ord 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", "async-trait", "chrono", "datafusion 52.5.0", @@ -4893,19 +5920,19 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccec8ce4d8e0a87a99c431dab2364398029f2ffb649c1a693c60c79e05ed30dd" +checksum = "9b90dbb2829875b3a3d00f88fd3a3e39a9e4c7d34c266f67da6550fcda54c76e" dependencies = [ - "arrow 57.3.0", - "arrow-array 57.3.0", - "arrow-cast 57.3.0", - "arrow-schema 57.3.0", + "arrow 57.3.1", + "arrow-array 57.3.1", + "arrow-cast 57.3.1", + "arrow-schema 57.3.1", "chrono", "futures", "half", "hex", - "rand 0.9.2", + "rand 0.9.4", "rand_distr 0.5.1", "rand_xoshiro 0.7.0", "random_word", @@ -4913,17 +5940,17 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c1aec0bbbac6bce829bc10f1ba066258126100596c375fb71908ecf11c2c2a5" +checksum = "65ec429cc2e18ad1b7e43cc7ec57a2f2e49229cfbd934da45e619751a886b8cd" dependencies = [ - "arrow-arith 57.3.0", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-cast 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-arith 57.3.1", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-cast 57.3.1", + "arrow-data 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", "bytemuck", "byteorder", "bytes", @@ -4941,7 +5968,7 @@ dependencies = [ "prost 0.14.3", "prost-build 0.14.3", "prost-types 0.14.3", - "rand 0.9.2", + "rand 0.9.4", "snafu", "strum 0.26.3", "tokio", @@ -4952,16 +5979,16 @@ dependencies = [ [[package]] name = "lance-file" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14a8c548804f5b17486dc2d3282356ed1957095a852780283bc401fdd69e9075" +checksum = "418afe3f82487615fa09222b95a4b5853103f3f0425996d24a537ca750381f83" dependencies = [ - "arrow-arith 57.3.0", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-arith 57.3.1", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-data 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", "async-recursion", "async-trait", "byteorder", @@ -4975,7 +6002,7 @@ dependencies = [ "lance-io", "log", "num-traits", - "object_store", + "object_store 0.12.5", "prost 0.14.3", "prost-build 0.14.3", "prost-types 0.14.3", @@ -4986,16 +6013,16 @@ dependencies = [ [[package]] name = "lance-index" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da212f0090ea59f79ac3686660f596520c167fe1cb5f408900cf71d215f0e03" +checksum = "936b3deeb6ee075646d18f27b01cf2d2e846c3f5f6c5fa45b30aa41dd5b4c4e2" dependencies = [ - "arrow 57.3.0", - "arrow-arith 57.3.0", - "arrow-array 57.3.0", - "arrow-ord 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow 57.3.1", + "arrow-arith 57.3.1", + "arrow-array 57.3.1", + "arrow-ord 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", "async-channel", "async-recursion", "async-trait", @@ -5029,11 +6056,11 @@ dependencies = [ "log", "ndarray", "num-traits", - "object_store", + "object_store 0.12.5", "prost 0.14.3", "prost-build 0.14.3", "prost-types 0.14.3", - "rand 0.9.2", + "rand 0.9.4", "rand_distr 0.5.1", "rangemap", "rayon", @@ -5052,18 +6079,18 @@ dependencies = [ [[package]] name = "lance-io" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d958eb4b56f03bbe0f5f85eb2b4e9657882812297b6f711f201ffc995f259f" -dependencies = [ - "arrow 57.3.0", - "arrow-arith 57.3.0", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-cast 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4103e4cebe146af15bfb198c8142d6ea37d5b25fa04158bf2d9be4597bf174d3" +dependencies = [ + "arrow 57.3.1", + "arrow-arith 57.3.1", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-cast 57.3.1", + "arrow-data 57.3.1", + "arrow-schema 57.3.1", + "arrow-select 57.3.1", "async-recursion", "async-trait", "aws-config", @@ -5078,13 +6105,13 @@ dependencies = [ "lance-core", "lance-namespace", "log", - "object_store", + "object_store 0.12.5", "object_store_opendal", "opendal", "path_abs", "pin-project", "prost 0.14.3", - "rand 0.9.2", + "rand 0.9.4", "serde", "snafu", "tempfile", @@ -5095,29 +6122,29 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0285b70da35def7ed95e150fae1d5308089554e1290470403ed3c50cb235bc5e" +checksum = "c00c7ad71eca93635404519e77add6689947c9342134bb2133578f81249bf809" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-schema 57.3.0", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-schema 57.3.1", "cc", "deepsize", "half", "lance-arrow", "lance-core", "num-traits", - "rand 0.9.2", + "rand 0.9.4", ] [[package]] name = "lance-namespace" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f78e2a828b654e062a495462c6e3eb4fcf0e7e907d761b8f217fc09ccd3ceac" +checksum = "e0c59a574e72a4b72da8096bcaaa1b1e5b44f6a83da164cc714c286fab30c369" dependencies = [ - "arrow 57.3.0", + "arrow 57.3.1", "async-trait", "bytes", "lance-core", @@ -5132,7 +6159,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee2e48de899e2931afb67fcddd0a08e439bf5d8b6ea2a2ed9cb8f4df669bd5cc" dependencies = [ - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "serde_repr", @@ -5141,15 +6168,15 @@ dependencies = [ [[package]] name = "lance-table" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df9c4adca3eb2074b3850432a9fb34248a3d90c3d6427d158b13ff9355664ee" +checksum = "943b9c503f23ebab9e0dbee356f528bc4cbcafded87a6848451f205b0bb473d7" dependencies = [ - "arrow 57.3.0", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-ipc 57.3.0", - "arrow-schema 57.3.0", + "arrow 57.3.1", + "arrow-array 57.3.1", + "arrow-buffer 57.3.1", + "arrow-ipc 57.3.1", + "arrow-schema 57.3.1", "async-trait", "byteorder", "bytes", @@ -5161,11 +6188,11 @@ dependencies = [ "lance-file", "lance-io", "log", - "object_store", + "object_store 0.12.5", "prost 0.14.3", "prost-build 0.14.3", "prost-types 0.14.3", - "rand 0.9.2", + "rand 0.9.4", "rangemap", "roaring", "semver", @@ -5192,9 +6219,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "leb128" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" +checksum = "6cc46bac87ef8093eed6f272babb833b6443374399985ac8ed28471ee0918545" [[package]] name = "leb128fmt" @@ -5331,9 +6358,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.184" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libloading" @@ -5353,9 +6380,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08" +checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" dependencies = [ "libc", ] @@ -5376,6 +6403,17 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "libz-sys" version = "1.1.28" @@ -5388,6 +6426,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "link-section" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b685d66585d646efe09fec763d796c291049c8b6bf84e04954bffc8748341f0d" + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -5485,13 +6529,28 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.12.1" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90071f8077f8e40adfc4b7fe9cd495ce316263f19e75c2211eeff3fdf475a3d9" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "lz4_flex" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c23545df7ecf1b16c303910a69b079e8e251d60f7dd2cc9b4177f2afaf1746" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" dependencies = [ "twox-hash", ] +[[package]] +name = "lzf" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c034ef7ca69017088f8522e6f74301232f13ba43b5640941abc4dce26f530521" + [[package]] name = "lzma-sys" version = "0.1.20" @@ -5546,6 +6605,26 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" +[[package]] +name = "mcap" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a5a8225e54ba304d8bf118679f422a884cfcf5353cf5618a1e1b8aa135ee728" +dependencies = [ + "bimap", + "binrw", + "byteorder", + "crc32fast", + "enumset", + "log", + "lz4", + "num_cpus", + "paste", + "static_assertions", + "thiserror 1.0.69", + "zstd", +] + [[package]] name = "md-5" version = "0.10.6" @@ -5553,7 +6632,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", ] [[package]] @@ -5700,7 +6779,7 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cfg-if", "cfg_aliases 0.1.1", "libc", @@ -5782,9 +6861,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" @@ -5867,7 +6946,7 @@ checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" dependencies = [ "crc32fast", "hashbrown 0.15.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "memchr", ] @@ -5892,9 +6971,49 @@ dependencies = [ "parking_lot", "percent-encoding", "quick-xml 0.38.4", - "rand 0.9.2", - "reqwest", + "rand 0.9.4", + "reqwest 0.12.28", + "ring", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "object_store" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +dependencies = [ + "async-trait", + "base64", + "bytes", + "chrono", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body-util", + "httparse", + "humantime", + "hyper", + "itertools 0.14.0", + "md-5", + "parking_lot", + "percent-encoding", + "quick-xml 0.39.4", + "rand 0.10.1", + "reqwest 0.12.28", "ring", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", @@ -5917,7 +7036,7 @@ dependencies = [ "bytes", "chrono", "futures", - "object_store", + "object_store 0.12.5", "opendal", "pin-project", "tokio", @@ -5962,7 +7081,7 @@ dependencies = [ "percent-encoding", "quick-xml 0.38.4", "reqsign", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "tokio", @@ -5970,6 +7089,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + [[package]] name = "openssl-probe" version = "0.2.1" @@ -5978,9 +7103,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.112" +version = "0.9.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb" +checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4" dependencies = [ "cc", "libc", @@ -6047,6 +7172,12 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "owo-colors" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" + [[package]] name = "parking" version = "2.2.1" @@ -6101,7 +7232,43 @@ dependencies = [ "lz4_flex 0.11.6", "num", "num-bigint", - "object_store", + "object_store 0.12.5", + "paste", + "seq-macro", + "simdutf8", + "snap", + "thrift", + "tokio", + "twox-hash", + "zstd", +] + +[[package]] +name = "parquet" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" +dependencies = [ + "ahash", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-ipc 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", + "base64", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.17.1", + "lz4_flex 0.13.1", + "num-bigint", + "num-integer", + "num-traits", + "object_store 0.13.2", "paste", "seq-macro", "simdutf8", @@ -6142,6 +7309,12 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "percent-encoding-rfc3986" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3637c05577168127568a64e9dc5a6887da720efef07b3d9472d45f63ab191166" + [[package]] name = "permutation" version = "0.4.1" @@ -6155,7 +7328,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset 0.4.2", - "indexmap 2.13.1", + "indexmap 2.14.0", ] [[package]] @@ -6165,7 +7338,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset 0.5.7", - "indexmap 2.13.1", + "indexmap 2.14.0", ] [[package]] @@ -6176,7 +7349,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset 0.5.7", "hashbrown 0.15.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "serde", ] @@ -6200,18 +7373,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.11" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.11" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", @@ -6232,9 +7405,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "portable-atomic" @@ -6306,6 +7479,28 @@ dependencies = [ "toml_edit", ] +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -6431,7 +7626,7 @@ dependencies = [ [[package]] name = "protocol" -version = "0.1.0" +version = "0.6.0" dependencies = [ "env_logger", "log", @@ -6443,9 +7638,9 @@ dependencies = [ [[package]] name = "psm" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3852766467df634d74f0b2d7819bf8dc483a0eb2e3b0f50f756f9cfe8b0d18d8" +checksum = "645dbe486e346d9b5de3ef16ede18c26e6c70ad97418f4874b8b1889d6e761ea" dependencies = [ "ar_archive_writer", "cc", @@ -6515,6 +7710,16 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.39.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quinn" version = "0.11.9" @@ -6527,7 +7732,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.2", - "rustls", + "rustls 0.23.40", "socket2 0.6.3", "thiserror 2.0.18", "tokio", @@ -6541,13 +7746,14 @@ version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand 0.9.4", "ring", "rustc-hash 2.1.2", - "rustls", + "rustls 0.23.40", "rustls-pki-types", "slab", "thiserror 2.0.18", @@ -6609,9 +7815,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -6620,14 +7826,25 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -6666,6 +7883,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rand_distr" version = "0.4.3" @@ -6673,7 +7896,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" dependencies = [ "num-traits", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -6683,7 +7906,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.2", + "rand 0.9.4", ] [[package]] @@ -6713,7 +7936,7 @@ dependencies = [ "ahash", "brotli", "paste", - "rand 0.9.2", + "rand 0.9.4", "unicase", ] @@ -6729,7 +7952,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", ] [[package]] @@ -6740,9 +7963,9 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" [[package]] name = "rayon" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" dependencies = [ "either", "rayon-core", @@ -6818,7 +8041,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", ] [[package]] @@ -6906,19 +8129,19 @@ dependencies = [ "form_urlencoded", "getrandom 0.2.17", "hex", - "hmac", + "hmac 0.12.1", "home", "http 1.4.0", "log", "percent-encoding", "quick-xml 0.37.5", - "rand 0.8.5", - "reqwest", + "rand 0.8.6", + "reqwest 0.12.28", "rust-ini", "serde", "serde_json", "sha1", - "sha2", + "sha2 0.10.9", "tokio", ] @@ -6947,15 +8170,15 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls", - "rustls-native-certs", + "rustls 0.23.40", + "rustls-native-certs 0.8.3", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tokio-util", "tower 0.5.3", "tower-http", @@ -6965,7 +8188,45 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots", + "webpki-roots 1.0.7", +] + +[[package]] +name = "reqwest" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "h2", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls 0.23.40", + "rustls-pki-types", + "rustls-platform-verifier", + "sync_wrapper", + "tokio", + "tokio-rustls 0.26.4", + "tower 0.5.3", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", ] [[package]] @@ -6983,23 +8244,69 @@ dependencies = [ ] [[package]] -name = "roaring" -version = "0.11.3" +name = "roaring" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9" +dependencies = [ + "bytemuck", + "byteorder", +] + +[[package]] +name = "rocksdb" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb6f170a4041d50a0ce04b0d2e14916d6ca863ea2e422689a5b694395d299ffe" +dependencies = [ + "libc", + "librocksdb-sys", +] + +[[package]] +name = "rosbag" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef77688c2fe3d1a94991e3d7256face190d6b6ea1a1ac60a6e2bad8e2aeabde7" +dependencies = [ + "base16ct", + "byteorder", + "bzip2 0.4.4", + "log", + "lz4", + "memmap2", +] + +[[package]] +name = "rumqttc" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ba9ce64a8f45d7fc86358410bb1a82e8c987504c0d4900e9141d69a9f26c885" +checksum = "e1568e15fab2d546f940ed3a21f48bbbd1c494c90c99c4481339364a497f94a9" dependencies = [ - "bytemuck", - "byteorder", + "bytes", + "flume", + "futures-util", + "log", + "rustls-native-certs 0.7.3", + "rustls-pemfile", + "rustls-webpki 0.102.8", + "thiserror 1.0.69", + "tokio", + "tokio-rustls 0.25.0", ] [[package]] -name = "rocksdb" -version = "0.21.0" +name = "rusqlite" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb6f170a4041d50a0ce04b0d2e14916d6ca863ea2e422689a5b694395d299ffe" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" dependencies = [ - "libc", - "librocksdb-sys", + "bitflags 2.11.1", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", ] [[package]] @@ -7055,7 +8362,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys 0.4.15", @@ -7068,7 +8375,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys 0.12.1", @@ -7087,46 +8394,120 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.37" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" +dependencies = [ + "log", + "ring", + "rustls-pki-types", + "rustls-webpki 0.102.8", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "aws-lc-rs", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.103.13", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" +dependencies = [ + "openssl-probe 0.1.6", + "rustls-pemfile", + "rustls-pki-types", + "schannel", + "security-framework 2.11.1", +] + [[package]] name = "rustls-native-certs" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" dependencies = [ - "openssl-probe", + "openssl-probe 0.2.1", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 3.7.0", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ "web-time", "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls 0.23.40", + "rustls-native-certs 0.8.3", + "rustls-platform-verifier-android", + "rustls-webpki 0.103.13", + "security-framework 3.7.0", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.102.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustls-webpki" -version = "0.103.11" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20a6af516fea4b20eccceaf166e8aa666ac996208e8a644ce3ef5aa783bc7cd4" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", "ring", @@ -7146,7 +8527,7 @@ version = "14.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7803e8936da37efd9b6d4478277f4b2b9bb5cdb37a113e8d63222e58da647e63" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cfg-if", "clipboard-win", "fd-lock", @@ -7210,14 +8591,27 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.11.1", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + [[package]] name = "security-framework" version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.11.0", - "core-foundation", + "bitflags 2.11.1", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -7352,6 +8746,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_repr" version = "0.1.20" @@ -7390,7 +8795,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.13.1", + "indexmap 2.14.0", "itoa", "ryu", "serde", @@ -7405,7 +8810,7 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", ] [[package]] @@ -7416,7 +8821,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -7482,6 +8898,16 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -7490,9 +8916,9 @@ checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "siphasher" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" [[package]] name = "sized-chunks" @@ -7575,6 +9001,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + [[package]] name = "spinning_top" version = "0.3.0" @@ -7604,6 +9039,16 @@ dependencies = [ "sqlparser_derive 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "sqlparser" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" +dependencies = [ + "log", + "recursive", +] + [[package]] name = "sqlparser_derive" version = "0.3.0" @@ -7633,15 +9078,15 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stacker" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013" +checksum = "640c8cdd92b6b12f5bcb1803ca3bbf5ab96e5e6b6b96b9ab77dabe9e880b3190" dependencies = [ "cc", "cfg-if", "libc", "psm", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7682,6 +9127,9 @@ name = "strum" version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros 0.27.2", +] [[package]] name = "strum_macros" @@ -7714,6 +9162,12 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "1.0.109" @@ -7756,13 +9210,34 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.11.1", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "system-interface" version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc4592f674ce18521c2a81483873a49596655b179f71c5e05d10c1fe66c78745" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cap-fs-ext", "cap-std", "fd-lock", @@ -8094,13 +9569,14 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.51.0" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bd1c4c0fc4a7ab90fc15ef6daaa3ec3b893f004f915f2392557ed23237820cd" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", "mio", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2 0.6.3", @@ -8119,13 +9595,24 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tokio-rustls" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +dependencies = [ + "rustls 0.22.4", + "rustls-pki-types", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls", + "rustls 0.23.40", "tokio", ] @@ -8140,6 +9627,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +dependencies = [ + "futures-util", + "log", + "rustls 0.23.40", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.4", + "tungstenite", + "webpki-roots 0.26.11", +] + [[package]] name = "tokio-util" version = "0.7.18" @@ -8159,7 +9662,7 @@ version = "0.9.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" dependencies = [ - "indexmap 2.13.1", + "indexmap 2.14.0", "serde_core", "serde_spanned", "toml_datetime 0.7.5+spec-1.1.0", @@ -8188,14 +9691,14 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.10+spec-1.1.0" +version = "0.25.11+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82418ca169e235e6c399a84e395ab6debeb3bc90edc959bf0f48647c6a32d1b" +checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" dependencies = [ - "indexmap 2.13.1", + "indexmap 2.14.0", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", - "winnow 1.0.1", + "winnow 1.0.3", ] [[package]] @@ -8204,7 +9707,7 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "winnow 1.0.1", + "winnow 1.0.3", ] [[package]] @@ -8268,7 +9771,7 @@ dependencies = [ "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand 0.8.5", + "rand 0.8.6", "slab", "tokio", "tokio-util", @@ -8290,29 +9793,30 @@ dependencies = [ "tokio", "tower-layer", "tower-service", + "tracing", ] [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "async-compression", - "bitflags 2.11.0", + "bitflags 2.11.1", "bytes", "futures-core", "futures-util", "http 1.4.0", "http-body 1.0.1", "http-body-util", - "iri-string", "pin-project-lite", "tokio", "tokio-util", "tower 0.5.3", "tower-layer", "tower-service", + "url", ] [[package]] @@ -8333,6 +9837,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -8340,11 +9845,12 @@ dependencies = [ [[package]] name = "tracing-appender" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" dependencies = [ "crossbeam-channel", + "symlink", "thiserror 2.0.18", "time", "tracing-subscriber", @@ -8406,20 +9912,60 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http 1.4.0", + "httparse", + "log", + "rand 0.8.6", + "rustls 0.23.40", + "rustls-pki-types", + "sha1", + "thiserror 1.0.69", + "utf-8", +] + [[package]] name = "twox-hash" version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" dependencies = [ - "rand 0.9.2", + "rand 0.9.4", +] + +[[package]] +name = "typed-builder" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31aa81521b70f94402501d848ccc0ecaa8f93c8eb6999eb9747e72287757ffda" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076a02dc54dd46795c2e9c8282ed40bcfb1e22747e955de9389a1de28190fb26" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", ] [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" [[package]] name = "unicase" @@ -8485,6 +10031,7 @@ dependencies = [ "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] @@ -8493,6 +10040,12 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf8-ranges" version = "1.0.5" @@ -8513,16 +10066,47 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ "getrandom 0.4.2", "js-sys", + "rand 0.10.1", "serde_core", "wasm-bindgen", ] +[[package]] +name = "validator" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0b4a29d8709210980a09379f27ee31549b73292c87ab9899beee1c0d3be6303" +dependencies = [ + "idna", + "once_cell", + "regex", + "serde", + "serde_derive", + "serde_json", + "url", + "validator_derive", +] + +[[package]] +name = "validator_derive" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bac855a2ce6f843beb229757e6e570a42e837bcb15e5f449dd48d5747d41bf77" +dependencies = [ + "darling 0.20.11", + "once_cell", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "valuable" version = "0.1.1" @@ -8580,11 +10164,11 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.2+wasi-0.2.9" +version = "1.0.3+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.57.1", ] [[package]] @@ -8593,14 +10177,14 @@ version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.51.0", ] [[package]] name = "wasm-bindgen" -version = "0.2.117" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" dependencies = [ "cfg-if", "once_cell", @@ -8611,9 +10195,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.67" +version = "0.4.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03623de6905b7206edd0a75f69f747f134b7f0a2323392d664448bf2d3c5d87e" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" dependencies = [ "js-sys", "wasm-bindgen", @@ -8621,9 +10205,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.117" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -8631,9 +10215,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.117" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" dependencies = [ "bumpalo", "proc-macro2", @@ -8644,9 +10228,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.117" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" dependencies = [ "unicode-ident", ] @@ -8660,7 +10244,7 @@ dependencies = [ "anyhow", "heck", "im-rc", - "indexmap 2.13.1", + "indexmap 2.14.0", "log", "petgraph 0.6.5", "serde", @@ -8694,12 +10278,12 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.246.2" +version = "0.249.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61fb705ce81adde29d2a8e99d87995e39a6e927358c91398f374474746070ef7" +checksum = "69830ccbbf41c55eb585991659fb70867ef628193af3a495f09a6956f7615e59" dependencies = [ "leb128fmt", - "wasmparser 0.246.2", + "wasmparser 0.249.0", ] [[package]] @@ -8709,7 +10293,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap 2.13.1", + "indexmap 2.14.0", "wasm-encoder 0.244.0", "wasmparser 0.244.0", ] @@ -8733,9 +10317,9 @@ version = "0.243.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6d8db401b0528ec316dfbe579e6ab4152d61739cfe076706d2009127970159d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "hashbrown 0.15.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "semver", "serde", ] @@ -8746,20 +10330,20 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "hashbrown 0.15.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "semver", ] [[package]] name = "wasmparser" -version = "0.246.2" +version = "0.249.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71cde4757396defafd25417cfb36aa3161027d06d865b0c24baaae229aac005d" +checksum = "30538cae9a794215f490b532df01c557e2e2bfac92569482554acd0992a102ea" dependencies = [ - "bitflags 2.11.0", - "indexmap 2.13.1", + "bitflags 2.11.1", + "indexmap 2.14.0", "semver", ] @@ -8783,7 +10367,7 @@ dependencies = [ "addr2line", "anyhow", "async-trait", - "bitflags 2.11.0", + "bitflags 2.11.1", "bumpalo", "cc", "cfg-if", @@ -8792,7 +10376,7 @@ dependencies = [ "fxprof-processed-profile", "gimli", "hashbrown 0.15.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "ittapi", "libc", "log", @@ -8842,7 +10426,7 @@ dependencies = [ "cranelift-bitset", "cranelift-entity", "gimli", - "indexmap 2.13.1", + "indexmap 2.14.0", "log", "object", "postcard", @@ -8871,7 +10455,7 @@ dependencies = [ "rustix 1.1.4", "serde", "serde_derive", - "sha2", + "sha2 0.10.9", "toml", "wasmtime-environ", "windows-sys 0.61.2", @@ -9028,9 +10612,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "587699ca7cae16b4a234ffcc834f37e75675933d533809919b52975f5609e2ef" dependencies = [ "anyhow", - "bitflags 2.11.0", + "bitflags 2.11.1", "heck", - "indexmap 2.13.1", + "indexmap 2.14.0", "wit-parser 0.243.0", ] @@ -9042,7 +10626,7 @@ checksum = "fc2eb9dc95baed3cd86fdfebf9f9f333337eb308bf8bd973e0c7b06d9418c35f" dependencies = [ "anyhow", "async-trait", - "bitflags 2.11.0", + "bitflags 2.11.1", "bytes", "cap-fs-ext", "cap-net-ext", @@ -9089,31 +10673,31 @@ dependencies = [ [[package]] name = "wast" -version = "246.0.2" +version = "249.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe3fe8e3bf88ad96d031b4181ddbd64634b17cb0d06dfc3de589ef43591a9a62" +checksum = "2474a321bf9ae2808e9fa23ac4ec2b27300e70985e30bcb5a38d43b76bfc901a" dependencies = [ "bumpalo", "leb128fmt", "memchr", "unicode-width 0.2.2", - "wasm-encoder 0.246.2", + "wasm-encoder 0.249.0", ] [[package]] name = "wat" -version = "1.246.2" +version = "1.249.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd7fda1199b94fff395c2d19a153f05dbe7807630316fa9673367666fd2ad8c" +checksum = "28af699d0a9c7e4e250b7b8e36167ae5215fbb4b7ae526bb4ce7b234ba0afc90" dependencies = [ - "wast 246.0.2", + "wast 249.0.0", ] [[package]] name = "web-sys" -version = "0.3.94" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd70027e39b12f0849461e08ffc50b9cd7688d942c1c8e3c7b22273236b4dd0a" +checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" dependencies = [ "js-sys", "wasm-bindgen", @@ -9129,6 +10713,24 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.7", +] + [[package]] name = "webpki-roots" version = "1.0.7" @@ -9145,7 +10747,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a69a60bcbe1475c5dc9ec89210ade54823d44f742e283cba64f98f89697c4cec" dependencies = [ "anyhow", - "bitflags 2.11.0", + "bitflags 2.11.1", "thiserror 2.0.18", "tracing", "wasmtime", @@ -9270,6 +10872,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -9461,9 +11074,9 @@ checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" [[package]] name = "winnow" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" dependencies = [ "memchr", ] @@ -9474,7 +11087,7 @@ version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "windows-sys 0.59.0", ] @@ -9487,6 +11100,12 @@ dependencies = [ "wit-bindgen-rust-macro", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "wit-bindgen-core" version = "0.51.0" @@ -9506,7 +11125,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck", - "indexmap 2.13.1", + "indexmap 2.14.0", "prettyplease", "syn 2.0.117", "wasm-metadata", @@ -9536,8 +11155,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags 2.11.0", - "indexmap 2.13.1", + "bitflags 2.11.1", + "indexmap 2.14.0", "log", "serde", "serde_derive", @@ -9556,7 +11175,7 @@ checksum = "df983a8608e513d8997f435bb74207bf0933d0e49ca97aa9d8a6157164b9b7fc" dependencies = [ "anyhow", "id-arena", - "indexmap 2.13.1", + "indexmap 2.14.0", "log", "semver", "serde", @@ -9574,7 +11193,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap 2.13.1", + "indexmap 2.14.0", "log", "semver", "serde", @@ -9655,6 +11274,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "z85" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6e61e59a957b7ccee15d2049f86e8bfd6f66968fcd88f018950662d9b86e675" + [[package]] name = "zerocopy" version = "0.8.48" @@ -9677,9 +11302,9 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] diff --git a/Cargo.toml b/Cargo.toml index eebf2a6c..eb03560f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,90 +1,28 @@ [workspace] members = [ - ".", + "src/function-stream", "protocol", "cli/cli", + "src/catalog", + "src/catalog_storage", + "src/common", + "src/config", + "src/coordinator", + "src/logger", + "src/runtime_common", + "src/servicer", + "src/sqlparser", + "src/streaming_runtime_core", + "src/streaming_runtime_operators", + "src/streaming_runtime_job_manager", + "src/streaming_runtime", + "src/streaming_planner", + "src/wasm_runtime", ] +resolver = "2" +default-members = ["src/function-stream"] -[package] -name = "function-stream" +# Shared crate metadata for workspace members (`version.workspace = true`, etc.). +[workspace.package] version = "0.6.0" edition = "2024" - -[lib] -name = "function_stream" -path = "src/lib.rs" - -[[bin]] -name = "function-stream" -path = "src/main.rs" - - -[dependencies] -tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "sync", "time", "net", "signal"] } -serde = { version = "1.0", features = ["derive"] } -serde_yaml = "0.9" -serde_json = "1.0" -uuid = { version = "1.0", features = ["v4", "v7"] } -log = "0.4" -tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } -tracing-appender = "0.2" -anyhow = "1.0" -thiserror = "2" -tonic = { version = "0.12", features = ["default"] } -async-trait = "0.1" -num_cpus = "1.0" -protocol = { path = "./protocol" } -prost = "0.13" -rdkafka = { version = "0.38", features = ["cmake-build", "ssl", "gssapi", "curl"] } -crossbeam-channel = "0.5" -wasmtime = { version = "41.0.3", features = ["component-model", "async"] } -base64 = "0.22" -wasmtime-wasi = "41.0.3" -rocksdb = { version = "0.21", features = ["multi-threaded-cf", "lz4"] } -bincode = { version = "2", features = ["serde"] } -chrono = "0.4" -tokio-stream = "0.1.18" -lru = "0.12" -parking_lot = "0.12" -arrow = { version = "55", default-features = false } -arrow-array = "55" -arrow-ipc = "55" -arrow-schema = { version = "55", features = ["serde"] } -parquet = "55" -object_store = { version = "0.12.5", features = ["aws"] } -bytes = "1" -futures = "0.3" -serde_json_path = "0.7" -xxhash-rust = { version = "0.8", features = ["xxh3"] } -proctitle = "0.1" -unicase = "2.7" -petgraph = "0.7" -rand = { version = "0.8", features = ["small_rng"] } -itertools = "0.14" -strum = { version = "0.26", features = ["derive"] } - -arrow-json = {version = '55.2.0'} -apache-avro = "0.21" -datafusion = {git = 'https://github.com/FunctionStream/datafusion', branch = '48.0.1/fs'} -datafusion-common = {git = 'https://github.com/FunctionStream/datafusion', branch = '48.0.1/fs'} -datafusion-execution = {git = 'https://github.com/FunctionStream/datafusion', branch = '48.0.1/fs'} -datafusion-expr = {git = 'https://github.com/FunctionStream/datafusion', branch = '48.0.1/fs'} -datafusion-physical-expr = {git = 'https://github.com/FunctionStream/datafusion', branch = '48.0.1/fs'} -datafusion-proto = {git = 'https://github.com/FunctionStream/datafusion', branch = '48.0.1/fs'} - -sqlparser = { git = "https://github.com/FunctionStream/sqlparser-rs", branch = "0.58.0/fs" } - -ahash = "0.8" -governor = "0.8.0" -lance = { version = "4.0.0", default-features = false, features = ["aws"] } -arrow-array-lance = { package = "arrow-array", version = "57.3.0" } -arrow-ipc-lance = { package = "arrow-ipc", version = "57.3.0" } - -[features] -default = ["incremental-cache", "python"] -incremental-cache = ["wasmtime/incremental-cache"] -python = [] - -[dev-dependencies] -tempfile = "3.27.0" diff --git a/Makefile b/Makefile index 78138dae..d6340d9d 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. APP_NAME := function-stream +# Version from root `[workspace.package]` (single source of truth). VERSION := $(shell grep '^version' Cargo.toml | head -1 | awk -F '"' '{print $$2}') DATE := $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") @@ -106,6 +107,7 @@ build: .check-env .ensure-target .build-wasm @RUSTFLAGS="$(OPTIMIZE_FLAGS)" \ cargo build --release \ --target $(TRIPLE) \ + -p $(APP_NAME) \ --features python \ --quiet $(call log,BUILD,CLI) @@ -118,14 +120,15 @@ build: .check-env .ensure-target .build-wasm build-lite: .check-env .ensure-target $(call log,BUILD,Rust Lite [$(OS_NAME) / $(TRIPLE)]) - @RUSTFLAGS="$(INDUSTRIAL_RUSTFLAGS)" \ + @RUSTFLAGS="$(OPTIMIZE_FLAGS)" \ cargo build --release \ --target $(TRIPLE) \ + -p $(APP_NAME) \ --no-default-features \ --features incremental-cache \ --quiet $(call log,BUILD,CLI for dist) - @RUSTFLAGS="$(INDUSTRIAL_RUSTFLAGS)" \ + @RUSTFLAGS="$(OPTIMIZE_FLAGS)" \ cargo build --release \ --target $(TRIPLE) \ -p function-stream-cli \ diff --git a/cli/cli/Cargo.toml b/cli/cli/Cargo.toml index 49c0a881..75bd6cb4 100644 --- a/cli/cli/Cargo.toml +++ b/cli/cli/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "function-stream-cli" -version = "0.1.0" -edition = "2021" +version.workspace = true +edition.workspace = true [[bin]] name = "cli" diff --git a/cli/cli/src/repl.rs b/cli/cli/src/repl.rs index b442bd07..01209ba1 100644 --- a/cli/cli/src/repl.rs +++ b/cli/cli/src/repl.rs @@ -17,7 +17,7 @@ use arrow_ipc::reader::StreamReader; use arrow_schema::DataType; use comfy_table::presets::UTF8_FULL; use comfy_table::{Attribute, Cell, Color, ContentArrangement, Table, TableComponent}; -use protocol::cli::{function_stream_service_client::FunctionStreamServiceClient, SqlRequest}; +use protocol::cli::{SqlRequest, function_stream_service_client::FunctionStreamServiceClient}; use rustyline::error::ReadlineError; use rustyline::{Config, DefaultEditor, EditMode}; use std::fmt; @@ -158,17 +158,17 @@ impl Repl { } // 3. Strict Data Check: Only proceed if data is explicitly present and non-empty - if let Some(bytes) = response.data { - if !bytes.is_empty() { - // format_arrow_data returns Ok(Some(Table)) ONLY if row_count > 0 - match self.format_arrow_data(&bytes) { - Ok(Some(table)) => println!("{}", table), - Ok(None) => { - // Data was present but contained 0 rows (e.g., empty result set) - // We print nothing here to keep output clean as requested - } - Err(e) => eprintln!("Failed to parse result data: {}", e), + if let Some(bytes) = response.data + && !bytes.is_empty() + { + // format_arrow_data returns Ok(Some(Table)) ONLY if row_count > 0 + match self.format_arrow_data(&bytes) { + Ok(Some(table)) => println!("{}", table), + Ok(None) => { + // Data was present but contained 0 rows (e.g., empty result set) + // We print nothing here to keep output clean as requested } + Err(e) => eprintln!("Failed to parse result data: {}", e), } } @@ -243,11 +243,7 @@ impl Repl { } } - if has_rows { - Ok(Some(table)) - } else { - Ok(None) - } + if has_rows { Ok(Some(table)) } else { Ok(None) } } fn extract_value(&self, column: &dyn Array, row: usize) -> String { @@ -317,7 +313,7 @@ impl Repl { #[cfg(unix)] let mut sigterm = { - use tokio::signal::unix::{signal, SignalKind}; + use tokio::signal::unix::{SignalKind, signal}; signal(SignalKind::terminate()).expect("failed to register SIGTERM handler") }; @@ -403,10 +399,8 @@ impl Repl { println!(); } - if !skip_save_history { - if let Some(ref mut ed) = repl.lock().await.editor { - let _ = ed.save_history(".function-stream-cli-history"); - } + if !skip_save_history && let Some(ref mut ed) = repl.lock().await.editor { + let _ = ed.save_history(".function-stream-cli-history"); } Ok(()) } @@ -448,10 +442,10 @@ impl Repl { } fn add_history_entry(&mut self, entry: &str) { - if let Some(ed) = self.editor.as_mut() { - if !entry.trim().is_empty() { - let _ = ed.add_history_entry(entry.trim()); - } + if let Some(ed) = self.editor.as_mut() + && !entry.trim().is_empty() + { + let _ = ed.add_history_entry(entry.trim()); } } diff --git a/docs/streaming-sql/Source/README-zh.md b/docs/streaming-sql/Source/README-zh.md index e38a763f..77075fc3 100644 --- a/docs/streaming-sql/Source/README-zh.md +++ b/docs/streaming-sql/Source/README-zh.md @@ -1,7 +1,20 @@ # Streaming SQL Source 文档 -当前 Streaming SQL Source 仅支持 Kafka。通常先创建 Source 表，再被 `CREATE STREAMING TABLE ... AS SELECT ...` 引用。 +当前 Streaming SQL Source 支持 Kafka、MQTT、HTTP、ROS/ROS2、robot-bag 等。通常先创建 Source 表，再被 `CREATE STREAMING TABLE ... AS SELECT ...` 引用。 ## 文档列表 +- [机器人场景 Source 总览](robot-sources-zh.md) - [Kafka Source](kafka-source-zh.md) +- [MQTT Source](mqtt-source-zh.md) +- [HTTP Source](http-source-zh.md) +- [ROS Source](ros-source-zh.md) +- [ROS2 Source](ros2-source-zh.md) +- [Robot Bag Source（ROS1 bag / PCD / MCAP / ROS2）](robot-bag-source-zh.md) + +## 英文文档 + +- [Source README (EN)](README.md) +- [Robot sources overview (EN)](robot-sources.md) +- [Robot bag source (EN)](robot-bag-source.md) +- [Kafka source (EN)](kafka-source.md) diff --git a/docs/streaming-sql/Source/README.md b/docs/streaming-sql/Source/README.md index a84d3437..1b24f733 100644 --- a/docs/streaming-sql/Source/README.md +++ b/docs/streaming-sql/Source/README.md @@ -1,6 +1,22 @@ # Streaming SQL Source Docs -## Supported sources +Create a source table with `CREATE TABLE ... WITH (connector='...', type='source', ...)`, then reference it in `CREATE STREAMING TABLE ... AS SELECT ...`. -- [Kafka Source](kafka-source.md) +## Documentation +### Overview + +- [Robot & industrial data sources](robot-sources.md) + +### Connectors + +| Connector | Doc (EN) | Doc (中文) | +|-----------|----------|------------| +| Kafka | [kafka-source.md](kafka-source.md) | [kafka-source-zh.md](kafka-source-zh.md) | +| MQTT | — | [mqtt-source-zh.md](mqtt-source-zh.md) | +| HTTP | — | [http-source-zh.md](http-source-zh.md) | +| ROS (ROS1) | — | [ros-source-zh.md](ros-source-zh.md) | +| ROS2 | — | [ros2-source-zh.md](ros2-source-zh.md) | +| Robot bag | [robot-bag-source.md](robot-bag-source.md) | [robot-bag-source-zh.md](robot-bag-source-zh.md) | + +中文总览：[README-zh.md](README-zh.md) diff --git a/docs/streaming-sql/Source/http-source-zh.md b/docs/streaming-sql/Source/http-source-zh.md new file mode 100644 index 00000000..2ca18b5a --- /dev/null +++ b/docs/streaming-sql/Source/http-source-zh.md @@ -0,0 +1,87 @@ +# HTTP Source（机器人 / REST） + +`http` source 面向机器人遥测与 REST 接口，配置命名对齐 Flink HTTP / `rest-lookup` 连接器习惯。 + +## 模式 + +| 模式 | 场景 | 必填参数 | +|------|------|----------| +| **poll**（默认，有 `url` 时） | 定时拉取机器人 HTTP API | `url`、`format` | +| **webhook**（有 `http.listen.port` 或 `mode='webhook'`） | 机器人/边缘网关 **POST** 上报 | `http.listen.port`、`format` | + +## 支持格式 + +- `json` +- `raw_string` +- `raw_bytes` + +## Flink 对齐参数 + +| 参数 | Flink 对应 | 说明 | +|------|------------|------| +| `connector='http'` | `connector` | 固定为 `http` | +| `url` | `url` | Poll 模式请求地址 | +| `format` | `format` | 如 `json` | +| `lookup-method` / `http.method` | `lookup-method` | `GET`（默认）/ `POST` / `PUT` | +| `scan.interval` | — | 轮询周期，如 `INTERVAL '1' SECOND` | +| `scan.interval.ms` | — | 轮询周期毫秒（默认 1000） | +| `http.request.body` / `body` | `http.request.body-template` | POST 请求体 | +| `header.` | 自定义 Header | 如 `'header.Authorization'='Bearer xxx'` | +| `http.request.timeout.ms` | `gid.connector.http.source.lookup.request.timeout` | 请求超时（默认 30000） | +| `http.response.split` | — | `single`（默认）/ `ndjson` / `json-array` | +| `rate_limit.messages_per_second` | — | 默认不限速 | + +## Webhook 参数（机器人上报） + +| 参数 | 说明 | 默认 | +|------|------|------| +| `http.listen.host` | 监听地址 | `0.0.0.0` | +| `http.listen.port` | 监听端口 | 必填 | +| `http.webhook.path` | POST 路径 | `/` | +| `mode` | `poll` / `webhook` | 按 `url` / 端口自动推断 | + +## 示例：轮询机器人状态 API + +```sql +CREATE TABLE robot_status ( + robot_id STRING, + battery DOUBLE, + pose STRING, + ts TIMESTAMP +) WITH ( + connector='http', + type='source', + url='http://127.0.0.1:8080/api/v1/robots/status', + 'lookup-method'='GET', + 'scan.interval'='INTERVAL ''2'' SECOND', + format='json' +); +``` + +## 示例：Webhook 接收机器人上报 + +```sql +CREATE TABLE robot_telemetry ( + robot_id STRING, + joint_states STRING, + ts TIMESTAMP +) WITH ( + connector='http', + type='source', + mode='webhook', + 'http.listen.host'='0.0.0.0', + 'http.listen.port'='18080', + 'http.webhook.path'='/ingest/telemetry', + format='json' +); +``` + +机器人向 `POST http://:18080/ingest/telemetry` 发送 JSON 即可。 + +## 并行度 + +Poll / Webhook 均建议 **`parallelism = 1`**。Webhook 多 subtask 会重复绑定端口。 + +## Checkpoint + +v1 不持久化 HTTP 偏移；Poll 模式按 `scan.interval` 重复拉取；Webhook 依赖上游重试。 diff --git a/docs/streaming-sql/Source/kafka-source-zh.md b/docs/streaming-sql/Source/kafka-source-zh.md index e3da7597..ed83185f 100644 --- a/docs/streaming-sql/Source/kafka-source-zh.md +++ b/docs/streaming-sql/Source/kafka-source-zh.md @@ -14,10 +14,24 @@ - `type='source'`（默认） - `topic='topic_in'` - `bootstrap.servers='host:9092'` -- `group.id='consumer_group'` +- `group.id='consumer_group'`（可选；未指定时自动生成为 `fs-<表名>-consumer` 并写入 catalog） - `format='json'|'raw_string'|'raw_bytes'` - `scan.startup.mode='earliest'|'latest'|'group-offsets'`（可选） +## Consumer Group + +创建 Kafka Source 表时若未指定 `group.id` 且未指定 `group.id.prefix`，系统会在保存 catalog 前自动分配： + +```text +group.id = fs-<表名>-consumer +``` + +`SHOW CREATE TABLE` 与重启后 catalog 恢复均使用该值，保证 offset 按固定 consumer group 提交。若需每个 subtask 独立 group，可设置 `group.id.prefix`，由 runtime 追加 job/subtask 后缀。 + +## 吞吐说明 + +Kafka Source **不限速**；默认 batch=4096、linger=20ms、fetch 高吞吐参数已内置。 + ## 示例 ```sql @@ -30,7 +44,8 @@ CREATE TABLE src_kafka_json ( type='source', topic='topic_in', 'bootstrap.servers'='127.0.0.1:9092', - 'group.id'='fs_demo', format='json' ); ``` + +上例未写 `group.id`，catalog 中会自动保存 `'group.id'='fs-src_kafka_json-consumer'`。 diff --git a/docs/streaming-sql/Source/kafka-source.md b/docs/streaming-sql/Source/kafka-source.md index 505a6f3e..b3dc5cf2 100644 --- a/docs/streaming-sql/Source/kafka-source.md +++ b/docs/streaming-sql/Source/kafka-source.md @@ -13,7 +13,35 @@ - `connector='kafka'` - `topic='topic_name'` - `bootstrap.servers='host:9092'` +- `group.id='consumer_group'` (optional; if omitted, defaults to `fs--consumer` and is persisted in the catalog) - `format='json'|'raw_string'|'raw_bytes'` -- `scan.startup.mode='earliest'|'latest'` (optional) -- `group.id='consumer_group'` (optional) +- `scan.startup.mode='earliest'|'latest'|'group-offsets'` (optional) + +## Consumer group + +When you create a Kafka source table without `group.id` and without `group.id.prefix`, the planner assigns before catalog save: + +```text +group.id = fs--consumer +``` + +`SHOW CREATE TABLE` and catalog reload after restart use this value so offsets commit under a stable consumer group. For per-subtask groups, set `group.id.prefix` instead; the runtime appends job/subtask suffixes. + +## Example + +```sql +CREATE TABLE src_kafka_json ( + user_id BIGINT, + event STRING, + ts TIMESTAMP +) WITH ( + connector='kafka', + type='source', + topic='topic_in', + 'bootstrap.servers'='127.0.0.1:9092', + format='json' +); +``` + +If `group.id` is omitted, the catalog stores `'group.id'='fs-src_kafka_json-consumer'`. diff --git a/docs/streaming-sql/Source/mqtt-source-zh.md b/docs/streaming-sql/Source/mqtt-source-zh.md new file mode 100644 index 00000000..a6d48a19 --- /dev/null +++ b/docs/streaming-sql/Source/mqtt-source-zh.md @@ -0,0 +1,49 @@ +# MQTT Source + +`mqtt` source 用于从 MQTT broker 订阅 topic 并持续消费消息。 + +## 支持格式 + +- `json` +- `raw_string` +- `raw_bytes` + +## 常用 WITH 参数 + +- `connector='mqtt'` +- `type='source'`（默认） +- `topic='sensors/#'` +- `mqtt.host='127.0.0.1'`（或 `host`） +- `mqtt.port='1883'`（或 `port`，默认 1883） +- `format='json'|'raw_string'|'raw_bytes'` +- `mqtt.client.id`（可选；未设置时运行时生成 `fs-{job_id}-{subtask}-mqtt`） +- `mqtt.username` / `mqtt.password`（可选） +- `mqtt.qos`（可选，0/1/2，默认 1） +- `mqtt.clean.session`（可选，默认 `true`） +- `mqtt.keep.alive.secs`（可选，默认 60） +- `rate_limit.messages_per_second`（可选）：**默认不配置 = 不限速** + +## 并行度 + +MQTT 订阅在多个 subtask 上会重复消费同一 topic，**建议 pipeline `parallelism = 1`**。 + +## Checkpoint + +v1 不向 catalog 持久化 MQTT 偏移；故障恢复依赖 broker 会话与 `client_id` / `clean_session` 配置（至少一次语义）。 + +## 示例 + +```sql +CREATE TABLE src_mqtt_json ( + device_id STRING, + value DOUBLE, + ts TIMESTAMP +) WITH ( + connector='mqtt', + type='source', + topic='sensors/data', + 'mqtt.host'='127.0.0.1', + 'mqtt.port'='1883', + format='json' +); +``` diff --git a/docs/streaming-sql/Source/robot-bag-source-zh.md b/docs/streaming-sql/Source/robot-bag-source-zh.md new file mode 100644 index 00000000..22c6a7d1 --- /dev/null +++ b/docs/streaming-sql/Source/robot-bag-source-zh.md @@ -0,0 +1,151 @@ +# Robot Bag Source（机器人录制文件） + +`robot-bag` source 用于回放机器人离线数据，支持多种工业常见文件格式。 + +## 支持格式 + +| `bag.format` | 文件类型 | 说明 | +|--------------|----------|------| +| `auto` | 按扩展名/魔数推断 | 默认 | +| `mcap` | `.mcap` | Foxglove / MCAP 录制 | +| `ros1` | `.bag` | ROS1 rosbag（v1.02 / v2.0，含 chunk/index） | +| `ros2` | `.db3` 或目录 | ROS2 rosbag2（SQLite） | +| `pcd` | `.pcd` 或目录 | PCL PCD v0.7（ascii / binary / binary_compressed） | +| `jsonl` | `.jsonl` / `.ndjson` | 每行一条 JSON | +| `csv` | `.csv` | 跳过表头，每行一条记录 | + +消息体解析格式由 `format` 指定（通常 `json` 或 `raw_bytes`）。 + +## 常用 WITH 参数 + +| 参数 | 说明 | 默认 | +|------|------|------| +| `connector='robot-bag'` | 固定 | — | +| `path` | 文件或目录 | 必填 | +| `bag.format` | 见上表 | `auto` | +| `topic` | MCAP/ROS1/ROS2 按 topic 过滤 | 全部 | +| `pcd.emit.mode` | PCD：`point`（每点一条）或 `cloud`（每文件一条） | `point` | +| `replay.interval.ms` | 两条消息间隔（0=尽快） | `0` | +| `loop` | 播完后是否循环 | `false` | +| `format` | 消息解码格式 | 必填 | + +## 自动识别（`bag.format='auto'`） + +- **目录**含 `.pcd` → PCD +- **目录**无 `.pcd` → ROS2 bag +- **文件**头为 `#ROSBAG V2.0` / `#ROSBAG V1.02` → ROS1 +- **扩展名**：`.mcap`、`.bag`、`.db3`、`.pcd`、`.jsonl`、`.csv` + +## 示例：ROS1 bag + +```sql +CREATE TABLE robot_ros1_bag ( + payload BYTES, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/data/lidar_run.bag', + 'bag.format'='ros1', + topic='/velodyne_points', + format='raw_bytes' +); +``` + +ROS1 bag 通过标准 `#ROSBAG V2.0` 头解析 chunk/index，按时间戳排序回放。`sensor_msgs/PointCloud2` 等二进制消息请使用 `format='raw_bytes'`。 + +## 示例：PCD 点云（逐点） + +```sql +CREATE TABLE lidar_points ( + x DOUBLE, + y DOUBLE, + z DOUBLE, + intensity DOUBLE, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/data/scans/', + 'bag.format'='pcd', + 'pcd.emit.mode'='point', + format='json' +); +``` + +每个点输出一条 JSON，字段名来自 PCD header（`x`、`y`、`z`、`intensity` 等），并附带 `_source_file`、`_point_index`。PCD 解析遵循 PCL v0.7，支持 `ascii`、`binary`、`binary_compressed`（LZF + SoA→AoS 重排）。目录模式下按文件名排序依次加载所有 `.pcd`。 + +## 示例：PCD 整云（单条 JSON） + +```sql +CREATE TABLE lidar_cloud ( + payload STRING, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/data/scan.pcd', + 'bag.format'='pcd', + 'pcd.emit.mode'='cloud', + format='json' +); +``` + +## 示例：MCAP + +```sql +CREATE TABLE robot_mcap ( + robot_id STRING, + pose STRING, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/data/robot_run.mcap', + 'bag.format'='mcap', + topic='/odom', + format='json' +); +``` + +## 示例：ROS2 bag 目录 + +```sql +CREATE TABLE robot_ros2_bag ( + joint_name STRING, + position DOUBLE, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/data/rosbag2_2024', + 'bag.format'='ros2', + topic='/joint_states', + format='raw_bytes' +); +``` + +## 示例：JSONL 循环回放 + +```sql +CREATE TABLE robot_jsonl ( + event STRING, + payload STRING, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/logs/robot_events.jsonl', + 'bag.format'='jsonl', + format='json', + loop='true' +); +``` + +## 回放完成后 + +`loop=false` 时读完文件先 flush 缓冲区，再发送 `EndOfStream`；`loop=true` 时重新加载文件循环播放。可用 `replay.interval.ms` 模拟实时播放间隔。 + +## 并行度 + +建议 **`parallelism = 1`**。 diff --git a/docs/streaming-sql/Source/robot-bag-source.md b/docs/streaming-sql/Source/robot-bag-source.md new file mode 100644 index 00000000..d2cf36b2 --- /dev/null +++ b/docs/streaming-sql/Source/robot-bag-source.md @@ -0,0 +1,153 @@ +# Robot Bag Source + +The `robot-bag` source replays offline robot recordings from industrial file formats. + +## Supported file formats + +| `bag.format` | File type | Description | +|--------------|-----------|-------------| +| `auto` | Extension / magic bytes | Default | +| `mcap` | `.mcap` | Foxglove / MCAP recordings | +| `ros1` | `.bag` | ROS1 rosbag (v1.02 / v2.0, chunk + index) | +| `ros2` | `.db3` or directory | ROS2 rosbag2 (SQLite) | +| `pcd` | `.pcd` or directory | PCL PCD v0.7 (`ascii` / `binary` / `binary_compressed`) | +| `jsonl` | `.jsonl` / `.ndjson` | One JSON object per line | +| `csv` | `.csv` | Header row skipped, one record per line | + +Payload decoding is controlled by `format` (typically `json` or `raw_bytes`). + +## Common `WITH` options + +| Option | Description | Default | +|--------|-------------|---------| +| `connector='robot-bag'` | Required | — | +| `path` | File or directory | Required | +| `bag.format` | See table above | `auto` | +| `topic` | Filter by topic (MCAP / ROS1 / ROS2) | All topics | +| `pcd.emit.mode` | PCD: `point` (one row per point) or `cloud` (one row per file) | `point` | +| `replay.interval.ms` | Delay between messages (`0` = as fast as possible) | `0` | +| `loop` | Reload file after EOF | `false` | +| `format` | Payload format (`json` / `raw_string` / `raw_bytes`) | Required | + +## Auto-detection + +When `bag.format='auto'`: + +- **Directory** with `.pcd` files → PCD +- **Directory** without `.pcd` → ROS2 bag +- **File** starting with `#ROSBAG V2.0` / `#ROSBAG V1.02` → ROS1 +- **Extension**: `.mcap`, `.bag`, `.db3`, `.pcd`, `.jsonl`, `.csv` + +## Example: ROS1 bag + +```sql +CREATE TABLE robot_ros1_bag ( + payload BYTES, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/data/lidar_run.bag', + 'bag.format'='ros1', + topic='/velodyne_points', + format='raw_bytes' +); +``` + +ROS1 bags are parsed via chunk/index records and replayed in timestamp order. Use `format='raw_bytes'` for binary messages such as `sensor_msgs/PointCloud2`. + +## Example: PCD point cloud (per point) + +```sql +CREATE TABLE lidar_points ( + x DOUBLE, + y DOUBLE, + z DOUBLE, + intensity DOUBLE, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/data/scans/', + 'bag.format'='pcd', + 'pcd.emit.mode'='point', + format='json' +); +``` + +Each point is emitted as a JSON object with field names from the PCD header (`x`, `y`, `z`, `intensity`, etc.) plus `_source_file` and `_point_index`. PCL v0.7 `binary_compressed` uses LZF decompression and SoA→AoS reordering. In directory mode, all `.pcd` files are loaded in sorted filename order. + +## Example: PCD whole cloud (single JSON row) + +```sql +CREATE TABLE lidar_cloud ( + payload STRING, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/data/scan.pcd', + 'bag.format'='pcd', + 'pcd.emit.mode'='cloud', + format='json' +); +``` + +## Example: MCAP + +```sql +CREATE TABLE robot_mcap ( + robot_id STRING, + pose STRING, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/data/robot_run.mcap', + 'bag.format'='mcap', + topic='/odom', + format='json' +); +``` + +## Example: ROS2 bag directory + +```sql +CREATE TABLE robot_ros2_bag ( + joint_name STRING, + position DOUBLE, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/data/rosbag2_2024', + 'bag.format'='ros2', + topic='/joint_states', + format='raw_bytes' +); +``` + +## Example: JSONL log with loop replay + +```sql +CREATE TABLE robot_jsonl ( + event STRING, + payload STRING, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/logs/robot_events.jsonl', + 'bag.format'='jsonl', + format='json', + loop='true' +); +``` + +## End of replay + +When `loop=false`, the source emits `EndOfStream` after the last message (buffered batches are flushed first). When `loop=true`, the file is reloaded and replay starts again. + +## Parallelism + +Use **`parallelism = 1`**. File replay is single-threaded and not partitionable. diff --git a/docs/streaming-sql/Source/robot-sources-zh.md b/docs/streaming-sql/Source/robot-sources-zh.md new file mode 100644 index 00000000..0c50934f --- /dev/null +++ b/docs/streaming-sql/Source/robot-sources-zh.md @@ -0,0 +1,98 @@ +# 机器人场景 Source 指南 + +面向移动机器人、机械臂、AGV 等设备的流式接入说明。配置风格参考 **Apache Flink** 连接器（`url`、`format`、`scan.interval`、`lookup-method` 等）。 + +## 已支持 Source + +| Connector | 典型场景 | 文档 | +|-----------|----------|------| +| **kafka** | 经 ROS2/网关桥接后的汇聚 topic、平台总线 | [kafka-source-zh.md](kafka-source-zh.md) | +| **mqtt** | 设备直连、边缘 MQTT Broker、IoT 规则引擎 | [mqtt-source-zh.md](mqtt-source-zh.md) | +| **http** | REST 轮询机器人 API；Webhook 接收 POST 遥测 | [http-source-zh.md](http-source-zh.md) | +| **ros** | ROS1 topic（rosbridge WebSocket） | [ros-source-zh.md](ros-source-zh.md) | +| **ros2** | ROS2 topic（rosbridge_server） | [ros2-source-zh.md](ros2-source-zh.md) | +| **robot-bag** | MCAP / ROS1 bag / ROS2 bag / PCD / JSONL / CSV 离线回放 | [robot-bag-source-zh.md](robot-bag-source-zh.md) | + +## 推荐选型 + +```text +机器人 ──MQTT──► Broker ──► function-stream (mqtt source) +机器人 ──HTTP POST──► function-stream (http webhook) +调度系统 ──HTTP GET API──► function-stream (http poll) +ROS/ROS2 ──rosbridge──► function-stream (ros / ros2 source) +离线录制 ──bag/MCAP/PCD──► function-stream (robot-bag source) +``` + +| 数据形态 | 推荐 Connector | 说明 | +|----------|----------------|------| +| 高频遥测、弱网重传 | `mqtt` | QoS 1/2、`client_id` 稳定 | +| 云平台/第三方 REST 回调 | `http` webhook | `http.listen.port` + `http.webhook.path` | +| 定时查询机器人状态 | `http` poll | `url` + `scan.interval` | +| ROS/ROS2 实时 topic | `ros` / `ros2` | rosbridge WebSocket | +| 离线 bag / MCAP / PCD / 日志 | `robot-bag` | `path` + `bag.format` | +| 已有 Kafka 管线 | `kafka` | `group.id`、`scan.startup.mode` | + +## robot-bag 格式对照 + +| `bag.format` | 输入 | topic 过滤 | 推荐 `format` | +|--------------|------|------------|---------------| +| `ros1` | `.bag` 文件 | 支持 | 二进制 ROS 消息用 `raw_bytes` | +| `ros2` | `.db3` 或目录 | 支持 | `raw_bytes` | +| `mcap` | `.mcap` 文件 | 支持 | `json` 或 `raw_bytes` | +| `pcd` | `.pcd` 或目录 | 不支持 | `json`（`pcd.emit.mode=point` 或 `cloud`） | +| `jsonl` | `.jsonl` | 不支持 | `json` | +| `csv` | `.csv` | 不支持 | `json` 或 `raw_string` | + +英文文档：[robot-sources.md](robot-sources.md) · [robot-bag-source.md](robot-bag-source.md) + +## 规划中（常量已预留，待实现） + +| Connector | 机器人场景 | Flink / 业界参考 | +|-----------|------------|------------------| +| `websocket` | 实时关节流、仿真器推送 | WebSocket Table Source | +| `nats` | 边缘集群、JetStream | NATS connector | +| `pulsar` | 多租户机器人云平台 | Pulsar source | +| `redis` | 设备在线状态、Stream | Redis Streams | +| `sse` | 浏览器/轻量网关 Server-Sent Events | — | + +工业现场 **Modbus / OPC-UA** 建议经边缘网关转为 MQTT/HTTP 后接入，不在引擎内直接实现现场总线。 + +## 通用约定 + +1. **`format='json'`** 最常用；单字段二进制可用 `raw_bytes`。 +2. **`parallelism = 1`**：MQTT 订阅、HTTP Webhook 端口、HTTP 单 URL 轮询均避免重复消费。 +3. **吞吐**：Source 默认 batch=4096、linger=20ms；Kafka **无限速**；Delta 缓冲默认 256MB 并合并小 batch 写 Parquet。 +4. **Checkpoint**：Kafka 支持分区 offset；MQTT/HTTP v1 为至少一次，依赖 broker 会话或轮询间隔。 + +## 端到端示例（MQTT + Delta） + +```sql +-- 1. 机器人经 MQTT 上报 +CREATE TABLE robot_events ( + robot_id STRING, + event_type STRING, + payload STRING, + ts TIMESTAMP +) WITH ( + connector='mqtt', + type='source', + topic='factory/robot/+/telemetry', + 'mqtt.host'='127.0.0.1', + 'mqtt.port'='1883', + format='json' +); + +-- 2. 流式作业（示意） +CREATE STREAMING TABLE robot_sink +WITH ( + connector='delta', + type='sink', + path='/data/robot', + format='parquet' +) AS +SELECT * FROM robot_events; +``` + +## 添加新 Source 的开发清单 + +见项目内 MQTT/HTTP 实现路径：Proto `XxxSourceConfig` → Planner `SourceProvider` → Runtime `SourceOperator` → `ConnectorSourceDispatcher` 注册。 diff --git a/docs/streaming-sql/Source/robot-sources.md b/docs/streaming-sql/Source/robot-sources.md new file mode 100644 index 00000000..9b3ef633 --- /dev/null +++ b/docs/streaming-sql/Source/robot-sources.md @@ -0,0 +1,79 @@ +# Robot & Industrial Data Sources + +Guide for mobile robots, manipulators, AGV, and LiDAR pipelines. Connector options follow a Flink-style `WITH` clause (`url`, `format`, `scan.interval`, etc.). + +## Supported sources + +| Connector | Typical use | Doc | +|-----------|-------------|-----| +| **kafka** | Aggregated topics via ROS2 bridge / platform bus | [kafka-source.md](kafka-source.md) | +| **mqtt** | Device MQTT, edge broker, IoT rules | [mqtt-source-zh.md](mqtt-source-zh.md) | +| **http** | REST poll or Webhook telemetry | [http-source-zh.md](http-source-zh.md) | +| **ros** | ROS1 topics via rosbridge WebSocket | [ros-source-zh.md](ros-source-zh.md) | +| **ros2** | ROS2 topics via rosbridge_server | [ros2-source-zh.md](ros2-source-zh.md) | +| **robot-bag** | Offline MCAP / ROS1 bag / ROS2 bag / PCD / JSONL / CSV | [robot-bag-source.md](robot-bag-source.md) | + +## Selection guide + +```text +Robot ──MQTT──► Broker ──► function-stream (mqtt source) +Robot ──HTTP POST──► function-stream (http webhook) +Scheduler ──HTTP GET API──► function-stream (http poll) +ROS/ROS2 ──rosbridge──► function-stream (ros / ros2 source) +Offline recording ──bag/MCAP/PCD──► function-stream (robot-bag source) +``` + +| Data shape | Connector | Notes | +|------------|-----------|-------| +| High-frequency telemetry, weak network | `mqtt` | QoS 1/2, stable `client_id` | +| Cloud REST callback | `http` webhook | `http.listen.port` + `http.webhook.path` | +| Periodic robot status poll | `http` poll | `url` + `scan.interval` | +| Live ROS/ROS2 topic | `ros` / `ros2` | rosbridge WebSocket | +| Offline bag / MCAP / PCD / logs | `robot-bag` | `path` + `bag.format` | +| Existing Kafka pipeline | `kafka` | `group.id`, `scan.startup.mode` | + +## robot-bag format matrix + +| `bag.format` | Input | Topic filter | Typical `format` | +|--------------|-------|--------------|------------------| +| `ros1` | `.bag` file | Yes | `raw_bytes` for CDR/binary ROS messages | +| `ros2` | `.db3` or directory | Yes | `raw_bytes` | +| `mcap` | `.mcap` file | Yes | `json` or `raw_bytes` | +| `pcd` | `.pcd` or directory | No | `json` (`pcd.emit.mode=point` or `cloud`) | +| `jsonl` | `.jsonl` | No | `json` | +| `csv` | `.csv` | No | `json` or `raw_string` | + +## Conventions + +1. Prefer **`format='json'`** for structured telemetry; use **`raw_bytes`** for ROS binary payloads. +2. Set **`parallelism = 1`** for MQTT, HTTP webhook, HTTP poll, rosbridge, and robot-bag sources. +3. Kafka source has **no rate limit**; default batch=4096, linger=20ms. +4. robot-bag replays files once unless `loop=true`; use `replay.interval.ms` to simulate real-time playback. + +## End-to-end example (robot-bag PCD → Delta) + +```sql +CREATE TABLE lidar_scan ( + x DOUBLE, + y DOUBLE, + z DOUBLE, + intensity DOUBLE, + ts TIMESTAMP +) WITH ( + connector='robot-bag', + type='source', + path='/data/lidar_frames/', + 'bag.format'='pcd', + 'pcd.emit.mode'='point', + format='json' +); + +CREATE STREAMING TABLE lidar_delta +WITH ( + connector='delta', + type='sink', + path='/data/lidar_out', + format='parquet' +) AS +SELECT x, y, z, intensity, ts FROM lidar_scan; +``` diff --git a/docs/streaming-sql/Source/ros-source-zh.md b/docs/streaming-sql/Source/ros-source-zh.md new file mode 100644 index 00000000..8d62b5ec --- /dev/null +++ b/docs/streaming-sql/Source/ros-source-zh.md @@ -0,0 +1,41 @@ +# ROS Source（rosbridge / ROS1） + +`ros` source 通过 [rosbridge_suite](https://github.com/RobotWebTools/rosbridge_suite) WebSocket 订阅 ROS topic。 + +ROS2 请使用 [`ros2`](ros2-source-zh.md) connector 或设置 `'ros.version'='ros2'`。 + +## 支持格式 + +- `json` +- `raw_string` / `raw_bytes` + +## 常用 WITH 参数 + +| 参数 | 说明 | 默认 | +|------|------|------| +| `connector='ros'` | 固定 | — | +| `topic` | ROS topic | 必填 | +| `url` | rosbridge WebSocket | `ws://127.0.0.1:9090` | +| `ros.message.field` | JSON 载荷字段 | `msg` | +| `format` | 如 `json` | 必填 | + +## 示例 + +```sql +CREATE TABLE ros1_chatter ( + data STRING, + ts TIMESTAMP +) WITH ( + connector='ros', + type='source', + topic='/chatter', + url='ws://127.0.0.1:9090', + format='json' +); +``` + +## 相关文档 + +- [ROS2 Source](ros2-source-zh.md) +- [Robot Bag 文件 Source](robot-bag-source-zh.md) +- [机器人 Source 总览](robot-sources-zh.md) diff --git a/docs/streaming-sql/Source/ros2-source-zh.md b/docs/streaming-sql/Source/ros2-source-zh.md new file mode 100644 index 00000000..39e567e7 --- /dev/null +++ b/docs/streaming-sql/Source/ros2-source-zh.md @@ -0,0 +1,58 @@ +# ROS2 Source（rosbridge） + +`ros2` source 通过 **rosbridge_suite** WebSocket 订阅 ROS2 topic，协议与 `ros` source 相同，默认按 ROS2 版本注册。 + +## 前置条件 + +ROS2 环境安装并启动 rosbridge： + +```bash +ros2 launch rosbridge_server rosbridge_websocket_launch.xml +``` + +默认 WebSocket：`ws://127.0.0.1:9090` + +## 支持格式 + +- `json`（推荐） +- `raw_string` / `raw_bytes` + +## 常用 WITH 参数 + +| 参数 | 说明 | 默认 | +|------|------|------| +| `connector='ros2'` | 固定 | — | +| `topic` | ROS2 topic，如 `/joint_states` | 必填 | +| `url` / `ros.url` | rosbridge 地址 | `ws://127.0.0.1:9090` | +| `ros.message.field` | publish 帧中的 JSON 字段 | `msg` | +| `format` | 如 `json` | 必填 | + +## 示例 + +```sql +CREATE TABLE ros2_joints ( + name STRING, + position STRING, + ts TIMESTAMP +) WITH ( + connector='ros2', + type='source', + topic='/joint_states', + url='ws://127.0.0.1:9090', + format='json' +); +``` + +## ROS1 与 ROS2 区别 + +| 项 | `ros` | `ros2` | +|----|-------|--------| +| 默认 `ros.version` | ros1 | ros2 | +| 桥接服务 | rosbridge (ROS1) | rosbridge_server (ROS2) | +| WebSocket 协议 | 相同 | 相同 | + +也可在 `connector='ros'` 时显式设置 `'ros.version'='ros2'`。 + +## 并行度 + +建议 **`parallelism = 1`**。 diff --git a/examples/examples-validator/Cargo.toml b/examples/examples-validator/Cargo.toml index de87dbb3..9c204935 100644 --- a/examples/examples-validator/Cargo.toml +++ b/examples/examples-validator/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "examples-validator" version = "0.1.0" -edition = "2021" +edition = "2024" [workspace] diff --git a/protocol/Cargo.toml b/protocol/Cargo.toml index 51b1f3c1..96501c30 100644 --- a/protocol/Cargo.toml +++ b/protocol/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "protocol" -version = "0.1.0" -edition = "2024" +version.workspace = true +edition.workspace = true description = "Protocol Buffers protocol definitions for function stream" license = "MIT OR Apache-2.0" repository = "https://github.com/your-username/rust-function-stream" diff --git a/protocol/proto/function_stream_graph.proto b/protocol/proto/function_stream_graph.proto index 18903e4f..b726af6f 100644 --- a/protocol/proto/function_stream_graph.proto +++ b/protocol/proto/function_stream_graph.proto @@ -75,9 +75,103 @@ message ConnectorOp { IcebergSinkConfig iceberg_sink = 11; S3SinkConfig s3_sink = 12; LanceDbSinkConfig lancedb_sink = 13; + MqttSourceConfig mqtt_source = 14; + HttpSourceConfig http_source = 15; + RosSourceConfig ros_source = 16; + RobotBagSourceConfig robot_bag_source = 17; } } +// ─────────────────────── ROS / rosbridge Source ─────────────────────── + +enum RosVersion { + ROS_VERSION_ROS1 = 0; + ROS_VERSION_ROS2 = 1; +} + +message RosSourceConfig { + string url = 1; + string topic = 2; + string message_field = 3; + FormatConfig format = 4; + BadDataPolicy bad_data_policy = 5; + map client_configs = 6; + RosVersion ros_version = 7; +} + +// ─────────────────────── Robot bag / file Source ─────────────────────── + +enum RobotBagFormat { + ROBOT_BAG_AUTO = 0; + ROBOT_BAG_MCAP = 1; + ROBOT_BAG_ROS2 = 2; + ROBOT_BAG_JSONL = 3; + ROBOT_BAG_CSV = 4; + ROBOT_BAG_ROS1 = 5; + ROBOT_BAG_PCD = 6; +} + +message RobotBagSourceConfig { + string path = 1; + RobotBagFormat bag_format = 2; + optional string topic = 3; + uint64 replay_interval_ms = 4; + bool loop_replay = 5; + FormatConfig format = 6; + BadDataPolicy bad_data_policy = 7; + map client_configs = 8; + string pcd_emit_mode = 9; +} + +// ─────────────────────── HTTP Connector Configs (robot / REST) ─────────────────────── + +enum HttpSourceMode { + HTTP_SOURCE_POLL = 0; + HTTP_SOURCE_WEBHOOK = 1; +} + +enum HttpResponseSplit { + HTTP_RESPONSE_SINGLE = 0; + HTTP_RESPONSE_NDJSON = 1; + HTTP_RESPONSE_JSON_ARRAY = 2; +} + +message HttpSourceConfig { + HttpSourceMode mode = 1; + string url = 2; + string method = 3; + uint64 scan_interval_ms = 4; + optional string request_body = 5; + map headers = 6; + string listen_host = 7; + uint32 listen_port = 8; + string webhook_path = 9; + FormatConfig format = 10; + BadDataPolicy bad_data_policy = 11; + uint32 rate_limit_msgs_per_sec = 12; + map client_configs = 13; + uint32 request_timeout_ms = 14; + HttpResponseSplit response_split = 15; +} + +// ─────────────────────── MQTT Connector Configs ─────────────────────── + +message MqttSourceConfig { + string topic = 1; + string host = 2; + uint32 port = 3; + optional string client_id = 4; + optional string username = 5; + optional string password = 6; + uint32 qos = 7; + bool clean_session = 8; + uint32 keep_alive_secs = 9; + FormatConfig format = 10; + BadDataPolicy bad_data_policy = 11; + uint32 rate_limit_msgs_per_sec = 12; + map client_configs = 13; +} + // ─────────────────────── Kafka Connector Configs ─────────────────────── message KafkaSourceConfig { diff --git a/src/catalog/Cargo.toml b/src/catalog/Cargo.toml new file mode 100644 index 00000000..2a50735c --- /dev/null +++ b/src/catalog/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "function-stream-catalog" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_catalog" +path = "src/lib.rs" + +[dependencies] +protocol = { path = "../../protocol" } diff --git a/src/catalog/src/error.rs b/src/catalog/src/error.rs new file mode 100644 index 00000000..f5a9378e --- /dev/null +++ b/src/catalog/src/error.rs @@ -0,0 +1,4 @@ +use std::error::Error; + +pub type CatalogError = Box; +pub type CatalogResult = Result; diff --git a/src/catalog/src/lib.rs b/src/catalog/src/lib.rs new file mode 100644 index 00000000..33fb2743 --- /dev/null +++ b/src/catalog/src/lib.rs @@ -0,0 +1,9 @@ +//! Catalog domain types and APIs. + +pub mod error; +pub mod meta_store; +pub mod stream_catalog; + +pub use error::{CatalogError, CatalogResult}; +pub use meta_store::MetaStore; +pub use stream_catalog::{GlobalStreamCatalog, StoredStreamingJob, StreamCatalog}; diff --git a/src/catalog/src/meta_store.rs b/src/catalog/src/meta_store.rs new file mode 100644 index 00000000..b361e03c --- /dev/null +++ b/src/catalog/src/meta_store.rs @@ -0,0 +1,23 @@ +use crate::CatalogResult; + +/// Synchronous metadata key-value backend for catalog records. +pub trait MetaStore: Send + Sync { + fn put(&self, key: &str, value: Vec) -> CatalogResult<()>; + fn get(&self, key: &str) -> CatalogResult>>; + fn delete(&self, key: &str) -> CatalogResult<()>; + fn scan_prefix(&self, prefix: &str) -> CatalogResult)>>; + + /// Atomic apply of many puts (`Some(value)`) and deletes (`None`). + /// + /// Backends should override this with a single transaction or write batch + /// when the storage engine supports it. + fn write_batch(&self, batch: Vec<(String, Option>)>) -> CatalogResult<()> { + for (key, value) in batch { + match value { + Some(value) => self.put(&key, value)?, + None => self.delete(&key)?, + } + } + Ok(()) + } +} diff --git a/src/catalog/src/stream_catalog.rs b/src/catalog/src/stream_catalog.rs new file mode 100644 index 00000000..a5641a34 --- /dev/null +++ b/src/catalog/src/stream_catalog.rs @@ -0,0 +1,69 @@ +use std::sync::Arc; + +use protocol::function_stream_graph::FsProgram; +use protocol::storage as pb; + +use crate::CatalogResult; + +/// One persisted streaming job row from catalog storage. +/// +/// This is intentionally storage-agnostic: the catalog keeps source checkpoint +/// payloads as protocol oneof envelopes and does not inspect source-specific +/// checkpoint data. +#[derive(Debug, Clone)] +pub struct StoredStreamingJob { + pub table_name: String, + pub program: FsProgram, + pub checkpoint_interval_ms: u64, + pub latest_checkpoint_epoch: u64, + pub source_checkpoints: Vec, +} + +/// Interface exposed by the stream catalog manager. +/// +/// The concrete table and planning types are generic so this crate can define +/// the catalog boundary without depending on the monolithic SQL/runtime crates. +pub trait StreamCatalog: Send + Sync { + fn persist_streaming_job( + &self, + table_name: &str, + fs_program: &FsProgram, + comment: &str, + checkpoint_interval_ms: u64, + ) -> CatalogResult<()>; + + fn remove_streaming_job(&self, table_name: &str) -> CatalogResult<()>; + + fn commit_job_checkpoint( + &self, + table_name: &str, + epoch: u64, + source_checkpoints: Vec, + ) -> CatalogResult<()>; + + fn load_streaming_job_definitions(&self) -> CatalogResult>; + + fn add_catalog_table(&self, table: Table) -> CatalogResult<()>; + fn has_catalog_table(&self, name: &str) -> bool; + fn drop_catalog_table(&self, table_name: &str, if_exists: bool) -> CatalogResult<()>; + fn restore_from_store(&self) -> CatalogResult<()>; + fn acquire_planning_context(&self) -> PlanningContext; + fn list_catalog_tables(&self) -> CatalogResult>>; + fn get_catalog_table(&self, name: &str) -> CatalogResult>>; + + fn add_table(&self, table: StreamTable) -> CatalogResult<()>; + fn has_stream_table(&self, name: &str) -> bool; + fn drop_table(&self, table_name: &str, if_exists: bool) -> CatalogResult<()>; + fn list_stream_tables(&self) -> Vec>; + fn get_stream_table(&self, name: &str) -> Option>; +} + +/// Process-global catalog access boundary. +pub trait GlobalStreamCatalog: Send + Sync { + fn init_global(manager: Arc) -> CatalogResult<()>; + fn try_global() -> Option>; + + fn global() -> CatalogResult> { + Self::try_global().ok_or_else(|| "CatalogManager not initialized".into()) + } +} diff --git a/src/catalog_storage/Cargo.toml b/src/catalog_storage/Cargo.toml new file mode 100644 index 00000000..3bb9fed8 --- /dev/null +++ b/src/catalog_storage/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "function-stream-catalog-storage" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_catalog_storage" +path = "src/lib.rs" + +[dependencies] +anyhow = "1.0" +function-stream-catalog = { path = "../catalog" } +parking_lot = "0.12" +rocksdb = { version = "0.21", features = ["multi-threaded-cf", "lz4"] } diff --git a/src/catalog_storage/src/lib.rs b/src/catalog_storage/src/lib.rs new file mode 100644 index 00000000..8fdca0bf --- /dev/null +++ b/src/catalog_storage/src/lib.rs @@ -0,0 +1,10 @@ +//! Persistent catalog storage implementations. +//! +//! The stream catalog manager and task persistence (`stream_catalog/`, `task/`) live in this +//! package and are compiled as part of `function-stream` via `#[path]` in `src/lib.rs` / `src/main.rs`. + +pub mod memory; +pub mod rocksdb; + +pub use memory::InMemoryMetaStore; +pub use rocksdb::RocksDbMetaStore; diff --git a/src/catalog_storage/src/memory.rs b/src/catalog_storage/src/memory.rs new file mode 100644 index 00000000..df6fd0c0 --- /dev/null +++ b/src/catalog_storage/src/memory.rs @@ -0,0 +1,63 @@ +use std::collections::HashMap; + +use function_stream_catalog::{CatalogResult, MetaStore}; +use parking_lot::RwLock; + +/// In-process KV store for single-node deployments and tests. +pub struct InMemoryMetaStore { + db: RwLock>>, +} + +impl InMemoryMetaStore { + pub fn new() -> Self { + Self { + db: RwLock::new(HashMap::new()), + } + } +} + +impl Default for InMemoryMetaStore { + fn default() -> Self { + Self::new() + } +} + +impl MetaStore for InMemoryMetaStore { + fn put(&self, key: &str, value: Vec) -> CatalogResult<()> { + self.db.write().insert(key.to_string(), value); + Ok(()) + } + + fn get(&self, key: &str) -> CatalogResult>> { + Ok(self.db.read().get(key).cloned()) + } + + fn delete(&self, key: &str) -> CatalogResult<()> { + self.db.write().remove(key); + Ok(()) + } + + fn scan_prefix(&self, prefix: &str) -> CatalogResult)>> { + let db = self.db.read(); + Ok(db + .iter() + .filter(|(key, _)| key.starts_with(prefix)) + .map(|(key, value)| (key.clone(), value.clone())) + .collect()) + } + + fn write_batch(&self, batch: Vec<(String, Option>)>) -> CatalogResult<()> { + let mut db = self.db.write(); + for (key, value) in batch { + match value { + Some(value) => { + db.insert(key, value); + } + None => { + db.remove(&key); + } + } + } + Ok(()) + } +} diff --git a/src/catalog_storage/src/rocksdb.rs b/src/catalog_storage/src/rocksdb.rs new file mode 100644 index 00000000..192b9d80 --- /dev/null +++ b/src/catalog_storage/src/rocksdb.rs @@ -0,0 +1,81 @@ +use std::path::Path; +use std::sync::Arc; + +use anyhow::Context; +use function_stream_catalog::{CatalogResult, MetaStore}; +use rocksdb::{DB, Direction, IteratorMode, Options, WriteBatch}; + +/// RocksDB-backed catalog metadata store. +pub struct RocksDbMetaStore { + db: Arc, +} + +impl RocksDbMetaStore { + pub fn open>(path: P) -> anyhow::Result { + let path = path.as_ref(); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("stream catalog: create parent directory {parent:?}"))?; + } + let mut opts = Options::default(); + opts.create_if_missing(true); + let db = DB::open(&opts, path) + .with_context(|| format!("stream catalog: open RocksDB at {}", path.display()))?; + Ok(Self { db: Arc::new(db) }) + } +} + +impl MetaStore for RocksDbMetaStore { + fn put(&self, key: &str, value: Vec) -> CatalogResult<()> { + self.db + .put(key.as_bytes(), value.as_slice()) + .map_err(|e| format!("stream catalog store put: {e}").into()) + } + + fn get(&self, key: &str) -> CatalogResult>> { + self.db + .get(key.as_bytes()) + .map_err(|e| format!("stream catalog store get: {e}").into()) + } + + fn delete(&self, key: &str) -> CatalogResult<()> { + self.db + .delete(key.as_bytes()) + .map_err(|e| format!("stream catalog store delete: {e}").into()) + } + + fn scan_prefix(&self, prefix: &str) -> CatalogResult)>> { + let mut out = Vec::new(); + let iter = self + .db + .iterator(IteratorMode::From(prefix.as_bytes(), Direction::Forward)); + for item in iter { + let (key, value) = item.map_err(|e| format!("stream catalog store scan: {e}"))?; + let key = String::from_utf8(key.to_vec()) + .map_err(|e| format!("stream catalog store: invalid utf8 key: {e}"))?; + if !key.starts_with(prefix) { + break; + } + out.push((key, value.to_vec())); + } + Ok(out) + } + + fn write_batch(&self, batch: Vec<(String, Option>)>) -> CatalogResult<()> { + if batch.is_empty() { + return Ok(()); + } + + let mut write_batch = WriteBatch::default(); + for (key, value) in batch { + match value { + Some(value) => write_batch.put(key.as_bytes(), value.as_slice()), + None => write_batch.delete(key.as_bytes()), + } + } + + self.db + .write(write_batch) + .map_err(|e| format!("stream catalog store write_batch: {e}").into()) + } +} diff --git a/src/storage/stream_catalog/codec.rs b/src/catalog_storage/src/stream_catalog/codec.rs similarity index 100% rename from src/storage/stream_catalog/codec.rs rename to src/catalog_storage/src/stream_catalog/codec.rs diff --git a/src/storage/stream_catalog/manager.rs b/src/catalog_storage/src/stream_catalog/manager.rs similarity index 99% rename from src/storage/stream_catalog/manager.rs rename to src/catalog_storage/src/stream_catalog/manager.rs index fa810e8d..cbdf30b3 100644 --- a/src/storage/stream_catalog/manager.rs +++ b/src/catalog_storage/src/stream_catalog/manager.rs @@ -627,7 +627,7 @@ pub fn restore_global_catalog_from_store() { } pub fn restore_streaming_jobs_from_store() { - use crate::runtime::streaming::job::JobManager; + use crate::streaming::job::JobManager; let Some(catalog) = CatalogManager::try_global() else { warn!("CatalogManager not available; skipping streaming job restore"); @@ -742,7 +742,7 @@ mod tests { use crate::sql::schema::column_descriptor::ColumnDescriptor; use crate::sql::schema::table::CatalogEntity; use crate::sql::schema::temporal_pipeline_config::TemporalPipelineConfig; - use crate::storage::stream_catalog::InMemoryMetaStore; + use crate::stream_catalog::InMemoryMetaStore; use super::CatalogManager; diff --git a/src/storage/stream_catalog/meta_store.rs b/src/catalog_storage/src/stream_catalog/meta_store.rs similarity index 100% rename from src/storage/stream_catalog/meta_store.rs rename to src/catalog_storage/src/stream_catalog/meta_store.rs diff --git a/src/storage/stream_catalog/mod.rs b/src/catalog_storage/src/stream_catalog/mod.rs similarity index 100% rename from src/storage/stream_catalog/mod.rs rename to src/catalog_storage/src/stream_catalog/mod.rs diff --git a/src/storage/stream_catalog/rocksdb_meta_store.rs b/src/catalog_storage/src/stream_catalog/rocksdb_meta_store.rs similarity index 98% rename from src/storage/stream_catalog/rocksdb_meta_store.rs rename to src/catalog_storage/src/stream_catalog/rocksdb_meta_store.rs index 1537f278..58c0b35c 100644 --- a/src/storage/stream_catalog/rocksdb_meta_store.rs +++ b/src/catalog_storage/src/stream_catalog/rocksdb_meta_store.rs @@ -20,7 +20,7 @@ use rocksdb::{DB, Direction, IteratorMode, Options, WriteBatch}; use super::MetaStore; -/// Single-node durable KV used by [`crate::storage::stream_catalog::CatalogManager`]. +/// Single-node durable KV used by [`crate::stream_catalog::CatalogManager`]. pub struct RocksDbMetaStore { db: Arc, } diff --git a/src/storage/task/factory.rs b/src/catalog_storage/src/task/factory.rs similarity index 100% rename from src/storage/task/factory.rs rename to src/catalog_storage/src/task/factory.rs diff --git a/src/storage/task/function_info.rs b/src/catalog_storage/src/task/function_info.rs similarity index 100% rename from src/storage/task/function_info.rs rename to src/catalog_storage/src/task/function_info.rs diff --git a/src/storage/task/mod.rs b/src/catalog_storage/src/task/mod.rs similarity index 100% rename from src/storage/task/mod.rs rename to src/catalog_storage/src/task/mod.rs diff --git a/src/storage/task/proto_codec.rs b/src/catalog_storage/src/task/proto_codec.rs similarity index 99% rename from src/storage/task/proto_codec.rs rename to src/catalog_storage/src/task/proto_codec.rs index 6c1bc8df..78a0426d 100644 --- a/src/storage/task/proto_codec.rs +++ b/src/catalog_storage/src/task/proto_codec.rs @@ -20,7 +20,7 @@ use protocol::storage::{ }; use serde::{Deserialize, Serialize}; -use crate::runtime::common::ComponentState; +use crate::common::ComponentState; use super::storage::TaskModuleBytes; diff --git a/src/storage/task/rocksdb_storage.rs b/src/catalog_storage/src/task/rocksdb_storage.rs similarity index 99% rename from src/storage/task/rocksdb_storage.rs rename to src/catalog_storage/src/task/rocksdb_storage.rs index cea0ceb9..a3927515 100644 --- a/src/storage/task/rocksdb_storage.rs +++ b/src/catalog_storage/src/task/rocksdb_storage.rs @@ -19,8 +19,8 @@ use super::proto_codec::{ encode_task_module_bytes, }; use super::storage::{StoredTaskInfo, TaskStorage}; +use crate::common::ComponentState; use crate::config::storage::RocksDBStorageConfig; -use crate::runtime::common::ComponentState; use anyhow::{Context, Result, anyhow}; use rocksdb::{ColumnFamilyDescriptor, DB, IteratorMode, Options, WriteBatch}; use std::path::Path; diff --git a/src/storage/task/storage.rs b/src/catalog_storage/src/task/storage.rs similarity index 97% rename from src/storage/task/storage.rs rename to src/catalog_storage/src/task/storage.rs index 156ee5d8..6cbec9ea 100644 --- a/src/storage/task/storage.rs +++ b/src/catalog_storage/src/task/storage.rs @@ -10,7 +10,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::common::ComponentState; +use crate::common::ComponentState; use anyhow::Result; use serde::{Deserialize, Serialize}; diff --git a/src/common/Cargo.toml b/src/common/Cargo.toml new file mode 100644 index 00000000..4c155c5e --- /dev/null +++ b/src/common/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "function-stream-common" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_common" +path = "src/lib.rs" diff --git a/src/common/fs_schema.rs b/src/common/src/fs_schema.rs similarity index 100% rename from src/common/fs_schema.rs rename to src/common/src/fs_schema.rs diff --git a/src/common/mod.rs b/src/common/src/legacy_mod.rs similarity index 100% rename from src/common/mod.rs rename to src/common/src/legacy_mod.rs diff --git a/src/common/src/lib.rs b/src/common/src/lib.rs new file mode 100644 index 00000000..0356f6de --- /dev/null +++ b/src/common/src/lib.rs @@ -0,0 +1,3 @@ +//! Shared types and utilities for FunctionStream crates. + +pub const CRATE_NAME: &str = "function-stream-common"; diff --git a/src/config/Cargo.toml b/src/config/Cargo.toml new file mode 100644 index 00000000..d1e437cd --- /dev/null +++ b/src/config/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "function-stream-config" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_config" +path = "src/lib.rs" + +[features] +default = ["python"] +python = [] + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_yaml = "0.9" +uuid = { version = "1.0", features = ["v4", "v7"] } diff --git a/src/config/global_config.rs b/src/config/src/global_config.rs similarity index 100% rename from src/config/global_config.rs rename to src/config/src/global_config.rs diff --git a/src/config/mod.rs b/src/config/src/lib.rs similarity index 61% rename from src/config/mod.rs rename to src/config/src/lib.rs index e60dcfde..c07c676c 100644 --- a/src/config/mod.rs +++ b/src/config/src/lib.rs @@ -1,14 +1,4 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//! Configuration loading and validation. pub mod global_config; pub mod loader; @@ -21,6 +11,11 @@ pub mod streaming_job; pub mod system; pub mod wasm_config; +// Compatibility shim for files that still reference `crate::config::*`. +pub mod config { + pub use crate::*; +} + pub use global_config::{ DEFAULT_OPERATOR_STATE_STORE_MEMORY_BYTES, DEFAULT_STREAMING_RUNTIME_MEMORY_BYTES, GlobalConfig, }; diff --git a/src/config/loader.rs b/src/config/src/loader.rs similarity index 100% rename from src/config/loader.rs rename to src/config/src/loader.rs diff --git a/src/config/log_config.rs b/src/config/src/log_config.rs similarity index 100% rename from src/config/log_config.rs rename to src/config/src/log_config.rs diff --git a/src/config/paths.rs b/src/config/src/paths.rs similarity index 100% rename from src/config/paths.rs rename to src/config/src/paths.rs diff --git a/src/config/python_config.rs b/src/config/src/python_config.rs similarity index 100% rename from src/config/python_config.rs rename to src/config/src/python_config.rs diff --git a/src/config/service_config.rs b/src/config/src/service_config.rs similarity index 100% rename from src/config/service_config.rs rename to src/config/src/service_config.rs diff --git a/src/config/storage.rs b/src/config/src/storage.rs similarity index 100% rename from src/config/storage.rs rename to src/config/src/storage.rs diff --git a/src/config/streaming_job.rs b/src/config/src/streaming_job.rs similarity index 100% rename from src/config/streaming_job.rs rename to src/config/src/streaming_job.rs diff --git a/src/config/system.rs b/src/config/src/system.rs similarity index 100% rename from src/config/system.rs rename to src/config/src/system.rs diff --git a/src/config/wasm_config.rs b/src/config/src/wasm_config.rs similarity index 100% rename from src/config/wasm_config.rs rename to src/config/src/wasm_config.rs diff --git a/src/coordinator/Cargo.toml b/src/coordinator/Cargo.toml new file mode 100644 index 00000000..f8a6e0b7 --- /dev/null +++ b/src/coordinator/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "function-stream-coordinator" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_coordinator" +path = "src/lib.rs" diff --git a/src/coordinator/analyze/analysis.rs b/src/coordinator/src/analyze/analysis.rs similarity index 100% rename from src/coordinator/analyze/analysis.rs rename to src/coordinator/src/analyze/analysis.rs diff --git a/src/coordinator/analyze/analyzer.rs b/src/coordinator/src/analyze/analyzer.rs similarity index 100% rename from src/coordinator/analyze/analyzer.rs rename to src/coordinator/src/analyze/analyzer.rs diff --git a/src/coordinator/analyze/mod.rs b/src/coordinator/src/analyze/mod.rs similarity index 100% rename from src/coordinator/analyze/mod.rs rename to src/coordinator/src/analyze/mod.rs diff --git a/src/coordinator/coordinator.rs b/src/coordinator/src/coordinator.rs similarity index 100% rename from src/coordinator/coordinator.rs rename to src/coordinator/src/coordinator.rs diff --git a/src/coordinator/mod.rs b/src/coordinator/src/coordinator_body.rs similarity index 95% rename from src/coordinator/mod.rs rename to src/coordinator/src/coordinator_body.rs index 86598bc5..0b5b7135 100644 --- a/src/coordinator/mod.rs +++ b/src/coordinator/src/coordinator_body.rs @@ -18,12 +18,14 @@ mod execution; mod execution_context; mod plan; mod runtime_context; +mod sql_classify; mod statement; mod streaming_table_options; mod tool; pub use coordinator::Coordinator; pub use dataset::{DataSet, ShowFunctionsResult}; +pub use sql_classify::classify_statement; pub use statement::{ CreateFunction, CreatePythonFunction, CreateTable, DropFunction, DropStreamingTableStatement, DropTableStatement, PythonModule, ShowCatalogTables, ShowCreateStreamingTable, ShowCreateTable, diff --git a/src/coordinator/dataset/data_set.rs b/src/coordinator/src/dataset/data_set.rs similarity index 100% rename from src/coordinator/dataset/data_set.rs rename to src/coordinator/src/dataset/data_set.rs diff --git a/src/coordinator/dataset/execute_result.rs b/src/coordinator/src/dataset/execute_result.rs similarity index 100% rename from src/coordinator/dataset/execute_result.rs rename to src/coordinator/src/dataset/execute_result.rs diff --git a/src/coordinator/dataset/mod.rs b/src/coordinator/src/dataset/mod.rs similarity index 100% rename from src/coordinator/dataset/mod.rs rename to src/coordinator/src/dataset/mod.rs diff --git a/src/coordinator/dataset/show_catalog_tables_result.rs b/src/coordinator/src/dataset/show_catalog_tables_result.rs similarity index 100% rename from src/coordinator/dataset/show_catalog_tables_result.rs rename to src/coordinator/src/dataset/show_catalog_tables_result.rs diff --git a/src/coordinator/dataset/show_create_streaming_table_result.rs b/src/coordinator/src/dataset/show_create_streaming_table_result.rs similarity index 100% rename from src/coordinator/dataset/show_create_streaming_table_result.rs rename to src/coordinator/src/dataset/show_create_streaming_table_result.rs diff --git a/src/coordinator/dataset/show_create_table_result.rs b/src/coordinator/src/dataset/show_create_table_result.rs similarity index 100% rename from src/coordinator/dataset/show_create_table_result.rs rename to src/coordinator/src/dataset/show_create_table_result.rs diff --git a/src/coordinator/dataset/show_functions_result.rs b/src/coordinator/src/dataset/show_functions_result.rs similarity index 98% rename from src/coordinator/dataset/show_functions_result.rs rename to src/coordinator/src/dataset/show_functions_result.rs index c16edf6d..18673f00 100644 --- a/src/coordinator/dataset/show_functions_result.rs +++ b/src/coordinator/src/dataset/show_functions_result.rs @@ -16,7 +16,7 @@ use arrow_array::{RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema}; use super::DataSet; -use crate::storage::task::FunctionInfo; +use crate::task::FunctionInfo; #[derive(Clone, Debug)] pub struct ShowFunctionsResult { diff --git a/src/coordinator/dataset/show_streaming_tables_result.rs b/src/coordinator/src/dataset/show_streaming_tables_result.rs similarity index 97% rename from src/coordinator/dataset/show_streaming_tables_result.rs rename to src/coordinator/src/dataset/show_streaming_tables_result.rs index cae597ac..5b5c8e84 100644 --- a/src/coordinator/dataset/show_streaming_tables_result.rs +++ b/src/coordinator/src/dataset/show_streaming_tables_result.rs @@ -16,7 +16,7 @@ use arrow_array::{Int32Array, StringArray}; use arrow_schema::{DataType, Field, Schema}; use super::DataSet; -use crate::runtime::streaming::job::StreamingJobSummary; +use crate::streaming::job::StreamingJobSummary; #[derive(Clone, Debug)] pub struct ShowStreamingTablesResult { diff --git a/src/coordinator/execution/executor.rs b/src/coordinator/src/execution/executor.rs similarity index 98% rename from src/coordinator/execution/executor.rs rename to src/coordinator/src/execution/executor.rs index 22c5c1b7..a58dc6c0 100644 --- a/src/coordinator/execution/executor.rs +++ b/src/coordinator/src/execution/executor.rs @@ -31,13 +31,13 @@ use crate::coordinator::statement::{ConfigSource, FunctionSource}; use crate::coordinator::streaming_table_options::{ parse_checkpoint_interval_ms, parse_pipeline_parallelism, }; -use crate::runtime::streaming::job::JobManager; -use crate::runtime::streaming::protocol::control::StopMode; -use crate::runtime::wasm::taskexecutor::TaskManager; use crate::sql::schema::catalog::ExternalTable; use crate::sql::schema::show_create_catalog_table; use crate::sql::schema::table::CatalogEntity; -use crate::storage::stream_catalog::CatalogManager; +use crate::stream_catalog::CatalogManager; +use crate::streaming::job::JobManager; +use crate::streaming::protocol::control::StopMode; +use crate::wasm::taskexecutor::TaskManager; #[derive(Error, Debug)] pub enum ExecuteError { diff --git a/src/coordinator/execution/mod.rs b/src/coordinator/src/execution/mod.rs similarity index 100% rename from src/coordinator/execution/mod.rs rename to src/coordinator/src/execution/mod.rs diff --git a/src/coordinator/execution_context.rs b/src/coordinator/src/execution_context.rs similarity index 100% rename from src/coordinator/execution_context.rs rename to src/coordinator/src/execution_context.rs diff --git a/src/coordinator/src/lib.rs b/src/coordinator/src/lib.rs new file mode 100644 index 00000000..6e5ad596 --- /dev/null +++ b/src/coordinator/src/lib.rs @@ -0,0 +1,3 @@ +//! Query planning and job coordination. + +pub const CRATE_NAME: &str = "function-stream-coordinator"; diff --git a/src/coordinator/plan/compile_error_plan.rs b/src/coordinator/src/plan/compile_error_plan.rs similarity index 100% rename from src/coordinator/plan/compile_error_plan.rs rename to src/coordinator/src/plan/compile_error_plan.rs diff --git a/src/coordinator/plan/create_function_plan.rs b/src/coordinator/src/plan/create_function_plan.rs similarity index 100% rename from src/coordinator/plan/create_function_plan.rs rename to src/coordinator/src/plan/create_function_plan.rs diff --git a/src/coordinator/plan/create_python_function_plan.rs b/src/coordinator/src/plan/create_python_function_plan.rs similarity index 100% rename from src/coordinator/plan/create_python_function_plan.rs rename to src/coordinator/src/plan/create_python_function_plan.rs diff --git a/src/coordinator/plan/create_table_plan.rs b/src/coordinator/src/plan/create_table_plan.rs similarity index 100% rename from src/coordinator/plan/create_table_plan.rs rename to src/coordinator/src/plan/create_table_plan.rs diff --git a/src/coordinator/plan/drop_function_plan.rs b/src/coordinator/src/plan/drop_function_plan.rs similarity index 100% rename from src/coordinator/plan/drop_function_plan.rs rename to src/coordinator/src/plan/drop_function_plan.rs diff --git a/src/coordinator/plan/drop_streaming_table_plan.rs b/src/coordinator/src/plan/drop_streaming_table_plan.rs similarity index 100% rename from src/coordinator/plan/drop_streaming_table_plan.rs rename to src/coordinator/src/plan/drop_streaming_table_plan.rs diff --git a/src/coordinator/plan/drop_table_plan.rs b/src/coordinator/src/plan/drop_table_plan.rs similarity index 100% rename from src/coordinator/plan/drop_table_plan.rs rename to src/coordinator/src/plan/drop_table_plan.rs diff --git a/src/coordinator/plan/logical_plan_visitor.rs b/src/coordinator/src/plan/logical_plan_visitor.rs similarity index 92% rename from src/coordinator/plan/logical_plan_visitor.rs rename to src/coordinator/src/plan/logical_plan_visitor.rs index cb22d24e..66797ced 100644 --- a/src/coordinator/plan/logical_plan_visitor.rs +++ b/src/coordinator/src/plan/logical_plan_visitor.rs @@ -11,7 +11,6 @@ // limitations under the License. use datafusion::sql::sqlparser::ast::{ObjectType, Statement as DFStatement}; -use sqlparser::ast::Statement; use tracing::debug; use crate::coordinator::analyze::analysis::Analysis; @@ -28,9 +27,8 @@ use crate::coordinator::statement::{ StatementVisitorResult, StopFunction, StreamingTableStatement, }; use crate::sql::analysis::StreamSchemaProvider; +use crate::sql::schema::try_compile_connector_create_table; -use super::ast_utils::AstUtils; -use super::ddl_compiler::DdlCompiler; use super::streaming_compiler::StreamingCompiler; #[derive(Clone)] @@ -135,15 +133,12 @@ impl StatementVisitor for LogicalPlanVisitor { stmt: &CreateTable, _ctx: &StatementVisitorContext, ) -> StatementVisitorResult { - if let Statement::CreateTable(ast_node) = &stmt.statement - && ast_node.query.is_none() - && AstUtils::contains_connector_property(&ast_node.with_options) + if let Some(result) = + try_compile_connector_create_table(&self.schema_provider, &stmt.statement) { - let declared_role = AstUtils::peek_table_role(&ast_node.with_options); - let compiler = DdlCompiler::new(&self.schema_provider); - return match compiler.compile(ast_node, declared_role.as_deref()) { - Ok(external_table) => StatementVisitorResult::Plan(Box::new( - CreateTablePlan::external_table(external_table, ast_node.if_not_exists), + return match result { + Ok((external_table, if_not_exists)) => StatementVisitorResult::Plan(Box::new( + CreateTablePlan::external_table(external_table, if_not_exists), )), Err(err) => StatementVisitorResult::Plan(Box::new(CompileErrorPlan::new(format!( "Ingest table resolution failed - {err:#}" diff --git a/src/coordinator/plan/lookup_table_plan.rs b/src/coordinator/src/plan/lookup_table_plan.rs similarity index 100% rename from src/coordinator/plan/lookup_table_plan.rs rename to src/coordinator/src/plan/lookup_table_plan.rs diff --git a/src/coordinator/plan/mod.rs b/src/coordinator/src/plan/mod.rs similarity index 98% rename from src/coordinator/plan/mod.rs rename to src/coordinator/src/plan/mod.rs index 95978467..7f4b279a 100644 --- a/src/coordinator/plan/mod.rs +++ b/src/coordinator/src/plan/mod.rs @@ -10,12 +10,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -mod ast_utils; mod compile_error_plan; mod create_function_plan; mod create_python_function_plan; mod create_table_plan; -mod ddl_compiler; mod drop_function_plan; mod drop_streaming_table_plan; mod drop_table_plan; diff --git a/src/coordinator/plan/optimizer.rs b/src/coordinator/src/plan/optimizer.rs similarity index 100% rename from src/coordinator/plan/optimizer.rs rename to src/coordinator/src/plan/optimizer.rs diff --git a/src/coordinator/plan/show_catalog_tables_plan.rs b/src/coordinator/src/plan/show_catalog_tables_plan.rs similarity index 100% rename from src/coordinator/plan/show_catalog_tables_plan.rs rename to src/coordinator/src/plan/show_catalog_tables_plan.rs diff --git a/src/coordinator/plan/show_create_streaming_table_plan.rs b/src/coordinator/src/plan/show_create_streaming_table_plan.rs similarity index 100% rename from src/coordinator/plan/show_create_streaming_table_plan.rs rename to src/coordinator/src/plan/show_create_streaming_table_plan.rs diff --git a/src/coordinator/plan/show_create_table_plan.rs b/src/coordinator/src/plan/show_create_table_plan.rs similarity index 100% rename from src/coordinator/plan/show_create_table_plan.rs rename to src/coordinator/src/plan/show_create_table_plan.rs diff --git a/src/coordinator/plan/show_functions_plan.rs b/src/coordinator/src/plan/show_functions_plan.rs similarity index 100% rename from src/coordinator/plan/show_functions_plan.rs rename to src/coordinator/src/plan/show_functions_plan.rs diff --git a/src/coordinator/plan/show_streaming_tables_plan.rs b/src/coordinator/src/plan/show_streaming_tables_plan.rs similarity index 100% rename from src/coordinator/plan/show_streaming_tables_plan.rs rename to src/coordinator/src/plan/show_streaming_tables_plan.rs diff --git a/src/coordinator/plan/start_function_plan.rs b/src/coordinator/src/plan/start_function_plan.rs similarity index 100% rename from src/coordinator/plan/start_function_plan.rs rename to src/coordinator/src/plan/start_function_plan.rs diff --git a/src/coordinator/plan/stop_function_plan.rs b/src/coordinator/src/plan/stop_function_plan.rs similarity index 100% rename from src/coordinator/plan/stop_function_plan.rs rename to src/coordinator/src/plan/stop_function_plan.rs diff --git a/src/coordinator/plan/streaming_compiler.rs b/src/coordinator/src/plan/streaming_compiler.rs similarity index 100% rename from src/coordinator/plan/streaming_compiler.rs rename to src/coordinator/src/plan/streaming_compiler.rs diff --git a/src/coordinator/plan/streaming_table_connector_plan.rs b/src/coordinator/src/plan/streaming_table_connector_plan.rs similarity index 100% rename from src/coordinator/plan/streaming_table_connector_plan.rs rename to src/coordinator/src/plan/streaming_table_connector_plan.rs diff --git a/src/coordinator/plan/streaming_table_plan.rs b/src/coordinator/src/plan/streaming_table_plan.rs similarity index 100% rename from src/coordinator/plan/streaming_table_plan.rs rename to src/coordinator/src/plan/streaming_table_plan.rs diff --git a/src/coordinator/plan/visitor.rs b/src/coordinator/src/plan/visitor.rs similarity index 100% rename from src/coordinator/plan/visitor.rs rename to src/coordinator/src/plan/visitor.rs diff --git a/src/coordinator/runtime_context.rs b/src/coordinator/src/runtime_context.rs similarity index 93% rename from src/coordinator/runtime_context.rs rename to src/coordinator/src/runtime_context.rs index 21b9d876..2f8dc2a7 100644 --- a/src/coordinator/runtime_context.rs +++ b/src/coordinator/src/runtime_context.rs @@ -16,10 +16,10 @@ use std::sync::Arc; use anyhow::Result; -use crate::runtime::streaming::job::JobManager; -use crate::runtime::wasm::taskexecutor::TaskManager; use crate::sql::schema::StreamSchemaProvider; -use crate::storage::stream_catalog::CatalogManager; +use crate::stream_catalog::CatalogManager; +use crate::streaming::job::JobManager; +use crate::wasm::taskexecutor::TaskManager; /// Dependencies shared by analyze / plan / execute, analogous to installing globals in /// [`TaskManager`], [`CatalogManager`], and [`JobManager`]. diff --git a/src/sql/parse.rs b/src/coordinator/src/sql_classify.rs similarity index 51% rename from src/sql/parse.rs rename to src/coordinator/src/sql_classify.rs index 0c6b9541..0c1417d4 100644 --- a/src/sql/parse.rs +++ b/src/coordinator/src/sql_classify.rs @@ -10,22 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Coordinator-facing SQL parsing (`parse_sql`). -//! -//! **Data-definition / pipeline shape (this entry point)** -//! Only these table-related forms are supported: -//! - **`CREATE TABLE ... (cols [, WATERMARK FOR ...]) WITH ('connector' = '...', 'format' = '...', ...)`** -//! connector-backed **source** DDL (no `AS SELECT`; `connector` in `WITH` selects this path) -//! - **`CREATE TABLE ...`** other forms (including `CREATE TABLE ... AS SELECT` where DataFusion accepts it) -//! - **`CREATE STREAMING TABLE ... WITH (...) AS SELECT ...`** (streaming sink DDL) -//! - **`DROP TABLE`** / **`DROP TABLE IF EXISTS`** / **`DROP STREAMING TABLE`** (alias for `DROP TABLE` on the stream catalog) -//! - **`SHOW TABLES`** — list stream catalog tables (connector sources and streaming sinks) -//! - **`SHOW CREATE TABLE `** — best-effort DDL text (full `WITH` / `AS SELECT` may not be stored) -//! -//! **`INSERT` is not supported** here — use `CREATE TABLE ... AS SELECT` or -//! `CREATE STREAMING TABLE ... AS SELECT` to define the query shape instead. -//! -//! Other supported statements include function lifecycle (`CREATE FUNCTION WITH`, `START FUNCTION`, …). +//! Map sqlparser [`Statement`](datafusion::sql::sqlparser::ast::Statement) values into +//! coordinator [`Statement`](super::statement::Statement) trait objects. use std::collections::HashMap; @@ -34,93 +20,15 @@ use datafusion::error::DataFusionError; use datafusion::sql::sqlparser::ast::{ ObjectType, ShowCreateObject, SqlOption, Statement as DFStatement, }; -use datafusion::sql::sqlparser::dialect::FunctionStreamDialect; -use datafusion::sql::sqlparser::parser::Parser; -use crate::coordinator::{ +use super::{ CreateFunction, CreateTable, DropFunction, DropStreamingTableStatement, DropTableStatement, ShowCatalogTables, ShowCreateStreamingTable, ShowCreateTable, ShowFunctions, - ShowStreamingTables, StartFunction, Statement as CoordinatorStatement, StopFunction, - StreamingTableStatement, + ShowStreamingTables, StartFunction, Statement, StopFunction, StreamingTableStatement, }; -/// Streaming-specific SQL that the sqlparser dialect does not natively handle. -/// -/// Returns `Some(statement)` if the SQL was intercepted, `None` otherwise so -/// the caller falls through to the normal sqlparser pipeline. -fn try_parse_streaming_statement(sql: &str) -> Option> { - let tokens: Vec<&str> = sql.split_whitespace().collect(); - if tokens.is_empty() { - return None; - } - - // SHOW STREAMING TABLES - if tokens.len() == 3 - && tokens[0].eq_ignore_ascii_case("show") - && tokens[1].eq_ignore_ascii_case("streaming") - && tokens[2].eq_ignore_ascii_case("tables") - { - return Some(Box::new(ShowStreamingTables::new())); - } - - // SHOW CREATE STREAMING TABLE - if tokens.len() == 5 - && tokens[0].eq_ignore_ascii_case("show") - && tokens[1].eq_ignore_ascii_case("create") - && tokens[2].eq_ignore_ascii_case("streaming") - && tokens[3].eq_ignore_ascii_case("table") - { - let name = tokens[4].trim_end_matches(';').to_string(); - return Some(Box::new(ShowCreateStreamingTable::new(name))); - } - - // DROP STREAMING TABLE [IF EXISTS] - if tokens.len() >= 4 - && tokens[0].eq_ignore_ascii_case("drop") - && tokens[1].eq_ignore_ascii_case("streaming") - && tokens[2].eq_ignore_ascii_case("table") - { - let (if_exists, name_idx) = if tokens.len() >= 6 - && tokens[3].eq_ignore_ascii_case("if") - && tokens[4].eq_ignore_ascii_case("exists") - { - (true, 5) - } else { - (false, 3) - }; - - if name_idx >= tokens.len() { - return None; - } - let name = tokens[name_idx].trim_end_matches(';').to_string(); - return Some(Box::new(DropStreamingTableStatement::new(name, if_exists))); - } - - None -} - -pub fn parse_sql(query: &str) -> Result>> { - let trimmed = query.trim(); - if trimmed.is_empty() { - return plan_err!("Query is empty"); - } - - if let Some(stmt) = try_parse_streaming_statement(trimmed) { - return Ok(vec![stmt]); - } - - let dialect = FunctionStreamDialect {}; - let statements = Parser::parse_sql(&dialect, trimmed) - .map_err(|e| DataFusionError::Plan(format!("SQL parse error: {e}")))?; - - if statements.is_empty() { - return plan_err!("No SQL statements found"); - } - - statements.into_iter().map(classify_statement).collect() -} - -fn classify_statement(stmt: DFStatement) -> Result> { +/// Convert [`DFStatement`] from the FunctionStream SQL dialect into a coordinator statement. +pub fn classify_statement(stmt: DFStatement) -> Result> { match stmt { DFStatement::CreateFunctionWith { options } => { let properties = sql_options_to_map(&options); @@ -139,34 +47,54 @@ fn classify_statement(stmt: DFStatement) -> Result } DFStatement::ShowFunctions { .. } => Ok(Box::new(ShowFunctions::new())), DFStatement::ShowTables { .. } => Ok(Box::new(ShowCatalogTables::new())), - DFStatement::ShowCreate { obj_type, obj_name } => { - if obj_type != ShowCreateObject::Table { - return plan_err!( - "SHOW CREATE {obj_type} is not supported; use SHOW CREATE TABLE " - ); - } - Ok(Box::new(ShowCreateTable::new(obj_name.to_string()))) - } + DFStatement::ShowStreamingTable => Ok(Box::new(ShowStreamingTables::new())), + DFStatement::ShowCreate { obj_type, obj_name } => match obj_type { + ShowCreateObject::Table => Ok(Box::new(ShowCreateTable::new(obj_name.to_string()))), + ShowCreateObject::StreamingTable => Ok(Box::new(ShowCreateStreamingTable::new( + obj_name.to_string(), + ))), + _ => plan_err!( + "SHOW CREATE {obj_type} is not supported; use SHOW CREATE TABLE or SHOW CREATE STREAMING TABLE " + ), + }, s @ DFStatement::CreateTable(_) => Ok(Box::new(CreateTable::new(s))), s @ DFStatement::CreateStreamingTable { .. } => { Ok(Box::new(StreamingTableStatement::new(s))) } stmt @ DFStatement::Drop { .. } => { - { - let DFStatement::Drop { - object_type, names, .. - } = &stmt - else { - unreachable!() - }; - if *object_type != ObjectType::Table { - return plan_err!("Only DROP TABLE is supported in this SQL frontend"); + let DFStatement::Drop { + object_type, + names, + if_exists, + .. + } = &stmt + else { + unreachable!() + }; + match object_type { + ObjectType::Table => { + if names.len() != 1 { + return plan_err!( + "DROP TABLE supports exactly one table name per statement" + ); + } + Ok(Box::new(DropTableStatement::new(stmt))) } - if names.len() != 1 { - return plan_err!("DROP TABLE supports exactly one table name per statement"); + ObjectType::StreamingTable => { + if names.len() != 1 { + return plan_err!( + "DROP STREAMING TABLE supports exactly one table name per statement" + ); + } + let table_name = names[0].to_string(); + Ok(Box::new(DropStreamingTableStatement::new( + table_name, *if_exists, + ))) } + _ => plan_err!( + "Only DROP TABLE and DROP STREAMING TABLE are supported in this SQL frontend" + ), } - Ok(Box::new(DropTableStatement::new(stmt))) } DFStatement::Insert { .. } => plan_err!( "INSERT is not supported; only CREATE TABLE and CREATE STREAMING TABLE (with AS SELECT) \ @@ -176,7 +104,6 @@ fn classify_statement(stmt: DFStatement) -> Result } } -/// Convert Vec (KeyValue pairs) into HashMap. fn sql_options_to_map(options: &[SqlOption]) -> HashMap { options .iter() @@ -193,71 +120,72 @@ fn sql_options_to_map(options: &[SqlOption]) -> HashMap { #[cfg(test)] mod tests { use super::*; + use crate::sql::parse::parse_sql; - fn first_stmt(sql: &str) -> Box { + fn first_classified(sql: &str) -> Box { let mut stmts = parse_sql(sql).unwrap(); assert!(!stmts.is_empty()); - stmts.remove(0) + classify_statement(stmts.remove(0)).unwrap() } - fn is_type(stmt: &dyn CoordinatorStatement, prefix: &str) -> bool { - format!("{:?}", stmt).starts_with(prefix) + fn is_type(stmt: &dyn Statement, prefix: &str) -> bool { + format!("{stmt:?}").starts_with(prefix) } #[test] fn test_parse_create_function() { let sql = "CREATE FUNCTION WITH ('function_path'='./test.wasm', 'config_path'='./config.yml')"; - let stmt = first_stmt(sql); + let stmt = first_classified(sql); assert!(is_type(stmt.as_ref(), "CreateFunction")); } #[test] fn test_parse_create_function_minimal() { let sql = "CREATE FUNCTION WITH ('function_path'='./processor.wasm')"; - let stmt = first_stmt(sql); + let stmt = first_classified(sql); assert!(is_type(stmt.as_ref(), "CreateFunction")); } #[test] fn test_parse_drop_function() { - let stmt = first_stmt("DROP FUNCTION my_task"); + let stmt = first_classified("DROP FUNCTION my_task"); assert!(is_type(stmt.as_ref(), "DropFunction")); } #[test] fn test_parse_start_function() { - let stmt = first_stmt("START FUNCTION my_task"); + let stmt = first_classified("START FUNCTION my_task"); assert!(is_type(stmt.as_ref(), "StartFunction")); } #[test] fn test_parse_stop_function() { - let stmt = first_stmt("STOP FUNCTION my_task"); + let stmt = first_classified("STOP FUNCTION my_task"); assert!(is_type(stmt.as_ref(), "StopFunction")); } #[test] fn test_parse_show_functions() { - let stmt = first_stmt("SHOW FUNCTIONS"); + let stmt = first_classified("SHOW FUNCTIONS"); assert!(is_type(stmt.as_ref(), "ShowFunctions")); } #[test] fn test_parse_show_tables() { - let stmt = first_stmt("SHOW TABLES"); + let stmt = first_classified("SHOW TABLES"); assert!(is_type(stmt.as_ref(), "ShowCatalogTables")); } #[test] fn test_parse_show_create_table() { - let stmt = first_stmt("SHOW CREATE TABLE my_src"); + let stmt = first_classified("SHOW CREATE TABLE my_src"); assert!(is_type(stmt.as_ref(), "ShowCreateTable")); } #[test] fn test_parse_create_table() { - let stmt = first_stmt("CREATE TABLE foo (id INT, name VARCHAR)"); + let stmt = first_classified("CREATE TABLE foo (id INT, name VARCHAR)"); assert!(is_type(stmt.as_ref(), "CreateTable")); } @@ -267,47 +195,46 @@ mod tests { "CREATE TABLE kafka_src (id BIGINT, ts TIMESTAMP NOT NULL, WATERMARK FOR ts) ", "WITH ('connector' = 'kafka', 'format' = 'json', 'topic' = 'events')", ); - let stmt = first_stmt(sql); + let stmt = first_classified(sql); assert!(is_type(stmt.as_ref(), "CreateTable")); } #[test] fn test_parse_drop_table() { - let stmt = first_stmt("DROP TABLE foo"); + let stmt = first_classified("DROP TABLE foo"); assert!(is_type(stmt.as_ref(), "DropTableStatement")); } #[test] fn test_parse_drop_table_if_exists() { - let stmt = first_stmt("DROP TABLE IF EXISTS foo"); + let stmt = first_classified("DROP TABLE IF EXISTS foo"); assert!(is_type(stmt.as_ref(), "DropTableStatement")); } #[test] fn test_parse_drop_streaming_table() { - let stmt = first_stmt("DROP STREAMING TABLE my_sink"); + let stmt = first_classified("DROP STREAMING TABLE my_sink"); assert!(is_type(stmt.as_ref(), "DropStreamingTableStatement")); } #[test] fn test_parse_drop_streaming_table_if_exists() { - let stmt = first_stmt("DROP STREAMING TABLE IF EXISTS my_sink"); + let stmt = first_classified("DROP STREAMING TABLE IF EXISTS my_sink"); assert!(is_type(stmt.as_ref(), "DropStreamingTableStatement")); } #[test] fn test_parse_show_streaming_tables() { - let stmt = first_stmt("SHOW STREAMING TABLES"); + let stmt = first_classified("SHOW STREAMING TABLES"); assert!(is_type(stmt.as_ref(), "ShowStreamingTables")); } #[test] fn test_parse_show_create_streaming_table() { - let stmt = first_stmt("SHOW CREATE STREAMING TABLE my_sink"); + let stmt = first_classified("SHOW CREATE STREAMING TABLE my_sink"); assert!(is_type(stmt.as_ref(), "ShowCreateStreamingTable")); } - /// `CREATE STREAMING TABLE` is the sink DDL supported by FunctionStream (not `CREATE STREAM TABLE`). #[test] fn test_parse_create_streaming_table() { let sql = concat!( @@ -315,7 +242,7 @@ mod tests { "WITH ('connector' = 'kafka') ", "AS SELECT id FROM src", ); - let stmt = first_stmt(sql); + let stmt = first_classified(sql); assert!( is_type(stmt.as_ref(), "StreamingTableStatement"), "expected StreamingTableStatement, got {:?}", @@ -330,22 +257,22 @@ mod tests { "with ('connector' = 'memory') ", "as select 1 as x", ); - let stmt = first_stmt(sql); + let stmt = first_classified(sql); assert!(is_type(stmt.as_ref(), "StreamingTableStatement")); } #[test] fn test_parse_case_insensitive() { assert!(is_type( - first_stmt("create function with ('function_path'='./test.wasm')").as_ref(), + first_classified("create function with ('function_path'='./test.wasm')").as_ref(), "CreateFunction" )); assert!(is_type( - first_stmt("show functions").as_ref(), + first_classified("show functions").as_ref(), "ShowFunctions" )); assert!(is_type( - first_stmt("start function my_task").as_ref(), + first_classified("start function my_task").as_ref(), "StartFunction" )); } @@ -356,27 +283,24 @@ mod tests { "CREATE TABLE t1 (id INT); ", "CREATE STREAMING TABLE sk WITH ('connector' = 'kafka') AS SELECT id FROM t1", ); - let stmts = parse_sql(sql).unwrap(); - assert_eq!(stmts.len(), 2); - assert!(is_type(stmts[0].as_ref(), "CreateTable")); - assert!(is_type(stmts[1].as_ref(), "StreamingTableStatement")); - } - - #[test] - fn test_parse_empty() { - assert!(parse_sql("").is_err()); - assert!(parse_sql(" ").is_err()); + let mut ast = parse_sql(sql).unwrap(); + assert_eq!(ast.len(), 2); + let s0 = classify_statement(ast.remove(0)).unwrap(); + let s1 = classify_statement(ast.remove(0)).unwrap(); + assert!(is_type(s0.as_ref(), "CreateTable")); + assert!(is_type(s1.as_ref(), "StreamingTableStatement")); } #[test] - fn test_parse_unsupported_statement() { - let result = parse_sql("SELECT 1"); - assert!(result.is_err()); + fn test_classify_unsupported_statement() { + let mut stmts = parse_sql("SELECT 1").unwrap(); + assert!(classify_statement(stmts.remove(0)).is_err()); } #[test] fn test_insert_not_supported() { - let err = parse_sql("INSERT INTO sink SELECT * FROM src").unwrap_err(); + let mut stmts = parse_sql("INSERT INTO sink SELECT * FROM src").unwrap(); + let err = classify_statement(stmts.remove(0)).unwrap_err(); let msg = err.to_string(); assert!( msg.contains("INSERT") && msg.contains("not supported"), @@ -396,7 +320,7 @@ mod tests { 'parallelism'='4', 'memory-limit'='256mb' )"#; - let stmt = first_stmt(sql); + let stmt = first_classified(sql); assert!(is_type(stmt.as_ref(), "CreateFunction")); } } diff --git a/src/coordinator/statement/create_function.rs b/src/coordinator/src/statement/create_function.rs similarity index 100% rename from src/coordinator/statement/create_function.rs rename to src/coordinator/src/statement/create_function.rs diff --git a/src/coordinator/statement/create_python_function.rs b/src/coordinator/src/statement/create_python_function.rs similarity index 100% rename from src/coordinator/statement/create_python_function.rs rename to src/coordinator/src/statement/create_python_function.rs diff --git a/src/coordinator/statement/create_table.rs b/src/coordinator/src/statement/create_table.rs similarity index 100% rename from src/coordinator/statement/create_table.rs rename to src/coordinator/src/statement/create_table.rs diff --git a/src/coordinator/statement/drop_function.rs b/src/coordinator/src/statement/drop_function.rs similarity index 100% rename from src/coordinator/statement/drop_function.rs rename to src/coordinator/src/statement/drop_function.rs diff --git a/src/coordinator/statement/drop_streaming_table.rs b/src/coordinator/src/statement/drop_streaming_table.rs similarity index 100% rename from src/coordinator/statement/drop_streaming_table.rs rename to src/coordinator/src/statement/drop_streaming_table.rs diff --git a/src/coordinator/statement/drop_table.rs b/src/coordinator/src/statement/drop_table.rs similarity index 100% rename from src/coordinator/statement/drop_table.rs rename to src/coordinator/src/statement/drop_table.rs diff --git a/src/coordinator/statement/mod.rs b/src/coordinator/src/statement/mod.rs similarity index 100% rename from src/coordinator/statement/mod.rs rename to src/coordinator/src/statement/mod.rs diff --git a/src/coordinator/statement/show_catalog_tables.rs b/src/coordinator/src/statement/show_catalog_tables.rs similarity index 100% rename from src/coordinator/statement/show_catalog_tables.rs rename to src/coordinator/src/statement/show_catalog_tables.rs diff --git a/src/coordinator/statement/show_create_streaming_table.rs b/src/coordinator/src/statement/show_create_streaming_table.rs similarity index 100% rename from src/coordinator/statement/show_create_streaming_table.rs rename to src/coordinator/src/statement/show_create_streaming_table.rs diff --git a/src/coordinator/statement/show_create_table.rs b/src/coordinator/src/statement/show_create_table.rs similarity index 100% rename from src/coordinator/statement/show_create_table.rs rename to src/coordinator/src/statement/show_create_table.rs diff --git a/src/coordinator/statement/show_functions.rs b/src/coordinator/src/statement/show_functions.rs similarity index 100% rename from src/coordinator/statement/show_functions.rs rename to src/coordinator/src/statement/show_functions.rs diff --git a/src/coordinator/statement/show_streaming_tables.rs b/src/coordinator/src/statement/show_streaming_tables.rs similarity index 100% rename from src/coordinator/statement/show_streaming_tables.rs rename to src/coordinator/src/statement/show_streaming_tables.rs diff --git a/src/coordinator/statement/start_function.rs b/src/coordinator/src/statement/start_function.rs similarity index 100% rename from src/coordinator/statement/start_function.rs rename to src/coordinator/src/statement/start_function.rs diff --git a/src/coordinator/statement/stop_function.rs b/src/coordinator/src/statement/stop_function.rs similarity index 100% rename from src/coordinator/statement/stop_function.rs rename to src/coordinator/src/statement/stop_function.rs diff --git a/src/coordinator/statement/streaming_table.rs b/src/coordinator/src/statement/streaming_table.rs similarity index 100% rename from src/coordinator/statement/streaming_table.rs rename to src/coordinator/src/statement/streaming_table.rs diff --git a/src/coordinator/statement/visitor.rs b/src/coordinator/src/statement/visitor.rs similarity index 100% rename from src/coordinator/statement/visitor.rs rename to src/coordinator/src/statement/visitor.rs diff --git a/src/coordinator/streaming_table_options.rs b/src/coordinator/src/streaming_table_options.rs similarity index 100% rename from src/coordinator/streaming_table_options.rs rename to src/coordinator/src/streaming_table_options.rs diff --git a/src/coordinator/tool/mod.rs b/src/coordinator/src/tool/mod.rs similarity index 100% rename from src/coordinator/tool/mod.rs rename to src/coordinator/src/tool/mod.rs diff --git a/src/function-stream/Cargo.toml b/src/function-stream/Cargo.toml new file mode 100644 index 00000000..184686f7 --- /dev/null +++ b/src/function-stream/Cargo.toml @@ -0,0 +1,87 @@ +[package] +name = "function-stream" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream" +path = "src/lib.rs" + +[[bin]] +name = "function-stream" +path = "src/main.rs" + +[dependencies] +tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "sync", "time", "net", "signal"] } +serde = { version = "1.0", features = ["derive"] } +serde_yaml = "0.9" +serde_json = "1.0" +uuid = { version = "1.0", features = ["v4", "v7"] } +log = "0.4" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing-appender = "0.2" +anyhow = "1.0" +thiserror = "2" +tonic = { version = "0.12", features = ["default"] } +async-trait = "0.1" +num_cpus = "1.0" +protocol = { path = "../../protocol" } +function-stream-config = { path = "../config" } +function-stream-logger = { path = "../logger" } +function-stream-runtime-common = { path = "../runtime_common" } +function-stream-streaming-runtime = { path = "../streaming_runtime" } +function-stream-streaming-planner = { path = "../streaming_planner" } +prost = "0.13" +rdkafka = { version = "0.38", features = ["cmake-build", "ssl", "gssapi", "curl"] } +crossbeam-channel = "0.5" +wasmtime = { version = "41.0.3", features = ["component-model", "async"] } +base64 = "0.22" +wasmtime-wasi = "41.0.3" +rocksdb = { version = "0.21", features = ["multi-threaded-cf", "lz4"] } +bincode = { version = "2", features = ["serde"] } +chrono = "0.4" +tokio-stream = "0.1.18" +lru = "0.12" +parking_lot = "0.12" +arrow = { version = "55", default-features = false } +arrow-array = "55" +arrow-ipc = "55" +arrow-schema = { version = "55", features = ["serde"] } +parquet = "55" +object_store = { version = "0.12.5", features = ["aws"] } +bytes = "1" +futures = "0.3" +serde_json_path = "0.7" +xxhash-rust = { version = "0.8", features = ["xxh3"] } +proctitle = "0.1" +unicase = "2.7" +petgraph = "0.7" +rand = { version = "0.8", features = ["small_rng"] } +itertools = "0.14" +strum = { version = "0.26", features = ["derive"] } + +arrow-json = { version = "55.2.0" } +apache-avro = "0.21" +datafusion = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-common = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-execution = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-expr = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-physical-expr = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-proto = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } + +sqlparser = { git = "https://github.com/FunctionStream/sqlparser-rs", branch = "0.58.0/fs" } + +ahash = "0.8" +governor = "0.8.0" +lance = { version = "4.0.0", default-features = false, features = ["aws"] } +arrow-array-lance = { package = "arrow-array", version = "57.3.0" } +arrow-ipc-lance = { package = "arrow-ipc", version = "57.3.0" } + +[features] +default = ["incremental-cache", "python"] +incremental-cache = ["wasmtime/incremental-cache"] +python = [] + +[dev-dependencies] +tempfile = "3.27.0" diff --git a/src/storage/mod.rs b/src/function-stream/src/lib.rs similarity index 71% rename from src/storage/mod.rs rename to src/function-stream/src/lib.rs index ec32bdfa..1731c4cc 100644 --- a/src/storage/mod.rs +++ b/src/function-stream/src/lib.rs @@ -10,12 +10,35 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Library crate for function-stream + +#![allow(dead_code)] + use std::sync::Arc; use anyhow::Context; +pub use function_stream_config as config; +#[path = "../../coordinator/src/coordinator_body.rs"] +pub mod coordinator; +pub use function_stream_logger as logging; + +pub use function_stream_runtime_common::{common, memory}; +pub use function_stream_streaming_runtime as streaming; +pub use function_stream_streaming_runtime::util; + +#[path = "../../wasm_runtime/src/wasm/mod.rs"] +pub mod wasm; + +pub use wasm::{input, output, processor}; + +#[path = "../../wasm_runtime/src/state_backend/mod.rs"] pub mod state_backend; + +#[path = "../../catalog_storage/src/stream_catalog/mod.rs"] pub mod stream_catalog; + +#[path = "../../catalog_storage/src/task/mod.rs"] pub mod task; /// Install the process-global [`stream_catalog::CatalogManager`] from configuration. @@ -51,3 +74,7 @@ pub fn initialize_stream_catalog(config: &crate::config::GlobalConfig) -> anyhow CatalogManager::init_global(store).context("Stream catalog (CatalogManager) global init failed") } + +#[path = "../../servicer/src/servicer_body.rs"] +pub mod server; +pub use function_stream_streaming_planner as sql; diff --git a/src/main.rs b/src/function-stream/src/main.rs similarity index 77% rename from src/main.rs rename to src/function-stream/src/main.rs index 46da3c7a..665be071 100644 --- a/src/main.rs +++ b/src/function-stream/src/main.rs @@ -12,15 +12,67 @@ #![allow(dead_code)] -mod config; +pub use function_stream_config as config; +#[path = "../../coordinator/src/coordinator_body.rs"] mod coordinator; -mod logging; -mod runtime; -mod server; -mod sql; -mod storage; +pub use function_stream_logger as logging; + +pub use function_stream_runtime_common::{common, memory}; +pub use function_stream_streaming_runtime as streaming; +pub use function_stream_streaming_runtime::util; + +use std::sync::Arc; use anyhow::{Context, Result}; + +#[path = "../../wasm_runtime/src/wasm/mod.rs"] +mod wasm; + +pub use wasm::{input, output, processor}; + +#[path = "../../wasm_runtime/src/state_backend/mod.rs"] +mod state_backend; + +#[path = "../../catalog_storage/src/stream_catalog/mod.rs"] +mod stream_catalog; + +#[path = "../../catalog_storage/src/task/mod.rs"] +mod task; + +pub fn initialize_stream_catalog(config: &crate::config::GlobalConfig) -> anyhow::Result<()> { + use stream_catalog::{CatalogManager, InMemoryMetaStore, MetaStore, RocksDbMetaStore}; + + let store: Arc = if !config.stream_catalog.persist { + Arc::new(InMemoryMetaStore::new()) + } else { + let path = config + .stream_catalog + .db_path + .as_ref() + .map(|p| crate::config::resolve_path(p)) + .unwrap_or_else(|| crate::config::get_data_dir().join("catalog.db")); + + std::fs::create_dir_all(&path).with_context(|| { + format!( + "Failed to create stream catalog RocksDB directory {}", + path.display() + ) + })?; + + Arc::new(RocksDbMetaStore::open(&path).with_context(|| { + format!( + "Failed to open stream catalog RocksDB at {}", + path.display() + ) + })?) + }; + + CatalogManager::init_global(store).context("Stream catalog (CatalogManager) global init failed") +} + +#[path = "../../servicer/src/servicer_body.rs"] +mod server; +pub use function_stream_streaming_planner as sql; use std::thread; use tokio::sync::oneshot; @@ -144,7 +196,8 @@ fn setup_environment() -> Result { config::GlobalConfig::default() }; - logging::init_logging(&config.logging).context("Logging initialization failed")?; + function_stream_logger::init_logging(&config.logging) + .context("Logging initialization failed")?; log::debug!( "Environment initialized. Data: {}, Conf: {}", diff --git a/src/logger/Cargo.toml b/src/logger/Cargo.toml new file mode 100644 index 00000000..273579a8 --- /dev/null +++ b/src/logger/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "function-stream-logger" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_logger" +path = "src/lib.rs" + +[dependencies] +anyhow = "1.0" +function-stream-config = { path = "../config" } +tracing = "0.1" +tracing-appender = "0.2" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/src/logger/src/lib.rs b/src/logger/src/lib.rs new file mode 100644 index 00000000..53fb0b31 --- /dev/null +++ b/src/logger/src/lib.rs @@ -0,0 +1,9 @@ +//! Logging setup and helpers. + +pub mod config { + pub use function_stream_config::*; +} + +mod logging; + +pub use logging::init_logging; diff --git a/src/logging/mod.rs b/src/logger/src/logging.rs similarity index 100% rename from src/logging/mod.rs rename to src/logger/src/logging.rs diff --git a/src/runtime/streaming/operators/sink/delta/mod.rs b/src/runtime/streaming/operators/sink/delta/mod.rs deleted file mode 100644 index 4df6b3b5..00000000 --- a/src/runtime/streaming/operators/sink/delta/mod.rs +++ /dev/null @@ -1,273 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::HashMap; -use std::fs::create_dir_all; -use std::path::PathBuf; -use std::sync::Arc; - -use anyhow::{Context, Result}; -use arrow_array::RecordBatch; -use async_trait::async_trait; -use bytes::Bytes; -use object_store::aws::AmazonS3Builder; -use object_store::path::Path as ObjectStorePath; -use object_store::{ObjectStore, PutPayload}; -use parquet::basic::Compression; -use tokio::io::AsyncWriteExt; -use tracing::{debug, info, warn}; - -use crate::runtime::memory::{MemoryBlock, try_global_memory_pool}; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::format::encoder::FormatEncoder; -use crate::sql::common::constants::factory_operator_name; -use crate::sql::common::with_option_keys as opt; -use crate::sql::common::{CheckpointBarrier, Watermark}; - -/// Flush early when buffered batches exceed this size. -const DEFAULT_MAX_BUFFER_BYTES: usize = 64 * 1024 * 1024; - -enum DeltaDestination { - Local(PathBuf), - S3 { - prefix: String, - client: Arc, - }, -} - -pub struct DeltaSinkOperator { - table_name: String, - destination: DeltaDestination, - parquet_compression: Compression, - pending: Vec, - pending_bytes: usize, - sink_memory_block: Option>, - early_flush_threshold_bytes: usize, - file_counter: u64, - format: DeltaFormat, -} - -#[derive(Debug, Clone, Copy)] -pub enum DeltaFormat { - Csv, - Parquet, - JsonL, - Avro, - Orc, -} - -impl DeltaSinkOperator { - pub fn try_new( - table_name: String, - path: String, - format: DeltaFormat, - parquet_compression: Compression, - sink_memory_bytes: u64, - options: HashMap, - ) -> Result { - let destination = if let Some(bucket) = options.get(opt::S3_BUCKET) { - let region = options - .get(opt::S3_REGION) - .cloned() - .unwrap_or_else(|| "us-east-1".to_string()); - let mut builder = AmazonS3Builder::new() - .with_bucket_name(bucket.clone()) - .with_region(region); - if let Some(endpoint) = options.get(opt::S3_ENDPOINT) { - builder = builder.with_endpoint(endpoint); - if endpoint.to_ascii_lowercase().starts_with("http://") { - builder = builder.with_allow_http(true); - } - } - if let Some(v) = options.get(opt::S3_ACCESS_KEY_ID) { - builder = builder.with_access_key_id(v); - } - if let Some(v) = options.get(opt::S3_SECRET_ACCESS_KEY) { - builder = builder.with_secret_access_key(v); - } - if let Some(v) = options.get(opt::S3_SESSION_TOKEN) { - builder = builder.with_token(v); - } - let client = builder - .build() - .context("failed to build s3 client for delta sink")?; - DeltaDestination::S3 { - prefix: path.trim_matches('/').to_string(), - client: Arc::new(client), - } - } else { - let root = PathBuf::from(path.clone()); - create_dir_all(&root) - .with_context(|| format!("failed to create delta sink dir {}", root.display()))?; - DeltaDestination::Local(root) - }; - - let mut sink_memory_block = None; - let reserve_bytes = usize::try_from(sink_memory_bytes).unwrap_or(DEFAULT_MAX_BUFFER_BYTES); - let mut early_flush_threshold_bytes = reserve_bytes; - if let Ok(pool) = try_global_memory_pool() - && let Ok(block) = pool.try_request_block(reserve_bytes as u64) - { - early_flush_threshold_bytes = ((block.capacity() as usize) * 8) / 10; - sink_memory_block = Some(block); - } - - Ok(Self { - table_name, - destination, - parquet_compression, - pending: Vec::new(), - pending_bytes: 0, - sink_memory_block, - early_flush_threshold_bytes, - file_counter: 0, - format, - }) - } - - async fn flush_epoch(&mut self, epoch: u64, subtask_idx: usize) -> Result<()> { - if self.pending.is_empty() { - return Ok(()); - } - - let batches = std::mem::take(&mut self.pending); - let format = self.format; - let compression = self.parquet_compression; - let bytes = tokio::task::spawn_blocking(move || -> Result> { - match format { - DeltaFormat::Csv => FormatEncoder::encode_csv(&batches), - DeltaFormat::Parquet => FormatEncoder::encode_parquet(&batches, compression), - DeltaFormat::JsonL => FormatEncoder::encode_jsonl(&batches), - DeltaFormat::Avro => FormatEncoder::encode_avro(&batches), - DeltaFormat::Orc => FormatEncoder::encode_orc(&batches), - } - }) - .await - .context("tokio blocking task panicked during serialization")??; - - self.file_counter += 1; - let file_name = format!( - "delta-part-{:05}-epoch-{:010}-{:06}.{}", - subtask_idx, - epoch, - self.file_counter, - match self.format { - DeltaFormat::Csv => "csv", - DeltaFormat::Parquet => "parquet", - DeltaFormat::JsonL => "jsonl", - DeltaFormat::Avro => "avro", - DeltaFormat::Orc => "orc", - } - ); - match &self.destination { - DeltaDestination::Local(root) => { - let out = root.join(file_name); - let mut f = tokio::fs::File::create(&out).await.with_context(|| { - format!("failed creating delta sink file {}", out.display()) - })?; - f.write_all(&bytes) - .await - .with_context(|| format!("failed writing delta sink file {}", out.display()))?; - } - DeltaDestination::S3 { prefix, client } => { - let key = if prefix.is_empty() { - file_name - } else { - format!("{prefix}/{file_name}") - }; - client - .put( - &ObjectStorePath::from(key), - PutPayload::from(Bytes::from(bytes)), - ) - .await - .context("failed writing object to s3")?; - } - } - self.pending_bytes = 0; - Ok(()) - } -} - -#[async_trait] -impl Operator for DeltaSinkOperator { - fn name(&self) -> &str { - factory_operator_name::CONNECTOR_SINK - } - - async fn on_start(&mut self, _ctx: &mut TaskContext) -> Result<()> { - let reserved_block_bytes = self - .sink_memory_block - .as_ref() - .map(|b| b.capacity()) - .unwrap_or(0); - info!( - table = %self.table_name, - format = ?self.format, - reserved_block_bytes, - early_flush_threshold_bytes = self.early_flush_threshold_bytes, - "Starting delta sink operator" - ); - Ok(()) - } - - async fn process_data( - &mut self, - _input_idx: usize, - batch: RecordBatch, - ctx: &mut TaskContext, - _collector: &mut dyn Collector, - ) -> Result<()> { - let batch_size = batch.get_array_memory_size(); - self.pending.push(batch); - self.pending_bytes += batch_size; - - if self.pending_bytes > self.early_flush_threshold_bytes { - debug!( - table = %self.table_name, - bytes = self.pending_bytes, - "memory watermark reached, triggering early flush" - ); - self.flush_epoch(0, ctx.subtask_index as usize).await?; - } - Ok(()) - } - - async fn process_watermark( - &mut self, - _watermark: Watermark, - _ctx: &mut TaskContext, - _collector: &mut dyn Collector, - ) -> Result<()> { - Ok(()) - } - - async fn snapshot_state( - &mut self, - barrier: CheckpointBarrier, - ctx: &mut TaskContext, - ) -> Result<()> { - self.flush_epoch(barrier.epoch, ctx.subtask_index as usize) - .await - } - - async fn on_close(&mut self, ctx: &mut TaskContext) -> Result> { - if !self.pending.is_empty() { - warn!(table = %self.table_name, "flushing remaining delta sink batches on close"); - self.flush_epoch(u64::MAX, ctx.subtask_index as usize) - .await?; - } - Ok(vec![]) - } -} diff --git a/src/runtime_common/Cargo.toml b/src/runtime_common/Cargo.toml new file mode 100644 index 00000000..f8e98ed6 --- /dev/null +++ b/src/runtime_common/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "function-stream-runtime-common" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_runtime_common" +path = "src/lib.rs" + +[dependencies] +arrow-array = "55" +bincode = { version = "2", features = ["serde"] } +parking_lot = "0.12" +serde = { version = "1.0", features = ["derive"] } +tokio = { version = "1.0", features = ["sync"] } +tracing = "0.1" diff --git a/src/runtime/common/component_state.rs b/src/runtime_common/src/common/component_state.rs similarity index 100% rename from src/runtime/common/component_state.rs rename to src/runtime_common/src/common/component_state.rs diff --git a/src/runtime/common/mod.rs b/src/runtime_common/src/common/mod.rs similarity index 100% rename from src/runtime/common/mod.rs rename to src/runtime_common/src/common/mod.rs diff --git a/src/runtime/common/task_completion.rs b/src/runtime_common/src/common/task_completion.rs similarity index 100% rename from src/runtime/common/task_completion.rs rename to src/runtime_common/src/common/task_completion.rs diff --git a/src/runtime/mod.rs b/src/runtime_common/src/lib.rs similarity index 81% rename from src/runtime/mod.rs rename to src/runtime_common/src/lib.rs index 8c72b507..1c97062b 100644 --- a/src/runtime/mod.rs +++ b/src/runtime_common/src/lib.rs @@ -10,14 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Runtime module +//! Shared runtime building blocks (task lifecycle, memory pool) used by streaming and WASM. pub mod common; pub mod memory; -pub mod streaming; -pub mod util; -pub mod wasm; - -pub use wasm::input; -pub use wasm::output; -pub use wasm::processor; +pub mod streaming_protocol; diff --git a/src/runtime/memory/block.rs b/src/runtime_common/src/memory/block.rs similarity index 100% rename from src/runtime/memory/block.rs rename to src/runtime_common/src/memory/block.rs diff --git a/src/runtime/memory/error.rs b/src/runtime_common/src/memory/error.rs similarity index 100% rename from src/runtime/memory/error.rs rename to src/runtime_common/src/memory/error.rs diff --git a/src/runtime/memory/global.rs b/src/runtime_common/src/memory/global.rs similarity index 100% rename from src/runtime/memory/global.rs rename to src/runtime_common/src/memory/global.rs diff --git a/src/runtime/memory/mod.rs b/src/runtime_common/src/memory/mod.rs similarity index 100% rename from src/runtime/memory/mod.rs rename to src/runtime_common/src/memory/mod.rs diff --git a/src/runtime/memory/pool.rs b/src/runtime_common/src/memory/pool.rs similarity index 100% rename from src/runtime/memory/pool.rs rename to src/runtime_common/src/memory/pool.rs diff --git a/src/runtime/memory/ticket.rs b/src/runtime_common/src/memory/ticket.rs similarity index 100% rename from src/runtime/memory/ticket.rs rename to src/runtime_common/src/memory/ticket.rs diff --git a/src/runtime/streaming/protocol/event.rs b/src/runtime_common/src/streaming_protocol.rs similarity index 79% rename from src/runtime/streaming/protocol/event.rs rename to src/runtime_common/src/streaming_protocol.rs index 093d99ba..7c578f7a 100644 --- a/src/runtime/streaming/protocol/event.rs +++ b/src/runtime_common/src/streaming_protocol.rs @@ -10,15 +10,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! Streaming control-plane types shared by the SQL planner and the execution runtime. + use bincode::{Decode, Encode}; use serde::{Deserialize, Serialize}; -use std::sync::Arc; use std::time::SystemTime; -use arrow_array::RecordBatch; - -use crate::runtime::memory::MemoryTicket; - #[derive(Debug, Copy, Clone, PartialEq, Eq, Encode, Decode, Serialize, Deserialize)] pub enum Watermark { EventTime(SystemTime), @@ -33,44 +30,6 @@ pub struct CheckpointBarrier { pub then_stop: bool, } -#[derive(Debug, Clone)] -pub enum StreamEvent { - Data(RecordBatch), - Watermark(Watermark), - Barrier(CheckpointBarrier), - EndOfStream, -} - -#[derive(Debug, Clone)] -pub enum StreamOutput { - Forward(RecordBatch), - Keyed(u64, RecordBatch), - Broadcast(RecordBatch), - Watermark(Watermark), -} - -#[derive(Debug, Clone)] -pub struct TrackedEvent { - pub event: StreamEvent, - pub _ticket: Option>, -} - -impl TrackedEvent { - pub fn new(event: StreamEvent, ticket: Option) -> Self { - Self { - event, - _ticket: ticket.map(Arc::new), - } - } - - pub fn control(event: StreamEvent) -> Self { - Self { - event, - _ticket: None, - } - } -} - pub fn merge_watermarks(per_input: &[Option]) -> Option { if per_input.iter().any(|w| w.is_none()) { return None; diff --git a/src/servicer/Cargo.toml b/src/servicer/Cargo.toml new file mode 100644 index 00000000..16cd3109 --- /dev/null +++ b/src/servicer/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "function-stream-servicer" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_servicer" +path = "src/lib.rs" diff --git a/src/server/handler.rs b/src/servicer/src/handler.rs similarity index 96% rename from src/server/handler.rs rename to src/servicer/src/handler.rs index 0319e352..c884eb6f 100644 --- a/src/server/handler.rs +++ b/src/servicer/src/handler.rs @@ -26,7 +26,7 @@ use protocol::service::{ use crate::coordinator::{ Coordinator, CreateFunction, CreatePythonFunction, DataSet, DropFunction, PythonModule, - ShowFunctions, ShowFunctionsResult, StartFunction, Statement, StopFunction, + ShowFunctions, ShowFunctionsResult, StartFunction, Statement, StopFunction, classify_statement, }; use crate::sql::parse::parse_sql; @@ -135,12 +135,20 @@ impl FunctionStreamService for FunctionStreamServiceImpl { let timer = Instant::now(); let req = request.into_inner(); - let statements = parse_sql(&req.sql).map_err(|e| { + let ast = parse_sql(&req.sql).map_err(|e| { let detail = e.to_string(); warn!("SQL parse rejection: {}", detail); Status::invalid_argument(detail) })?; + let statements: Result>, _> = + ast.into_iter().map(classify_statement).collect(); + let statements = statements.map_err(|e| { + let detail = e.to_string(); + warn!("SQL classification rejection: {}", detail); + Status::invalid_argument(detail) + })?; + if statements.is_empty() { return Ok(TonicResponse::new(Self::build_success_response( StatusCode::Ok, diff --git a/src/server/initializer.rs b/src/servicer/src/initializer.rs similarity index 79% rename from src/server/initializer.rs rename to src/servicer/src/initializer.rs index 8a04608e..8f95cc1a 100644 --- a/src/server/initializer.rs +++ b/src/servicer/src/initializer.rs @@ -113,7 +113,7 @@ pub fn build_core_registry() -> ComponentRegistry { builder .register( "StreamCatalog", - crate::storage::stream_catalog::initialize_stream_catalog, + crate::stream_catalog::initialize_stream_catalog, ) .register("Coordinator", initialize_coordinator) .build() @@ -124,16 +124,16 @@ pub fn bootstrap_system(config: &GlobalConfig) -> Result<()> { registry.initialize_all(config)?; - crate::storage::stream_catalog::restore_global_catalog_from_store(); - crate::storage::stream_catalog::restore_streaming_jobs_from_store(); + crate::stream_catalog::restore_global_catalog_from_store(); + crate::stream_catalog::restore_streaming_jobs_from_store(); info!("System bootstrap finished. Node is ready to accept traffic."); Ok(()) } fn initialize_wasm_cache(config: &GlobalConfig) -> Result<()> { - crate::runtime::processor::wasm::wasm_cache::set_cache_config( - crate::runtime::processor::wasm::wasm_cache::WasmCacheConfig { + crate::processor::wasm::wasm_cache::set_cache_config( + crate::processor::wasm::wasm_cache::WasmCacheConfig { enabled: config.wasm.enable_cache, cache_dir: crate::config::paths::resolve_path(&config.wasm.cache_dir), max_size: config.wasm.max_cache_size, @@ -151,14 +151,14 @@ fn initialize_wasm_cache(config: &GlobalConfig) -> Result<()> { } fn initialize_task_manager(config: &GlobalConfig) -> Result<()> { - crate::runtime::wasm::taskexecutor::TaskManager::init(config) + crate::wasm::taskexecutor::TaskManager::init(config) .context("TaskManager service failed to start")?; Ok(()) } #[cfg(feature = "python")] fn initialize_python_service(config: &GlobalConfig) -> Result<()> { - crate::runtime::processor::python::PythonService::initialize(config) + crate::processor::python::PythonService::initialize(config) .context("Python Runtime initialization failed")?; Ok(()) } @@ -168,11 +168,28 @@ fn initialize_memory_service(config: &GlobalConfig) -> Result<()> { } fn initialize_job_manager(config: &GlobalConfig) -> Result<()> { - use crate::runtime::streaming::factory::OperatorFactory; - use crate::runtime::streaming::factory::Registry; - use crate::runtime::streaming::job::{JobManager, StateConfig}; + use crate::streaming::factory::OperatorFactory; + use crate::streaming::factory::Registry; + use crate::streaming::job::{ + CheckpointCatalog, JobManager, StateConfig, install_global_checkpoint_catalog, + }; use std::sync::Arc; + struct StreamCatalogCheckpointStore; + + impl CheckpointCatalog for StreamCatalogCheckpointStore { + fn commit_job_checkpoint( + &self, + job_id: &str, + epoch: u64, + source_infos: Vec, + ) -> Result<()> { + let catalog = crate::stream_catalog::CatalogManager::global() + .context("Checkpoint commit requires StreamCatalog")?; + Ok(catalog.commit_job_checkpoint(job_id, epoch, source_infos)?) + } + } + let per_operator_memory_bytes = config .streaming .operator_state_store_memory_bytes @@ -181,6 +198,8 @@ fn initialize_job_manager(config: &GlobalConfig) -> Result<()> { let registry = Arc::new(Registry::new()); let factory = Arc::new(OperatorFactory::new(registry)); + install_global_checkpoint_catalog(Arc::new(StreamCatalogCheckpointStore)) + .context("JobManager checkpoint catalog bridge install failed")?; let state_base_dir = std::env::temp_dir().join("function-stream").join("state"); let state_config = StateConfig { @@ -199,16 +218,16 @@ fn initialize_job_manager(config: &GlobalConfig) -> Result<()> { } fn initialize_coordinator(_config: &GlobalConfig) -> Result<()> { - crate::runtime::wasm::taskexecutor::TaskManager::get() + crate::wasm::taskexecutor::TaskManager::get() .context("Dependency violation: Coordinator requires TaskManager")?; - crate::runtime::memory::try_global_memory_pool() + crate::memory::try_global_memory_pool() .context("Dependency violation: Coordinator requires MemoryService")?; - crate::storage::stream_catalog::CatalogManager::global() + crate::stream_catalog::CatalogManager::global() .context("Dependency violation: Coordinator requires StreamCatalog")?; - crate::runtime::streaming::job::JobManager::global() + crate::streaming::job::JobManager::global() .context("Dependency violation: Coordinator requires JobManager")?; Ok(()) diff --git a/src/servicer/src/lib.rs b/src/servicer/src/lib.rs new file mode 100644 index 00000000..8a442937 --- /dev/null +++ b/src/servicer/src/lib.rs @@ -0,0 +1,3 @@ +//! Service layer implementations and request handling. + +pub const CRATE_NAME: &str = "function-stream-servicer"; diff --git a/src/server/memory_service.rs b/src/servicer/src/memory_service.rs similarity index 97% rename from src/server/memory_service.rs rename to src/servicer/src/memory_service.rs index 2ba24eee..5c03350b 100644 --- a/src/server/memory_service.rs +++ b/src/servicer/src/memory_service.rs @@ -52,7 +52,7 @@ impl MemoryService { let total_pool_bytes = streaming_runtime_memory_bytes.saturating_add(operator_state_store_memory_bytes); - crate::runtime::memory::init_global_memory_pool(total_pool_bytes) + crate::memory::init_global_memory_pool(total_pool_bytes) .context("Global memory pool initialization failed")?; info!("MemoryService initialized"); diff --git a/src/server/service.rs b/src/servicer/src/service.rs similarity index 100% rename from src/server/service.rs rename to src/servicer/src/service.rs diff --git a/src/server/mod.rs b/src/servicer/src/servicer_body.rs similarity index 100% rename from src/server/mod.rs rename to src/servicer/src/servicer_body.rs diff --git a/src/sql/logical_node/mod.rs b/src/sql/logical_node/mod.rs deleted file mode 100644 index 2da1b8a4..00000000 --- a/src/sql/logical_node/mod.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub mod logical; - -mod macros; - -pub(crate) mod streaming_operator_blueprint; -pub(crate) use streaming_operator_blueprint::{CompiledTopologyNode, StreamingOperatorBlueprint}; - -pub(crate) mod aggregate; -pub(crate) mod debezium; -pub(crate) mod join; -pub(crate) mod key_calculation; -pub(crate) mod lookup; -pub(crate) mod projection; -pub(crate) mod remote_table; -pub(crate) mod sink; -pub(crate) mod table_source; -pub(crate) mod updating_aggregate; -pub(crate) mod watermark_node; -pub(crate) mod windows_function; - -pub(crate) mod timestamp_append; -pub(crate) use timestamp_append::SystemTimestampInjectorNode; - -pub(crate) mod async_udf; -pub(crate) use async_udf::AsyncFunctionExecutionNode; - -pub(crate) mod is_retract; -pub(crate) use is_retract::IsRetractExtension; - -mod extension_try_from; diff --git a/src/sqlparser/Cargo.toml b/src/sqlparser/Cargo.toml new file mode 100644 index 00000000..7d6b40a6 --- /dev/null +++ b/src/sqlparser/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "function-stream-sqlparser" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_sqlparser" +path = "src/lib.rs" diff --git a/src/sqlparser/src/lib.rs b/src/sqlparser/src/lib.rs new file mode 100644 index 00000000..b436f895 --- /dev/null +++ b/src/sqlparser/src/lib.rs @@ -0,0 +1,3 @@ +//! SQL parsing facade for FunctionStream. + +pub const CRATE_NAME: &str = "function-stream-sqlparser"; diff --git a/src/streaming_planner/Cargo.toml b/src/streaming_planner/Cargo.toml new file mode 100644 index 00000000..766aba1c --- /dev/null +++ b/src/streaming_planner/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "function-stream-streaming-planner" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_streaming_planner" +path = "src/lib.rs" + +[dependencies] +protocol = { path = "../../protocol" } +prost = "0.13" +function-stream-config = { path = "../config" } +function-stream-runtime-common = { path = "../runtime_common" } +tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "sync", "time", "net"] } +tokio-stream = "0.1.18" +anyhow = "1.0" +xxhash-rust = { version = "0.8", features = ["xxh3"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +thiserror = "2" +tracing = "0.1" +async-trait = "0.1" +futures = "0.3" +itertools = "0.14" +petgraph = "0.7" +unicase = "2.7" +rand = { version = "0.8", features = ["small_rng"] } +bincode = { version = "2", features = ["serde"] } +chrono = "0.4" +bytes = "1" +ahash = "0.8" +strum = { version = "0.26", features = ["derive"] } +serde_json_path = "0.7" + +arrow = { version = "55", default-features = false } +arrow-array = "55" +arrow-schema = { version = "55", features = ["serde"] } +arrow-json = { version = "55.2.0" } +apache-avro = "0.21" + +datafusion = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-common = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-execution = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-expr = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-physical-expr = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-proto = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } + +sqlparser = { git = "https://github.com/FunctionStream/sqlparser-rs", branch = "0.58.0/fs" } diff --git a/src/sql/analysis/aggregate_rewriter.rs b/src/streaming_planner/src/analysis/aggregate_rewriter.rs similarity index 96% rename from src/sql/analysis/aggregate_rewriter.rs rename to src/streaming_planner/src/analysis/aggregate_rewriter.rs index ddcb0294..22dcb03c 100644 --- a/src/sql/analysis/aggregate_rewriter.rs +++ b/src/streaming_planner/src/analysis/aggregate_rewriter.rs @@ -17,19 +17,19 @@ use datafusion::logical_expr::{Aggregate, Expr, Extension, LogicalPlan, Projecti use datafusion::prelude::col; use std::sync::Arc; -use crate::sql::analysis::streaming_window_analzer::StreamingWindowAnalzer; -use crate::sql::logical_node::aggregate::StreamWindowAggregateNode; -use crate::sql::logical_node::key_calculation::{KeyExtractionNode, KeyExtractionStrategy}; -use crate::sql::logical_node::updating_aggregate::ContinuousAggregateNode; -use crate::sql::schema::StreamSchemaProvider; -use crate::sql::types::{ +use crate::analysis::streaming_window_analzer::StreamingWindowAnalzer; +use crate::logical_node::aggregate::StreamWindowAggregateNode; +use crate::logical_node::key_calculation::{KeyExtractionNode, KeyExtractionStrategy}; +use crate::logical_node::updating_aggregate::ContinuousAggregateNode; +use crate::schema::StreamSchemaProvider; +use crate::types::{ QualifiedField, TIMESTAMP_FIELD, WindowBehavior, WindowType, build_df_schema_with_metadata, extract_qualified_fields, extract_window_type, }; /// AggregateRewriter transforms batch DataFusion aggregates into streaming stateful operators. /// It handles windowing (Tumble/Hop/Session), watermarks, and continuous updating aggregates. -pub(crate) struct AggregateRewriter<'a> { +pub struct AggregateRewriter<'a> { pub schema_provider: &'a StreamSchemaProvider, } diff --git a/src/sql/analysis/async_udf_rewriter.rs b/src/streaming_planner/src/analysis/async_udf_rewriter.rs similarity index 95% rename from src/sql/analysis/async_udf_rewriter.rs rename to src/streaming_planner/src/analysis/async_udf_rewriter.rs index d6d9b54b..80f3828d 100644 --- a/src/sql/analysis/async_udf_rewriter.rs +++ b/src/streaming_planner/src/analysis/async_udf_rewriter.rs @@ -10,10 +10,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sql::common::constants::sql_field; -use crate::sql::logical_node::AsyncFunctionExecutionNode; -use crate::sql::logical_node::remote_table::RemoteTableBoundaryNode; -use crate::sql::schema::StreamSchemaProvider; +use crate::common::constants::sql_field; +use crate::logical_node::AsyncFunctionExecutionNode; +use crate::logical_node::remote_table::RemoteTableBoundaryNode; +use crate::schema::StreamSchemaProvider; use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRewriter}; use datafusion::common::{Column, Result as DFResult, TableReference, plan_err}; use datafusion::logical_expr::expr::ScalarFunction; diff --git a/src/sql/analysis/join_rewriter.rs b/src/streaming_planner/src/analysis/join_rewriter.rs similarity index 94% rename from src/sql/analysis/join_rewriter.rs rename to src/streaming_planner/src/analysis/join_rewriter.rs index 8a9e5280..07bc0006 100644 --- a/src/sql/analysis/join_rewriter.rs +++ b/src/streaming_planner/src/analysis/join_rewriter.rs @@ -10,13 +10,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sql::analysis::streaming_window_analzer::StreamingWindowAnalzer; -use crate::sql::common::TIMESTAMP_FIELD; -use crate::sql::common::constants::mem_exec_join_side; -use crate::sql::logical_node::join::StreamingJoinNode; -use crate::sql::logical_node::key_calculation::KeyExtractionNode; -use crate::sql::schema::StreamSchemaProvider; -use crate::sql::types::{WindowType, build_df_schema_with_metadata, extract_qualified_fields}; +use crate::analysis::streaming_window_analzer::StreamingWindowAnalzer; +use crate::common::TIMESTAMP_FIELD; +use crate::common::constants::mem_exec_join_side; +use crate::logical_node::join::StreamingJoinNode; +use crate::logical_node::key_calculation::KeyExtractionNode; +use crate::schema::StreamSchemaProvider; +use crate::types::{WindowType, build_df_schema_with_metadata, extract_qualified_fields}; use datafusion::common::tree_node::{Transformed, TreeNodeRewriter}; use datafusion::common::{ JoinConstraint, JoinType, Result, ScalarValue, TableReference, not_impl_err, plan_err, @@ -29,7 +29,7 @@ use std::sync::Arc; /// JoinRewriter handles the transformation of standard SQL joins into streaming-capable joins. /// It manages stateful "Updating Joins" and time-aligned "Instant Joins". -pub(crate) struct JoinRewriter<'a> { +pub struct JoinRewriter<'a> { pub schema_provider: &'a StreamSchemaProvider, } diff --git a/src/sql/analysis/mod.rs b/src/streaming_planner/src/analysis/mod.rs similarity index 90% rename from src/sql/analysis/mod.rs rename to src/streaming_planner/src/analysis/mod.rs index 019d8bf1..7ac1d4e8 100644 --- a/src/sql/analysis/mod.rs +++ b/src/streaming_planner/src/analysis/mod.rs @@ -12,12 +12,12 @@ #![allow(clippy::new_without_default)] -pub(crate) mod aggregate_rewriter; -pub(crate) mod join_rewriter; -pub(crate) mod row_time_rewriter; -pub(crate) mod stream_rewriter; -pub(crate) mod streaming_window_analzer; -pub(crate) mod window_function_rewriter; +pub mod aggregate_rewriter; +pub mod join_rewriter; +pub mod row_time_rewriter; +pub mod stream_rewriter; +pub mod streaming_window_analzer; +pub mod window_function_rewriter; pub mod async_udf_rewriter; pub mod sink_input_rewriter; @@ -31,7 +31,7 @@ pub use sink_input_rewriter::SinkInputRewriter; pub use time_window::{TimeWindowNullCheckRemover, TimeWindowUdfChecker}; pub use unnest_rewriter::UNNESTED_COL; -pub use crate::sql::schema::schema_provider::StreamSchemaProvider; +pub use crate::schema::schema_provider::StreamSchemaProvider; use std::collections::HashMap; use std::sync::Arc; @@ -42,11 +42,11 @@ use datafusion::error::DataFusionError; use datafusion::logical_expr::{Extension, LogicalPlan, UserDefinedLogicalNodeCore}; use tracing::{debug, info, instrument}; -use crate::sql::logical_node::StreamingOperatorBlueprint; -use crate::sql::logical_node::key_calculation::{KeyExtractionNode, KeyExtractionStrategy}; -use crate::sql::logical_node::projection::StreamProjectionNode; -use crate::sql::logical_node::sink::StreamEgressNode; -use crate::sql::logical_planner::planner::NamedNode; +use crate::logical_node::StreamingOperatorBlueprint; +use crate::logical_node::key_calculation::{KeyExtractionNode, KeyExtractionStrategy}; +use crate::logical_node::projection::StreamProjectionNode; +use crate::logical_node::sink::StreamEgressNode; +use crate::logical_planner::planner::NamedNode; fn duration_from_sql_expr( expr: &datafusion::sql::sqlparser::ast::Expr, @@ -110,7 +110,7 @@ fn build_sink_inputs(extensions: &[LogicalPlan]) -> HashMap Result { +pub fn maybe_add_key_extension_to_sink(plan: LogicalPlan) -> Result { let LogicalPlan::Extension(ref ext) = plan else { return Ok(plan); }; diff --git a/src/sql/analysis/row_time_rewriter.rs b/src/streaming_planner/src/analysis/row_time_rewriter.rs similarity index 95% rename from src/sql/analysis/row_time_rewriter.rs rename to src/streaming_planner/src/analysis/row_time_rewriter.rs index 13e2a048..aa558696 100644 --- a/src/sql/analysis/row_time_rewriter.rs +++ b/src/streaming_planner/src/analysis/row_time_rewriter.rs @@ -14,8 +14,8 @@ use datafusion::common::tree_node::{Transformed, TreeNodeRewriter}; use datafusion::common::{Column, Result as DFResult}; use datafusion::logical_expr::Expr; -use crate::sql::common::constants::planning_placeholder_udf; -use crate::sql::types::TIMESTAMP_FIELD; +use crate::common::constants::planning_placeholder_udf; +use crate::types::TIMESTAMP_FIELD; /// Replaces the virtual `row_time()` scalar function with a physical reference to `_timestamp`. /// diff --git a/src/sql/analysis/sink_input_rewriter.rs b/src/streaming_planner/src/analysis/sink_input_rewriter.rs similarity index 89% rename from src/sql/analysis/sink_input_rewriter.rs rename to src/streaming_planner/src/analysis/sink_input_rewriter.rs index 201439cc..9f8fdcb7 100644 --- a/src/sql/analysis/sink_input_rewriter.rs +++ b/src/streaming_planner/src/analysis/sink_input_rewriter.rs @@ -10,9 +10,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sql::logical_node::StreamingOperatorBlueprint; -use crate::sql::logical_node::sink::StreamEgressNode; -use crate::sql::logical_planner::planner::NamedNode; +use crate::logical_node::StreamingOperatorBlueprint; +use crate::logical_node::sink::StreamEgressNode; +use crate::logical_planner::planner::NamedNode; use datafusion::common::Result as DFResult; use datafusion::common::tree_node::{Transformed, TreeNodeRecursion, TreeNodeRewriter}; use datafusion::logical_expr::{Extension, LogicalPlan, UserDefinedLogicalNodeCore}; @@ -28,7 +28,7 @@ pub struct SinkInputRewriter<'a> { } impl<'a> SinkInputRewriter<'a> { - pub(crate) fn new(sink_inputs: &'a mut SinkInputs) -> Self { + pub fn new(sink_inputs: &'a mut SinkInputs) -> Self { Self { sink_inputs, was_removed: false, diff --git a/src/sql/analysis/source_metadata_visitor.rs b/src/streaming_planner/src/analysis/source_metadata_visitor.rs similarity index 82% rename from src/sql/analysis/source_metadata_visitor.rs rename to src/streaming_planner/src/analysis/source_metadata_visitor.rs index b37d3a1b..ea2821b9 100644 --- a/src/sql/analysis/source_metadata_visitor.rs +++ b/src/streaming_planner/src/analysis/source_metadata_visitor.rs @@ -10,9 +10,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sql::logical_node::sink::{STREAM_EGRESS_NODE_NAME, StreamEgressNode}; -use crate::sql::logical_node::table_source::{STREAM_INGESTION_NODE_NAME, StreamIngestionNode}; -use crate::sql::schema::StreamSchemaProvider; +use crate::logical_node::sink::{STREAM_EGRESS_NODE_NAME, StreamEgressNode}; +use crate::logical_node::table_source::{STREAM_INGESTION_NODE_NAME, StreamIngestionNode}; +use crate::schema::StreamSchemaProvider; use datafusion::common::Result as DFResult; use datafusion::common::tree_node::{TreeNodeRecursion, TreeNodeVisitor}; use datafusion::logical_expr::{Extension, LogicalPlan}; @@ -51,9 +51,9 @@ impl<'a> SourceMetadataVisitor<'a> { let table = self.schema_provider.get_catalog_table(&table_name)?; match table { - crate::sql::schema::table::CatalogEntity::ExternalConnector(b) => match b.as_ref() { - crate::sql::schema::catalog::ExternalTable::Source(t) => t.registry_id, - crate::sql::schema::catalog::ExternalTable::Lookup(t) => t.registry_id, + crate::schema::table::CatalogEntity::ExternalConnector(b) => match b.as_ref() { + crate::schema::catalog::ExternalTable::Source(t) => t.registry_id, + crate::schema::catalog::ExternalTable::Lookup(t) => t.registry_id, _ => None, }, _ => None, diff --git a/src/sql/analysis/source_rewriter.rs b/src/streaming_planner/src/analysis/source_rewriter.rs similarity index 95% rename from src/sql/analysis/source_rewriter.rs rename to src/streaming_planner/src/analysis/source_rewriter.rs index 620ea336..e1a7c75a 100644 --- a/src/sql/analysis/source_rewriter.rs +++ b/src/streaming_planner/src/analysis/source_rewriter.rs @@ -20,16 +20,16 @@ use datafusion::logical_expr::{ self, BinaryExpr, Expr, Extension, LogicalPlan, Projection, TableScan, }; -use crate::sql::common::UPDATING_META_FIELD; -use crate::sql::logical_node::debezium::UnrollDebeziumPayloadNode; -use crate::sql::logical_node::remote_table::RemoteTableBoundaryNode; -use crate::sql::logical_node::table_source::StreamIngestionNode; -use crate::sql::logical_node::watermark_node::EventTimeWatermarkNode; -use crate::sql::schema::ColumnDescriptor; -use crate::sql::schema::StreamSchemaProvider; -use crate::sql::schema::catalog::{ExternalTable, SourceTable}; -use crate::sql::schema::table::CatalogEntity; -use crate::sql::types::TIMESTAMP_FIELD; +use crate::common::UPDATING_META_FIELD; +use crate::logical_node::debezium::UnrollDebeziumPayloadNode; +use crate::logical_node::remote_table::RemoteTableBoundaryNode; +use crate::logical_node::table_source::StreamIngestionNode; +use crate::logical_node::watermark_node::EventTimeWatermarkNode; +use crate::schema::ColumnDescriptor; +use crate::schema::StreamSchemaProvider; +use crate::schema::catalog::{ExternalTable, SourceTable}; +use crate::schema::table::CatalogEntity; +use crate::types::TIMESTAMP_FIELD; /// Rewrites table scans: projections are lifted out of scans into a dedicated projection node /// (including virtual fields), using a connector table-source extension instead of a bare diff --git a/src/sql/analysis/stream_rewriter.rs b/src/streaming_planner/src/analysis/stream_rewriter.rs similarity index 96% rename from src/sql/analysis/stream_rewriter.rs rename to src/streaming_planner/src/analysis/stream_rewriter.rs index a4393bd1..efc4e40c 100644 --- a/src/sql/analysis/stream_rewriter.rs +++ b/src/streaming_planner/src/analysis/stream_rewriter.rs @@ -13,16 +13,16 @@ use std::sync::Arc; use super::StreamSchemaProvider; -use crate::sql::analysis::TimeWindowNullCheckRemover; -use crate::sql::analysis::row_time_rewriter::RowTimeRewriter; -use crate::sql::analysis::{ +use crate::analysis::TimeWindowNullCheckRemover; +use crate::analysis::row_time_rewriter::RowTimeRewriter; +use crate::analysis::{ aggregate_rewriter::AggregateRewriter, join_rewriter::JoinRewriter, window_function_rewriter::WindowFunctionRewriter, }; -use crate::sql::logical_node::StreamingOperatorBlueprint; -use crate::sql::logical_node::remote_table::RemoteTableBoundaryNode; -use crate::sql::schema::utils::{add_timestamp_field, has_timestamp_field}; -use crate::sql::types::{QualifiedField, TIMESTAMP_FIELD}; +use crate::logical_node::StreamingOperatorBlueprint; +use crate::logical_node::remote_table::RemoteTableBoundaryNode; +use crate::schema::utils::{add_timestamp_field, has_timestamp_field}; +use crate::types::{QualifiedField, TIMESTAMP_FIELD}; use datafusion::common::tree_node::{Transformed, TreeNodeRewriter}; use datafusion::common::{Column, DataFusionError, Result, Spans, TableReference, plan_err}; use datafusion::logical_expr::{ diff --git a/src/sql/analysis/streaming_window_analzer.rs b/src/streaming_planner/src/analysis/streaming_window_analzer.rs similarity index 95% rename from src/sql/analysis/streaming_window_analzer.rs rename to src/streaming_planner/src/analysis/streaming_window_analzer.rs index b8a7f78f..5e6d3e91 100644 --- a/src/sql/analysis/streaming_window_analzer.rs +++ b/src/streaming_planner/src/analysis/streaming_window_analzer.rs @@ -17,16 +17,16 @@ use datafusion::common::tree_node::{TreeNodeRecursion, TreeNodeVisitor}; use datafusion::common::{Column, DFSchema, DataFusionError, Result}; use datafusion::logical_expr::{Expr, Extension, LogicalPlan, expr::Alias}; -use crate::sql::logical_node::aggregate::{STREAM_AGG_EXTENSION_NAME, StreamWindowAggregateNode}; -use crate::sql::logical_node::join::STREAM_JOIN_NODE_TYPE; -use crate::sql::types::{ +use crate::logical_node::aggregate::{STREAM_AGG_EXTENSION_NAME, StreamWindowAggregateNode}; +use crate::logical_node::join::STREAM_JOIN_NODE_TYPE; +use crate::types::{ QualifiedField, WindowBehavior, WindowType, extract_qualified_fields, extract_window_type, }; /// WindowDetectingVisitor identifies windowing strategies and tracks window-carrying fields /// as they propagate upward through the logical plan tree. #[derive(Debug, Default)] -pub(crate) struct StreamingWindowAnalzer { +pub struct StreamingWindowAnalzer { /// The specific window type discovered (Tumble, Hop, etc.) pub(crate) window: Option, /// Set of fields in the current plan node that carry window semantics. @@ -35,7 +35,7 @@ pub(crate) struct StreamingWindowAnalzer { impl StreamingWindowAnalzer { /// Entry point to resolve the WindowType of a given plan branch. - pub(crate) fn get_window(logical_plan: &LogicalPlan) -> Result> { + pub fn get_window(logical_plan: &LogicalPlan) -> Result> { let mut visitor = Self::default(); logical_plan.visit_with_subqueries(&mut visitor)?; Ok(visitor.window) @@ -89,7 +89,7 @@ impl StreamingWindowAnalzer { } } -pub(crate) fn extract_column(expr: &Expr) -> Option<&Column> { +pub fn extract_column(expr: &Expr) -> Option<&Column> { match expr { Expr::Column(column) => Some(column), Expr::Alias(Alias { expr, .. }) => extract_column(expr), diff --git a/src/sql/analysis/time_window.rs b/src/streaming_planner/src/analysis/time_window.rs similarity index 100% rename from src/sql/analysis/time_window.rs rename to src/streaming_planner/src/analysis/time_window.rs diff --git a/src/sql/analysis/udafs.rs b/src/streaming_planner/src/analysis/udafs.rs similarity index 100% rename from src/sql/analysis/udafs.rs rename to src/streaming_planner/src/analysis/udafs.rs diff --git a/src/sql/analysis/unnest_rewriter.rs b/src/streaming_planner/src/analysis/unnest_rewriter.rs similarity index 97% rename from src/sql/analysis/unnest_rewriter.rs rename to src/streaming_planner/src/analysis/unnest_rewriter.rs index 147b1f49..e63878f7 100644 --- a/src/sql/analysis/unnest_rewriter.rs +++ b/src/streaming_planner/src/analysis/unnest_rewriter.rs @@ -18,8 +18,8 @@ use datafusion::common::{Column, Result as DFResult, plan_err}; use datafusion::logical_expr::expr::ScalarFunction; use datafusion::logical_expr::{ColumnUnnestList, Expr, LogicalPlan, Projection, Unnest}; -use crate::sql::common::constants::planning_placeholder_udf; -use crate::sql::types::{QualifiedField, build_df_schema, extract_qualified_fields}; +use crate::common::constants::planning_placeholder_udf; +use crate::types::{QualifiedField, build_df_schema, extract_qualified_fields}; pub const UNNESTED_COL: &str = "__unnested"; diff --git a/src/sql/analysis/window_function_rewriter.rs b/src/streaming_planner/src/analysis/window_function_rewriter.rs similarity index 93% rename from src/sql/analysis/window_function_rewriter.rs rename to src/streaming_planner/src/analysis/window_function_rewriter.rs index c1e3396d..569a3c00 100644 --- a/src/sql/analysis/window_function_rewriter.rs +++ b/src/streaming_planner/src/analysis/window_function_rewriter.rs @@ -20,14 +20,14 @@ use datafusion_common::DataFusionError; use std::sync::Arc; use tracing::debug; -use crate::sql::analysis::streaming_window_analzer::{StreamingWindowAnalzer, extract_column}; -use crate::sql::logical_node::key_calculation::{KeyExtractionNode, KeyExtractionStrategy}; -use crate::sql::logical_node::windows_function::StreamingWindowFunctionNode; -use crate::sql::types::{WindowType, build_df_schema, extract_qualified_fields}; +use crate::analysis::streaming_window_analzer::{StreamingWindowAnalzer, extract_column}; +use crate::logical_node::key_calculation::{KeyExtractionNode, KeyExtractionStrategy}; +use crate::logical_node::windows_function::StreamingWindowFunctionNode; +use crate::types::{WindowType, build_df_schema, extract_qualified_fields}; /// WindowFunctionRewriter transforms standard SQL Window functions into streaming-compatible /// stateful operators, ensuring proper data partitioning and sorting for distributed execution. -pub(crate) struct WindowFunctionRewriter; +pub struct WindowFunctionRewriter; impl WindowFunctionRewriter { /// Recursively unwraps Aliases to find the underlying WindowFunction. @@ -48,7 +48,7 @@ impl WindowFunctionRewriter { &self, params: &WindowFunctionParams, input: &LogicalPlan, - input_window_fields: &std::collections::HashSet, + input_window_fields: &std::collections::HashSet, ) -> DFResult { let matched: Vec<_> = params .partition_by diff --git a/src/sql/api/checkpoints.rs b/src/streaming_planner/src/api/checkpoints.rs similarity index 98% rename from src/sql/api/checkpoints.rs rename to src/streaming_planner/src/api/checkpoints.rs index d9bdc139..bd326b10 100644 --- a/src/sql/api/checkpoints.rs +++ b/src/streaming_planner/src/api/checkpoints.rs @@ -10,7 +10,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sql::common::to_micros; +use crate::common::to_micros; use serde::{Deserialize, Serialize}; use std::time::SystemTime; diff --git a/src/sql/api/connections.rs b/src/streaming_planner/src/api/connections.rs similarity index 99% rename from src/sql/api/connections.rs rename to src/streaming_planner/src/api/connections.rs index 3c5caf76..d2d92bcb 100644 --- a/src/sql/api/connections.rs +++ b/src/streaming_planner/src/api/connections.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sql::common::formats::{BadData, Format, Framing}; -use crate::sql::common::{FsExtensionType, FsSchema}; +use crate::common::formats::{BadData, Format, Framing}; +use crate::common::{FsExtensionType, FsSchema}; use datafusion::arrow::datatypes::{DataType, Field, Fields, TimeUnit}; use serde::ser::SerializeMap; use serde::{Deserialize, Serialize, Serializer}; diff --git a/src/sql/api/metrics.rs b/src/streaming_planner/src/api/metrics.rs similarity index 100% rename from src/sql/api/metrics.rs rename to src/streaming_planner/src/api/metrics.rs diff --git a/src/sql/api/mod.rs b/src/streaming_planner/src/api/mod.rs similarity index 100% rename from src/sql/api/mod.rs rename to src/streaming_planner/src/api/mod.rs diff --git a/src/sql/api/pipelines.rs b/src/streaming_planner/src/api/pipelines.rs similarity index 98% rename from src/sql/api/pipelines.rs rename to src/streaming_planner/src/api/pipelines.rs index d6cc5253..990c1ba9 100644 --- a/src/sql/api/pipelines.rs +++ b/src/streaming_planner/src/api/pipelines.rs @@ -11,7 +11,7 @@ // limitations under the License. use super::udfs::Udf; -use crate::sql::common::control::ErrorDomain; +use crate::common::control::ErrorDomain; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Clone, Debug)] diff --git a/src/sql/api/public_ids.rs b/src/streaming_planner/src/api/public_ids.rs similarity index 100% rename from src/sql/api/public_ids.rs rename to src/streaming_planner/src/api/public_ids.rs diff --git a/src/sql/api/schema_resolver.rs b/src/streaming_planner/src/api/schema_resolver.rs similarity index 100% rename from src/sql/api/schema_resolver.rs rename to src/streaming_planner/src/api/schema_resolver.rs diff --git a/src/sql/api/udfs.rs b/src/streaming_planner/src/api/udfs.rs similarity index 100% rename from src/sql/api/udfs.rs rename to src/streaming_planner/src/api/udfs.rs diff --git a/src/sql/api/var_str.rs b/src/streaming_planner/src/api/var_str.rs similarity index 100% rename from src/sql/api/var_str.rs rename to src/streaming_planner/src/api/var_str.rs diff --git a/src/sql/common/arrow_ext.rs b/src/streaming_planner/src/common/arrow_ext.rs similarity index 100% rename from src/sql/common/arrow_ext.rs rename to src/streaming_planner/src/common/arrow_ext.rs diff --git a/src/coordinator/plan/ast_utils.rs b/src/streaming_planner/src/common/ast_utils.rs similarity index 98% rename from src/coordinator/plan/ast_utils.rs rename to src/streaming_planner/src/common/ast_utils.rs index d2432888..cc0eb95f 100644 --- a/src/coordinator/plan/ast_utils.rs +++ b/src/streaming_planner/src/common/ast_utils.rs @@ -20,7 +20,7 @@ use datafusion::common::{Result, plan_err}; use datafusion::sql::sqlparser::ast::{Expr as SqlExpr, SqlOption, TableConstraint}; -use crate::sql::common::with_option_keys as opt; +use crate::common::with_option_keys as opt; /// Namespace for AST extraction helpers. pub struct AstUtils; diff --git a/src/sql/common/connector_options.rs b/src/streaming_planner/src/common/connector_options.rs similarity index 100% rename from src/sql/common/connector_options.rs rename to src/streaming_planner/src/common/connector_options.rs diff --git a/src/sql/common/constants.rs b/src/streaming_planner/src/common/constants.rs similarity index 97% rename from src/sql/common/constants.rs rename to src/streaming_planner/src/common/constants.rs index 8cdb68e3..467247b1 100644 --- a/src/sql/common/constants.rs +++ b/src/streaming_planner/src/common/constants.rs @@ -257,6 +257,10 @@ pub mod connector_type { pub const NATS: &str = "nats"; pub const REDIS: &str = "redis"; pub const MQTT: &str = "mqtt"; + pub const HTTP: &str = "http"; + pub const ROS: &str = "ros"; + pub const ROS2: &str = "ros2"; + pub const ROBOT_BAG: &str = "robot-bag"; pub const WEBSOCKET: &str = "websocket"; pub const SSE: &str = "sse"; pub const NEXMARK: &str = "nexmark"; @@ -273,6 +277,11 @@ pub mod connection_table_role { pub const SUPPORTED_CONNECTOR_ADAPTERS: &[&str] = &[ connector_type::KAFKA, + connector_type::MQTT, + connector_type::HTTP, + connector_type::ROS, + connector_type::ROS2, + connector_type::ROBOT_BAG, connector_type::FILESYSTEM, connector_type::S3, connector_type::DELTA, diff --git a/src/sql/common/control.rs b/src/streaming_planner/src/common/control.rs similarity index 98% rename from src/sql/common/control.rs rename to src/streaming_planner/src/common/control.rs index eba88596..59ef409e 100644 --- a/src/sql/common/control.rs +++ b/src/streaming_planner/src/common/control.rs @@ -13,7 +13,7 @@ use std::collections::HashMap; use std::time::SystemTime; -use crate::runtime::streaming::protocol::CheckpointBarrier; +use function_stream_runtime_common::streaming_protocol::CheckpointBarrier; /// Control messages sent from the controller to worker tasks. #[derive(Debug, Clone)] diff --git a/src/sql/common/converter.rs b/src/streaming_planner/src/common/converter.rs similarity index 100% rename from src/sql/common/converter.rs rename to src/streaming_planner/src/common/converter.rs diff --git a/src/sql/common/date.rs b/src/streaming_planner/src/common/date.rs similarity index 100% rename from src/sql/common/date.rs rename to src/streaming_planner/src/common/date.rs diff --git a/src/sql/common/debezium.rs b/src/streaming_planner/src/common/debezium.rs similarity index 100% rename from src/sql/common/debezium.rs rename to src/streaming_planner/src/common/debezium.rs diff --git a/src/sql/common/errors.rs b/src/streaming_planner/src/common/errors.rs similarity index 97% rename from src/sql/common/errors.rs rename to src/streaming_planner/src/common/errors.rs index fa4a722e..b7e97400 100644 --- a/src/sql/common/errors.rs +++ b/src/streaming_planner/src/common/errors.rs @@ -67,7 +67,7 @@ impl From for DataflowError { #[macro_export] macro_rules! connector_err { ($($arg:tt)*) => { - $crate::sql::common::errors::DataflowError::Connector(format!($($arg)*)) + $crate::common::errors::DataflowError::Connector(format!($($arg)*)) }; } diff --git a/src/sql/common/format_from_opts.rs b/src/streaming_planner/src/common/format_from_opts.rs similarity index 100% rename from src/sql/common/format_from_opts.rs rename to src/streaming_planner/src/common/format_from_opts.rs diff --git a/src/sql/common/formats.rs b/src/streaming_planner/src/common/formats.rs similarity index 100% rename from src/sql/common/formats.rs rename to src/streaming_planner/src/common/formats.rs diff --git a/src/sql/common/fs_schema.rs b/src/streaming_planner/src/common/fs_schema.rs similarity index 99% rename from src/sql/common/fs_schema.rs rename to src/streaming_planner/src/common/fs_schema.rs index 76a08537..9c548f69 100644 --- a/src/sql/common/fs_schema.rs +++ b/src/streaming_planner/src/common/fs_schema.rs @@ -11,7 +11,7 @@ // limitations under the License. use super::{TIMESTAMP_FIELD, to_nanos}; -use crate::sql::common::converter::Converter; +use crate::common::converter::Converter; use arrow::compute::kernels::cmp::gt_eq; use arrow::compute::kernels::numeric::div; use arrow::compute::{SortColumn, filter_record_batch, lexsort_to_indices, partition, take}; diff --git a/src/sql/common/kafka_catalog.rs b/src/streaming_planner/src/common/kafka_catalog.rs similarity index 100% rename from src/sql/common/kafka_catalog.rs rename to src/streaming_planner/src/common/kafka_catalog.rs diff --git a/src/sql/common/mod.rs b/src/streaming_planner/src/common/mod.rs similarity index 90% rename from src/sql/common/mod.rs rename to src/streaming_planner/src/common/mod.rs index af44cb0f..0bf6bdc0 100644 --- a/src/sql/common/mod.rs +++ b/src/streaming_planner/src/common/mod.rs @@ -10,12 +10,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Shared core types and constants for FunctionStream (`crate::sql::common`). +//! Shared core types and constants for FunctionStream streaming planner (`crate::common`). //! //! Used by the runtime, SQL planner, coordinator, and other subsystems — //! analogous to `arroyo-types` + `arroyo-rpc` in Arroyo. pub mod arrow_ext; +pub mod ast_utils; pub mod connector_options; pub mod constants; pub mod control; @@ -33,11 +34,12 @@ pub mod topology; pub mod with_option_keys; // ── Re-exports from existing modules ── -pub use crate::runtime::streaming::protocol::{CheckpointBarrier, Watermark}; pub use arrow_ext::FsExtensionType; +pub use function_stream_runtime_common::streaming_protocol::{CheckpointBarrier, Watermark}; pub use time_utils::{from_nanos, to_micros, to_millis, to_nanos}; // ── Re-exports from new modules ── +pub use ast_utils::AstUtils; pub use connector_options::ConnectorOptions; pub use formats::{BadData, Format, Framing, JsonCompression, JsonFormat}; pub use fs_schema::{FsSchema, FsSchemaRef}; diff --git a/src/sql/common/operator_config.rs b/src/streaming_planner/src/common/operator_config.rs similarity index 100% rename from src/sql/common/operator_config.rs rename to src/streaming_planner/src/common/operator_config.rs diff --git a/src/sql/common/time_utils.rs b/src/streaming_planner/src/common/time_utils.rs similarity index 100% rename from src/sql/common/time_utils.rs rename to src/streaming_planner/src/common/time_utils.rs diff --git a/src/sql/common/topology.rs b/src/streaming_planner/src/common/topology.rs similarity index 100% rename from src/sql/common/topology.rs rename to src/streaming_planner/src/common/topology.rs diff --git a/src/sql/common/with_option_keys.rs b/src/streaming_planner/src/common/with_option_keys.rs similarity index 58% rename from src/sql/common/with_option_keys.rs rename to src/streaming_planner/src/common/with_option_keys.rs index 21bfa691..dbb21f67 100644 --- a/src/sql/common/with_option_keys.rs +++ b/src/streaming_planner/src/common/with_option_keys.rs @@ -50,6 +50,64 @@ pub const KAFKA_KEY_FIELD_LEGACY: &str = "key.field"; pub const KAFKA_SINK_TIMESTAMP_FIELD: &str = "sink.timestamp.field"; pub const KAFKA_TIMESTAMP_FIELD_LEGACY: &str = "timestamp.field"; +// ── MQTT ────────────────────────────────────────────────────────────────── + +pub const MQTT_HOST: &str = "mqtt.host"; +pub const MQTT_HOST_LEGACY: &str = "host"; +pub const MQTT_PORT: &str = "mqtt.port"; +pub const MQTT_PORT_LEGACY: &str = "port"; +pub const MQTT_TOPIC: &str = "topic"; +pub const MQTT_CLIENT_ID: &str = "mqtt.client.id"; +pub const MQTT_CLIENT_ID_LEGACY: &str = "client.id"; +pub const MQTT_USERNAME: &str = "mqtt.username"; +pub const MQTT_USERNAME_LEGACY: &str = "username"; +pub const MQTT_PASSWORD: &str = "mqtt.password"; +pub const MQTT_PASSWORD_LEGACY: &str = "password"; +pub const MQTT_QOS: &str = "mqtt.qos"; +pub const MQTT_QOS_LEGACY: &str = "qos"; +pub const MQTT_CLEAN_SESSION: &str = "mqtt.clean.session"; +pub const MQTT_CLEAN_SESSION_LEGACY: &str = "clean.session"; +pub const MQTT_KEEP_ALIVE_SECS: &str = "mqtt.keep.alive.secs"; +pub const MQTT_KEEP_ALIVE_SECS_LEGACY: &str = "keep.alive.secs"; +pub const MQTT_RATE_LIMIT_MESSAGES_PER_SECOND: &str = "rate_limit.messages_per_second"; + +// ── HTTP (Flink rest-lookup / robot REST aligned) ───────────────────────── + +pub const HTTP_MODE: &str = "mode"; +pub const HTTP_URL: &str = "url"; +pub const HTTP_METHOD: &str = "http.method"; +pub const HTTP_LOOKUP_METHOD: &str = "lookup-method"; +pub const HTTP_SCAN_INTERVAL: &str = "scan.interval"; +pub const HTTP_SCAN_INTERVAL_MS: &str = "scan.interval.ms"; +pub const HTTP_POLL_INTERVAL_MS: &str = "http.poll.interval.ms"; +pub const HTTP_REQUEST_BODY: &str = "http.request.body"; +pub const HTTP_REQUEST_BODY_LEGACY: &str = "body"; +pub const HTTP_LISTEN_HOST: &str = "http.listen.host"; +pub const HTTP_LISTEN_PORT: &str = "http.listen.port"; +pub const HTTP_WEBHOOK_PATH: &str = "http.webhook.path"; +pub const HTTP_REQUEST_TIMEOUT_MS: &str = "http.request.timeout.ms"; +pub const HTTP_RESPONSE_SPLIT: &str = "http.response.split"; +pub const HTTP_HEADER_PREFIX: &str = "header."; + +// ── ROS / rosbridge ─────────────────────────────────────────────────────── + +pub const ROS_URL: &str = "url"; +pub const ROS_URL_LEGACY: &str = "ros.url"; +pub const ROS_TOPIC: &str = "topic"; +pub const ROS_TOPIC_LEGACY: &str = "ros.topic"; +pub const ROS_MESSAGE_FIELD: &str = "ros.message.field"; +pub const ROS_MESSAGE_FIELD_LEGACY: &str = "message.field"; +pub const ROS_VERSION: &str = "ros.version"; + +// ── Robot bag / file formats ────────────────────────────────────────────── + +pub const ROBOT_BAG_PATH: &str = "path"; +pub const ROBOT_BAG_FORMAT: &str = "bag.format"; +pub const ROBOT_BAG_TOPIC: &str = "topic"; +pub const ROBOT_BAG_REPLAY_INTERVAL_MS: &str = "replay.interval.ms"; +pub const ROBOT_BAG_LOOP: &str = "loop"; +pub const ROBOT_BAG_PCD_EMIT_MODE: &str = "pcd.emit.mode"; + // ── JSON format ─────────────────────────────────────────────────────────── pub const JSON_CONFLUENT_SCHEMA_REGISTRY: &str = "json.confluent_schema_registry"; @@ -82,6 +140,24 @@ pub const S3_ACCESS_KEY_ID: &str = "s3.access_key_id"; pub const S3_SECRET_ACCESS_KEY: &str = "s3.secret_access_key"; pub const S3_SESSION_TOKEN: &str = "s3.session_token"; +// ── Delta Lake commit coordination (S3-compatible backends) ─────────────── +// +// `delta.commit.strategy` selects how concurrent-writer safety is enforced +// when committing to `_delta_log` on S3-compatible object stores: +// +// - `single-writer` (default): rely on Function-Stream's pipeline scheduler +// that guarantees a single writer per Delta table. Commits use direct +// PUT and are atomic per object. Suitable for AWS S3, MinIO, R2, GCS-S3. +// - `dynamodb`: use deltalake-aws DynamoDB lock client for multi-writer +// safety. Requires `delta.dynamodb.table` and AWS credentials/region. +// +pub const DELTA_COMMIT_STRATEGY: &str = "delta.commit.strategy"; +pub const DELTA_DYNAMODB_TABLE: &str = "delta.dynamodb.table"; +pub const DELTA_DYNAMODB_REGION: &str = "delta.dynamodb.region"; +pub const DELTA_DYNAMODB_ENDPOINT: &str = "delta.dynamodb.endpoint"; +pub const DELTA_DYNAMODB_ACCESS_KEY_ID: &str = "delta.dynamodb.access_key_id"; +pub const DELTA_DYNAMODB_SECRET_ACCESS_KEY: &str = "delta.dynamodb.secret_access_key"; + // ── Protobuf ──────────────────────────────────────────────────────────────── pub const PROTOBUF_INTO_UNSTRUCTURED_JSON: &str = "protobuf.into_unstructured_json"; diff --git a/src/sql/connector/config.rs b/src/streaming_planner/src/connector/config.rs similarity index 70% rename from src/sql/connector/config.rs rename to src/streaming_planner/src/connector/config.rs index c3eaf00f..4d4c473b 100644 --- a/src/sql/connector/config.rs +++ b/src/streaming_planner/src/connector/config.rs @@ -12,12 +12,17 @@ use protocol::function_stream_graph::{ DeltaSinkConfig, FilesystemSinkConfig, IcebergSinkConfig, KafkaSinkConfig, KafkaSourceConfig, - LanceDbSinkConfig, S3SinkConfig, connector_op, + HttpSourceConfig, LanceDbSinkConfig, MqttSourceConfig, RobotBagSourceConfig, + RosSourceConfig, S3SinkConfig, connector_op, }; #[derive(Debug, Clone)] pub enum ConnectorConfig { KafkaSource(KafkaSourceConfig), + MqttSource(MqttSourceConfig), + HttpSource(HttpSourceConfig), + RosSource(RosSourceConfig), + RobotBagSource(RobotBagSourceConfig), KafkaSink(KafkaSinkConfig), FilesystemSink(FilesystemSinkConfig), DeltaSink(DeltaSinkConfig), @@ -30,6 +35,12 @@ impl ConnectorConfig { pub fn to_proto_config(&self) -> connector_op::Config { match self { ConnectorConfig::KafkaSource(cfg) => connector_op::Config::KafkaSource(cfg.clone()), + ConnectorConfig::MqttSource(cfg) => connector_op::Config::MqttSource(cfg.clone()), + ConnectorConfig::HttpSource(cfg) => connector_op::Config::HttpSource(cfg.clone()), + ConnectorConfig::RosSource(cfg) => connector_op::Config::RosSource(cfg.clone()), + ConnectorConfig::RobotBagSource(cfg) => { + connector_op::Config::RobotBagSource(cfg.clone()) + } ConnectorConfig::KafkaSink(cfg) => connector_op::Config::KafkaSink(cfg.clone()), ConnectorConfig::FilesystemSink(cfg) => { connector_op::Config::FilesystemSink(cfg.clone()) @@ -49,6 +60,18 @@ impl PartialEq for ConnectorConfig { (ConnectorConfig::KafkaSource(a), ConnectorConfig::KafkaSource(b)) => { a.encode_to_vec() == b.encode_to_vec() } + (ConnectorConfig::MqttSource(a), ConnectorConfig::MqttSource(b)) => { + a.encode_to_vec() == b.encode_to_vec() + } + (ConnectorConfig::HttpSource(a), ConnectorConfig::HttpSource(b)) => { + a.encode_to_vec() == b.encode_to_vec() + } + (ConnectorConfig::RosSource(a), ConnectorConfig::RosSource(b)) => { + a.encode_to_vec() == b.encode_to_vec() + } + (ConnectorConfig::RobotBagSource(a), ConnectorConfig::RobotBagSource(b)) => { + a.encode_to_vec() == b.encode_to_vec() + } (ConnectorConfig::KafkaSink(a), ConnectorConfig::KafkaSink(b)) => { a.encode_to_vec() == b.encode_to_vec() } @@ -80,6 +103,10 @@ impl std::hash::Hash for ConnectorConfig { std::mem::discriminant(self).hash(state); match self { ConnectorConfig::KafkaSource(cfg) => cfg.encode_to_vec().hash(state), + ConnectorConfig::MqttSource(cfg) => cfg.encode_to_vec().hash(state), + ConnectorConfig::HttpSource(cfg) => cfg.encode_to_vec().hash(state), + ConnectorConfig::RosSource(cfg) => cfg.encode_to_vec().hash(state), + ConnectorConfig::RobotBagSource(cfg) => cfg.encode_to_vec().hash(state), ConnectorConfig::KafkaSink(cfg) => cfg.encode_to_vec().hash(state), ConnectorConfig::FilesystemSink(cfg) => cfg.encode_to_vec().hash(state), ConnectorConfig::DeltaSink(cfg) => cfg.encode_to_vec().hash(state), diff --git a/src/sql/connector/factory.rs b/src/streaming_planner/src/connector/factory.rs similarity index 93% rename from src/sql/connector/factory.rs rename to src/streaming_planner/src/connector/factory.rs index 8c37a15a..e89bafa1 100644 --- a/src/sql/connector/factory.rs +++ b/src/streaming_planner/src/connector/factory.rs @@ -18,9 +18,9 @@ use datafusion::common::Result; use super::config::ConnectorConfig; use super::registry::REGISTRY; use super::sink::runtime_config::SinkRuntimeConfig; -use crate::sql::common::connector_options::ConnectorOptions; -use crate::sql::common::formats::{BadData, Format}; -use crate::sql::schema::table_role::TableRole; +use crate::common::connector_options::ConnectorOptions; +use crate::common::formats::{BadData, Format}; +use crate::schema::table_role::TableRole; pub fn build_connector_config( connector_name: &str, diff --git a/src/sql/connector/mod.rs b/src/streaming_planner/src/connector/mod.rs similarity index 100% rename from src/sql/connector/mod.rs rename to src/streaming_planner/src/connector/mod.rs diff --git a/src/sql/connector/provider.rs b/src/streaming_planner/src/connector/provider.rs similarity index 93% rename from src/sql/connector/provider.rs rename to src/streaming_planner/src/connector/provider.rs index 8875ee0c..83e46aa7 100644 --- a/src/sql/connector/provider.rs +++ b/src/streaming_planner/src/connector/provider.rs @@ -14,8 +14,8 @@ use datafusion::common::{DataFusionError, Result}; use super::config::ConnectorConfig; use super::sink::runtime_config::SinkRuntimeProperties; -use crate::sql::common::connector_options::ConnectorOptions; -use crate::sql::common::formats::{BadData, Format}; +use crate::common::connector_options::ConnectorOptions; +use crate::common::formats::{BadData, Format}; pub trait SourceProvider: Send + Sync { fn name(&self) -> &'static str; diff --git a/src/sql/connector/registry.rs b/src/streaming_planner/src/connector/registry.rs similarity index 84% rename from src/sql/connector/registry.rs rename to src/streaming_planner/src/connector/registry.rs index 4a8a8c1c..b8e2daff 100644 --- a/src/sql/connector/registry.rs +++ b/src/streaming_planner/src/connector/registry.rs @@ -22,7 +22,12 @@ use super::sink::iceberg::IcebergSinkConnector; use super::sink::kafka::KafkaSinkConnector; use super::sink::lancedb::LanceDbSinkConnector; use super::sink::s3::S3SinkConnector; +use super::source::http::HttpSourceConnector; use super::source::kafka::KafkaSourceConnector; +use super::source::mqtt::MqttSourceConnector; +use super::source::robot_bag::RobotBagSourceConnector; +use super::source::ros::RosSourceConnector; +use super::source::ros2::Ros2SourceConnector; pub struct ConnectorRegistry { sources: HashMap>, @@ -37,6 +42,11 @@ impl ConnectorRegistry { }; registry.register_source(Arc::new(KafkaSourceConnector)); + registry.register_source(Arc::new(MqttSourceConnector)); + registry.register_source(Arc::new(HttpSourceConnector)); + registry.register_source(Arc::new(RosSourceConnector)); + registry.register_source(Arc::new(Ros2SourceConnector)); + registry.register_source(Arc::new(RobotBagSourceConnector)); registry.register_sink(Arc::new(KafkaSinkConnector)); registry.register_sink(Arc::new(S3SinkConnector)); diff --git a/src/sql/connector/sink/delta.rs b/src/streaming_planner/src/connector/sink/delta.rs similarity index 83% rename from src/sql/connector/sink/delta.rs rename to src/streaming_planner/src/connector/sink/delta.rs index cd86660d..08d86d96 100644 --- a/src/sql/connector/sink/delta.rs +++ b/src/streaming_planner/src/connector/sink/delta.rs @@ -13,13 +13,13 @@ use datafusion::common::Result; use protocol::function_stream_graph::{DeltaSinkConfig, SinkFormatProto}; -use crate::sql::common::Format; -use crate::sql::common::connector_options::ConnectorOptions; -use crate::sql::common::constants::connector_type; -use crate::sql::connector::config::ConnectorConfig; -use crate::sql::connector::provider::SinkProvider; -use crate::sql::connector::sink::runtime_config::SinkRuntimeProperties; -use crate::sql::connector::sink::utils::SinkUtils; +use crate::common::Format; +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::connector_type; +use crate::connector::config::ConnectorConfig; +use crate::connector::provider::SinkProvider; +use crate::connector::sink::runtime_config::SinkRuntimeProperties; +use crate::connector::sink::utils::SinkUtils; pub struct DeltaSinkConnector; diff --git a/src/sql/connector/sink/filesystem.rs b/src/streaming_planner/src/connector/sink/filesystem.rs similarity index 83% rename from src/sql/connector/sink/filesystem.rs rename to src/streaming_planner/src/connector/sink/filesystem.rs index 224b1805..e529ed79 100644 --- a/src/sql/connector/sink/filesystem.rs +++ b/src/streaming_planner/src/connector/sink/filesystem.rs @@ -13,13 +13,13 @@ use datafusion::common::Result; use protocol::function_stream_graph::{FilesystemSinkConfig, SinkFormatProto}; -use crate::sql::common::Format; -use crate::sql::common::connector_options::ConnectorOptions; -use crate::sql::common::constants::connector_type; -use crate::sql::connector::config::ConnectorConfig; -use crate::sql::connector::provider::SinkProvider; -use crate::sql::connector::sink::runtime_config::SinkRuntimeProperties; -use crate::sql::connector::sink::utils::SinkUtils; +use crate::common::Format; +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::connector_type; +use crate::connector::config::ConnectorConfig; +use crate::connector::provider::SinkProvider; +use crate::connector::sink::runtime_config::SinkRuntimeProperties; +use crate::connector::sink::utils::SinkUtils; pub struct FilesystemSinkConnector; diff --git a/src/sql/connector/sink/iceberg.rs b/src/streaming_planner/src/connector/sink/iceberg.rs similarity index 82% rename from src/sql/connector/sink/iceberg.rs rename to src/streaming_planner/src/connector/sink/iceberg.rs index 12f0d378..4f4854a1 100644 --- a/src/sql/connector/sink/iceberg.rs +++ b/src/streaming_planner/src/connector/sink/iceberg.rs @@ -13,13 +13,13 @@ use datafusion::common::Result; use protocol::function_stream_graph::{IcebergSinkConfig, SinkFormatProto}; -use crate::sql::common::Format; -use crate::sql::common::connector_options::ConnectorOptions; -use crate::sql::common::constants::connector_type; -use crate::sql::connector::config::ConnectorConfig; -use crate::sql::connector::provider::SinkProvider; -use crate::sql::connector::sink::runtime_config::SinkRuntimeProperties; -use crate::sql::connector::sink::utils::SinkUtils; +use crate::common::Format; +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::connector_type; +use crate::connector::config::ConnectorConfig; +use crate::connector::provider::SinkProvider; +use crate::connector::sink::runtime_config::SinkRuntimeProperties; +use crate::connector::sink::utils::SinkUtils; pub struct IcebergSinkConnector; diff --git a/src/sql/connector/sink/kafka.rs b/src/streaming_planner/src/connector/sink/kafka.rs similarity index 94% rename from src/sql/connector/sink/kafka.rs rename to src/streaming_planner/src/connector/sink/kafka.rs index a6fd115c..79292929 100644 --- a/src/sql/connector/sink/kafka.rs +++ b/src/streaming_planner/src/connector/sink/kafka.rs @@ -17,16 +17,16 @@ use protocol::function_stream_graph::{ TimestampFormatProto, format_config, kafka_auth_config, }; -use crate::sql::common::connector_options::ConnectorOptions; -use crate::sql::common::constants::{connector_type, kafka_with_value}; -use crate::sql::common::formats::{ +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::{connector_type, kafka_with_value}; +use crate::common::formats::{ DecimalEncoding as SqlDecimalEncoding, Format as SqlFormat, TimestampFormat as SqlTimestampFormat, }; -use crate::sql::common::with_option_keys as opt; -use crate::sql::connector::config::ConnectorConfig; -use crate::sql::connector::provider::SinkProvider; -use crate::sql::connector::sink::runtime_config::SinkRuntimeProperties; +use crate::common::with_option_keys as opt; +use crate::connector::config::ConnectorConfig; +use crate::connector::provider::SinkProvider; +use crate::connector::sink::runtime_config::SinkRuntimeProperties; pub struct KafkaSinkConnector; diff --git a/src/sql/connector/sink/lancedb.rs b/src/streaming_planner/src/connector/sink/lancedb.rs similarity index 84% rename from src/sql/connector/sink/lancedb.rs rename to src/streaming_planner/src/connector/sink/lancedb.rs index aee79735..87ce99be 100644 --- a/src/sql/connector/sink/lancedb.rs +++ b/src/streaming_planner/src/connector/sink/lancedb.rs @@ -13,13 +13,13 @@ use datafusion::common::Result; use protocol::function_stream_graph::{LanceDbSinkConfig, SinkFormatProto}; -use crate::sql::common::Format; -use crate::sql::common::connector_options::ConnectorOptions; -use crate::sql::common::with_option_keys as opt; -use crate::sql::connector::config::ConnectorConfig; -use crate::sql::connector::provider::SinkProvider; -use crate::sql::connector::sink::runtime_config::SinkRuntimeProperties; -use crate::sql::connector::sink::utils::SinkUtils; +use crate::common::Format; +use crate::common::connector_options::ConnectorOptions; +use crate::common::with_option_keys as opt; +use crate::connector::config::ConnectorConfig; +use crate::connector::provider::SinkProvider; +use crate::connector::sink::runtime_config::SinkRuntimeProperties; +use crate::connector::sink::utils::SinkUtils; pub struct LanceDbSinkConnector; diff --git a/src/sql/connector/sink/mod.rs b/src/streaming_planner/src/connector/sink/mod.rs similarity index 100% rename from src/sql/connector/sink/mod.rs rename to src/streaming_planner/src/connector/sink/mod.rs diff --git a/src/sql/connector/sink/runtime_config.rs b/src/streaming_planner/src/connector/sink/runtime_config.rs similarity index 95% rename from src/sql/connector/sink/runtime_config.rs rename to src/streaming_planner/src/connector/sink/runtime_config.rs index e0ffaeee..b68f7aa9 100644 --- a/src/sql/connector/sink/runtime_config.rs +++ b/src/streaming_planner/src/connector/sink/runtime_config.rs @@ -14,12 +14,12 @@ use std::collections::HashMap; use datafusion::common::{DataFusionError, Result, plan_err}; -use crate::config::global_config::{ +use crate::common::connector_options::ConnectorOptions; +use crate::common::with_option_keys as opt; +use function_stream_config::global_config::{ DEFAULT_OPERATOR_STATE_STORE_MEMORY_BYTES, DEFAULT_SINK_BUFFER_MEMORY_BYTES, }; -use crate::config::streaming_job::DEFAULT_CHECKPOINT_INTERVAL_MS; -use crate::sql::common::connector_options::ConnectorOptions; -use crate::sql::common::with_option_keys as opt; +use function_stream_config::streaming_job::DEFAULT_CHECKPOINT_INTERVAL_MS; #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct SinkRuntimeConfig { diff --git a/src/sql/connector/sink/s3.rs b/src/streaming_planner/src/connector/sink/s3.rs similarity index 84% rename from src/sql/connector/sink/s3.rs rename to src/streaming_planner/src/connector/sink/s3.rs index 5d04ce46..51cf85bf 100644 --- a/src/sql/connector/sink/s3.rs +++ b/src/streaming_planner/src/connector/sink/s3.rs @@ -13,14 +13,14 @@ use datafusion::common::Result; use protocol::function_stream_graph::{S3SinkConfig, SinkFormatProto}; -use crate::sql::common::Format; -use crate::sql::common::connector_options::ConnectorOptions; -use crate::sql::common::constants::connector_type; -use crate::sql::common::with_option_keys as opt; -use crate::sql::connector::config::ConnectorConfig; -use crate::sql::connector::provider::SinkProvider; -use crate::sql::connector::sink::runtime_config::SinkRuntimeProperties; -use crate::sql::connector::sink::utils::SinkUtils; +use crate::common::Format; +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::connector_type; +use crate::common::with_option_keys as opt; +use crate::connector::config::ConnectorConfig; +use crate::connector::provider::SinkProvider; +use crate::connector::sink::runtime_config::SinkRuntimeProperties; +use crate::connector::sink::utils::SinkUtils; pub struct S3SinkConnector; diff --git a/src/sql/connector/sink/utils.rs b/src/streaming_planner/src/connector/sink/utils.rs similarity index 94% rename from src/sql/connector/sink/utils.rs rename to src/streaming_planner/src/connector/sink/utils.rs index e61cd870..481ad3b2 100644 --- a/src/sql/connector/sink/utils.rs +++ b/src/streaming_planner/src/connector/sink/utils.rs @@ -13,10 +13,10 @@ use datafusion::common::{DataFusionError, Result, plan_err}; use protocol::function_stream_graph::{ParquetCompressionProto, SinkFormatProto}; -use crate::sql::common::Format; -use crate::sql::common::connector_options::ConnectorOptions; -use crate::sql::common::constants::parquet_compression_value; -use crate::sql::common::with_option_keys as opt; +use crate::common::Format; +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::parquet_compression_value; +use crate::common::with_option_keys as opt; pub struct SinkUtils; diff --git a/src/streaming_planner/src/connector/source/http.rs b/src/streaming_planner/src/connector/source/http.rs new file mode 100644 index 00000000..eef29123 --- /dev/null +++ b/src/streaming_planner/src/connector/source/http.rs @@ -0,0 +1,258 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; + +use datafusion::common::{Result, plan_datafusion_err, plan_err}; +use protocol::function_stream_graph::{ + BadDataPolicy, DecimalEncodingProto, FormatConfig, HttpResponseSplit, HttpSourceConfig, + HttpSourceMode, JsonFormatConfig, RawBytesFormatConfig, RawStringFormatConfig, + TimestampFormatProto, format_config, +}; + +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::connector_type; +use crate::common::formats::{ + BadData, DecimalEncoding as SqlDecimalEncoding, Format as SqlFormat, + TimestampFormat as SqlTimestampFormat, +}; +use crate::common::with_option_keys as opt; +use crate::connector::config::ConnectorConfig; +use crate::connector::provider::SourceProvider; + +const DEFAULT_SCAN_INTERVAL_MS: u64 = 1000; +const DEFAULT_REQUEST_TIMEOUT_MS: u32 = 30_000; +const DEFAULT_WEBHOOK_HOST: &str = "0.0.0.0"; +const DEFAULT_WEBHOOK_PATH: &str = "/"; + +pub struct HttpSourceConnector; + +impl HttpSourceConnector { + fn sql_format_to_proto(fmt: &SqlFormat) -> Result { + match fmt { + SqlFormat::Json(j) => Ok(FormatConfig { + format: Some(format_config::Format::Json(JsonFormatConfig { + timestamp_format: match j.timestamp_format { + SqlTimestampFormat::RFC3339 => { + TimestampFormatProto::TimestampRfc3339 as i32 + } + SqlTimestampFormat::UnixMillis => { + TimestampFormatProto::TimestampUnixMillis as i32 + } + }, + decimal_encoding: match j.decimal_encoding { + SqlDecimalEncoding::Number => DecimalEncodingProto::DecimalNumber as i32, + SqlDecimalEncoding::String => DecimalEncodingProto::DecimalString as i32, + SqlDecimalEncoding::Bytes => DecimalEncodingProto::DecimalBytes as i32, + }, + include_schema: j.include_schema, + confluent_schema_registry: j.confluent_schema_registry, + schema_id: j.schema_id, + debezium: j.debezium, + unstructured: j.unstructured, + })), + }), + SqlFormat::RawString(_) => Ok(FormatConfig { + format: Some(format_config::Format::RawString(RawStringFormatConfig {})), + }), + SqlFormat::RawBytes(_) => Ok(FormatConfig { + format: Some(format_config::Format::RawBytes(RawBytesFormatConfig {})), + }), + other => plan_err!( + "HTTP source connector: format '{}' is not supported", + other.name() + ), + } + } + + fn bad_data_to_proto(bad: &BadData) -> i32 { + match bad { + BadData::Fail {} => BadDataPolicy::BadDataFail as i32, + BadData::Drop {} => BadDataPolicy::BadDataDrop as i32, + } + } + + fn pull_scan_interval_ms(options: &mut ConnectorOptions) -> Result { + if let Some(d) = options.pull_opt_duration(opt::HTTP_SCAN_INTERVAL)? { + return Ok(d.as_millis() as u64); + } + if let Some(ms) = options.pull_opt_u64(opt::HTTP_SCAN_INTERVAL_MS)? { + return Ok(ms.max(1)); + } + if let Some(ms) = options.pull_opt_u64(opt::HTTP_POLL_INTERVAL_MS)? { + return Ok(ms.max(1)); + } + Ok(DEFAULT_SCAN_INTERVAL_MS) + } + + fn pull_method(options: &mut ConnectorOptions) -> Result { + if let Some(m) = options.pull_opt_str(opt::HTTP_METHOD)? { + return Ok(m.to_ascii_uppercase()); + } + if let Some(m) = options.pull_opt_str(opt::HTTP_LOOKUP_METHOD)? { + return Ok(m.to_ascii_uppercase()); + } + Ok("GET".to_string()) + } + + fn pull_response_split(options: &mut ConnectorOptions) -> Result { + let split = options + .pull_opt_str(opt::HTTP_RESPONSE_SPLIT)? + .map(|s| s.to_ascii_lowercase()); + Ok(match split.as_deref() { + Some("ndjson") | Some("newline") => HttpResponseSplit::HttpResponseNdjson as i32, + Some("json-array") | Some("array") => HttpResponseSplit::HttpResponseJsonArray as i32, + Some("single") | None => HttpResponseSplit::HttpResponseSingle as i32, + Some(other) => { + return plan_err!( + "invalid http.response.split '{other}'; expected single, ndjson, or json-array" + ); + } + }) + } + + fn extract_headers(client_configs: &mut HashMap) -> HashMap { + let mut headers = HashMap::new(); + let keys: Vec = client_configs + .keys() + .filter(|k| k.starts_with(opt::HTTP_HEADER_PREFIX)) + .cloned() + .collect(); + for key in keys { + if let Some(v) = client_configs.remove(&key) { + let name = key[opt::HTTP_HEADER_PREFIX.len()..].to_string(); + headers.insert(name, v); + } + } + headers + } + + fn resolve_mode( + options: &mut ConnectorOptions, + listen_port: Option, + url: Option, + ) -> Result { + if let Some(mode) = options.pull_opt_str(opt::HTTP_MODE)? { + return match mode.to_ascii_lowercase().as_str() { + "poll" | "polling" => Ok(HttpSourceMode::HttpSourcePoll), + "webhook" | "push" => Ok(HttpSourceMode::HttpSourceWebhook), + other => plan_err!("invalid http mode '{other}'; expected poll or webhook"), + }; + } + if listen_port.is_some() { + return Ok(HttpSourceMode::HttpSourceWebhook); + } + if url.is_some() { + return Ok(HttpSourceMode::HttpSourcePoll); + } + Err(plan_datafusion_err!( + "HTTP source requires 'url' (poll) or 'http.listen.port' (webhook)" + )) + } +} + +impl SourceProvider for HttpSourceConnector { + fn name(&self) -> &'static str { + connector_type::HTTP + } + + fn build_source_config( + &self, + options: &mut ConnectorOptions, + format: &Option, + bad_data: BadData, + ) -> Result { + let url = options.pull_opt_str(opt::HTTP_URL)?; + let listen_port = options.pull_opt_u64(opt::HTTP_LISTEN_PORT)?.map(|p| p as u32); + let mode = Self::resolve_mode(options, listen_port, url.clone())?; + + let sql_format = format.as_ref().ok_or_else(|| { + plan_datafusion_err!( + "HTTP source requires 'format' in the WITH clause (e.g. format = 'json')" + ) + })?; + let proto_format = Self::sql_format_to_proto(sql_format)?; + + let rate_limit = options + .pull_opt_u64(opt::MQTT_RATE_LIMIT_MESSAGES_PER_SECOND)? + .map(|v| v.clamp(0, u32::MAX as u64) as u32) + .unwrap_or(0); + + let scan_interval_ms = Self::pull_scan_interval_ms(options)?; + let method = Self::pull_method(options)?; + let response_split = Self::pull_response_split(options)?; + + let request_body = match options.pull_opt_str(opt::HTTP_REQUEST_BODY)? { + Some(s) => Some(s), + None => options.pull_opt_str(opt::HTTP_REQUEST_BODY_LEGACY)?, + }; + + let listen_host = options + .pull_opt_str(opt::HTTP_LISTEN_HOST)? + .unwrap_or_else(|| DEFAULT_WEBHOOK_HOST.to_string()); + + let webhook_path = options + .pull_opt_str(opt::HTTP_WEBHOOK_PATH)? + .unwrap_or_else(|| DEFAULT_WEBHOOK_PATH.to_string()); + + let request_timeout_ms = options + .pull_opt_u64(opt::HTTP_REQUEST_TIMEOUT_MS)? + .map(|v| v.clamp(1, u32::MAX as u64) as u32) + .unwrap_or(DEFAULT_REQUEST_TIMEOUT_MS); + + let _ = options.pull_opt_str(opt::TYPE)?; + let _ = options.pull_opt_str(opt::CONNECTOR)?; + + let mut client_configs = options.drain_remaining_string_values()?; + client_configs.remove(opt::CHECKPOINT_INTERVAL_MS); + client_configs.remove(opt::PIPELINE_PARALLELISM); + client_configs.remove(opt::KEY_BY_PARALLELISM); + client_configs.remove(opt::FORMAT); + + let headers = Self::extract_headers(&mut client_configs); + + let (url, listen_port) = match mode { + HttpSourceMode::HttpSourcePoll => { + let url = url.ok_or_else(|| { + plan_datafusion_err!("HTTP poll mode requires 'url' in the WITH clause") + })?; + (url, listen_port.unwrap_or(0)) + } + HttpSourceMode::HttpSourceWebhook => { + let port = listen_port.ok_or_else(|| { + plan_datafusion_err!( + "HTTP webhook mode requires 'http.listen.port' in the WITH clause" + ) + })?; + (String::new(), port) + } + }; + + Ok(ConnectorConfig::HttpSource(HttpSourceConfig { + mode: mode as i32, + url, + method, + scan_interval_ms, + request_body, + headers, + listen_host, + listen_port, + webhook_path, + format: Some(proto_format), + bad_data_policy: Self::bad_data_to_proto(&bad_data), + rate_limit_msgs_per_sec: rate_limit, + client_configs, + request_timeout_ms, + response_split, + })) + } +} diff --git a/src/sql/connector/source/kafka.rs b/src/streaming_planner/src/connector/source/kafka.rs similarity index 68% rename from src/sql/connector/source/kafka.rs rename to src/streaming_planner/src/connector/source/kafka.rs index 0bc220d9..f735e56d 100644 --- a/src/sql/connector/source/kafka.rs +++ b/src/streaming_planner/src/connector/source/kafka.rs @@ -17,18 +17,49 @@ use protocol::function_stream_graph::{ RawStringFormatConfig, TimestampFormatProto, format_config, kafka_auth_config, }; -use crate::sql::common::connector_options::ConnectorOptions; -use crate::sql::common::constants::{connector_type, kafka_with_value}; -use crate::sql::common::formats::{ +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::{connector_type, kafka_with_value}; +use crate::common::formats::{ BadData, DecimalEncoding as SqlDecimalEncoding, Format as SqlFormat, TimestampFormat as SqlTimestampFormat, }; -use crate::sql::common::with_option_keys as opt; -use crate::sql::connector::config::ConnectorConfig; -use crate::sql::connector::provider::SourceProvider; +use crate::common::with_option_keys as opt; +use crate::connector::config::ConnectorConfig; +use crate::connector::provider::SourceProvider; pub struct KafkaSourceConnector; +pub fn default_kafka_consumer_group_id(table_name: &str) -> String { + let sanitized: String = table_name + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '.' || c == '_' || c == '-' { + c + } else { + '-' + } + }) + .collect(); + format!("fs-{sanitized}-consumer") +} + +pub fn ensure_default_consumer_group( + table_name: &str, + options: &mut ConnectorOptions, +) -> Result<()> { + let has_group = options.peek_opt_str(opt::KAFKA_GROUP_ID)?.is_some() + || options.peek_opt_str(opt::KAFKA_GROUP_ID_LEGACY)?.is_some(); + let has_prefix = options.peek_opt_str(opt::KAFKA_GROUP_ID_PREFIX)?.is_some(); + if has_group || has_prefix { + return Ok(()); + } + options.insert_str( + opt::KAFKA_GROUP_ID, + default_kafka_consumer_group_id(table_name), + )?; + Ok(()) +} + impl KafkaSourceConnector { fn sql_format_to_proto(fmt: &SqlFormat) -> Result { match fmt { @@ -108,11 +139,6 @@ impl SourceProvider for KafkaSourceConnector { })?; let proto_format = Self::sql_format_to_proto(sql_format)?; - let rate_limit = options - .pull_opt_u64(opt::KAFKA_RATE_LIMIT_MESSAGES_PER_SECOND)? - .map(|v| v.clamp(1, u32::MAX as u64) as u32) - .unwrap_or(0); - let value_subject = options.pull_opt_str(opt::KAFKA_VALUE_SUBJECT)?; let offset_mode = match options @@ -178,8 +204,68 @@ impl SourceProvider for KafkaSourceConnector { client_configs, format: Some(proto_format), bad_data_policy: Self::bad_data_to_proto(&bad_data), - rate_limit_msgs_per_sec: rate_limit, + rate_limit_msgs_per_sec: 0, value_subject, })) } } + +#[cfg(test)] +mod tests { + use super::*; + use datafusion::sql::sqlparser::ast::{Expr, Ident, SqlOption, Value as SqlValue}; + + fn options_from_map(pairs: &[(&str, &str)]) -> ConnectorOptions { + let opts: Vec = pairs + .iter() + .map(|(k, v)| SqlOption::KeyValue { + key: Ident::new(*k), + value: Expr::Value( + SqlValue::SingleQuotedString((*v).to_string()).with_empty_span(), + ), + }) + .collect(); + ConnectorOptions::new(&opts, &None).expect("options") + } + + #[test] + fn assigns_default_group_id_when_missing() { + let mut options = options_from_map(&[ + ("connector", "kafka"), + ("topic", "events"), + ("bootstrap.servers", "localhost:9092"), + ]); + ensure_default_consumer_group("my_events", &mut options).expect("ensure"); + assert_eq!( + options.peek_opt_str(opt::KAFKA_GROUP_ID).expect("peek"), + Some("fs-my_events-consumer".to_string()) + ); + } + + #[test] + fn keeps_user_group_id() { + let mut options = options_from_map(&[ + ("connector", "kafka"), + ("topic", "events"), + ("bootstrap.servers", "localhost:9092"), + ("group.id", "custom-group"), + ]); + ensure_default_consumer_group("my_events", &mut options).expect("ensure"); + assert_eq!( + options.peek_opt_str(opt::KAFKA_GROUP_ID).expect("peek"), + Some("custom-group".to_string()) + ); + } + + #[test] + fn skips_when_group_id_prefix_set() { + let mut options = options_from_map(&[ + ("connector", "kafka"), + ("topic", "events"), + ("bootstrap.servers", "localhost:9092"), + ("group.id.prefix", "prefix"), + ]); + ensure_default_consumer_group("my_events", &mut options).expect("ensure"); + assert!(options.peek_opt_str(opt::KAFKA_GROUP_ID).expect("peek").is_none()); + } +} diff --git a/src/sql/connector/source/mod.rs b/src/streaming_planner/src/connector/source/mod.rs similarity index 85% rename from src/sql/connector/source/mod.rs rename to src/streaming_planner/src/connector/source/mod.rs index b9574391..4f20c39c 100644 --- a/src/sql/connector/source/mod.rs +++ b/src/streaming_planner/src/connector/source/mod.rs @@ -10,4 +10,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod http; pub mod kafka; +pub mod mqtt; +pub mod robot_bag; +pub mod ros; +pub mod ros2; +pub mod ros_common; diff --git a/src/streaming_planner/src/connector/source/mqtt.rs b/src/streaming_planner/src/connector/source/mqtt.rs new file mode 100644 index 00000000..6dd21349 --- /dev/null +++ b/src/streaming_planner/src/connector/source/mqtt.rs @@ -0,0 +1,206 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use datafusion::common::{Result, plan_datafusion_err, plan_err}; +use protocol::function_stream_graph::{ + BadDataPolicy, DecimalEncodingProto, FormatConfig, JsonFormatConfig, MqttSourceConfig, + RawBytesFormatConfig, RawStringFormatConfig, TimestampFormatProto, format_config, +}; + +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::connector_type; +use crate::common::formats::{ + BadData, DecimalEncoding as SqlDecimalEncoding, Format as SqlFormat, + TimestampFormat as SqlTimestampFormat, +}; +use crate::common::with_option_keys as opt; +use crate::connector::config::ConnectorConfig; +use crate::connector::provider::SourceProvider; + +const DEFAULT_MQTT_PORT: u32 = 1883; +const DEFAULT_MQTT_QOS: u32 = 1; +const DEFAULT_MQTT_KEEP_ALIVE_SECS: u32 = 60; + +pub struct MqttSourceConnector; + +impl MqttSourceConnector { + fn sql_format_to_proto(fmt: &SqlFormat) -> Result { + match fmt { + SqlFormat::Json(j) => Ok(FormatConfig { + format: Some(format_config::Format::Json(JsonFormatConfig { + timestamp_format: match j.timestamp_format { + SqlTimestampFormat::RFC3339 => { + TimestampFormatProto::TimestampRfc3339 as i32 + } + SqlTimestampFormat::UnixMillis => { + TimestampFormatProto::TimestampUnixMillis as i32 + } + }, + decimal_encoding: match j.decimal_encoding { + SqlDecimalEncoding::Number => DecimalEncodingProto::DecimalNumber as i32, + SqlDecimalEncoding::String => DecimalEncodingProto::DecimalString as i32, + SqlDecimalEncoding::Bytes => DecimalEncodingProto::DecimalBytes as i32, + }, + include_schema: j.include_schema, + confluent_schema_registry: j.confluent_schema_registry, + schema_id: j.schema_id, + debezium: j.debezium, + unstructured: j.unstructured, + })), + }), + SqlFormat::RawString(_) => Ok(FormatConfig { + format: Some(format_config::Format::RawString(RawStringFormatConfig {})), + }), + SqlFormat::RawBytes(_) => Ok(FormatConfig { + format: Some(format_config::Format::RawBytes(RawBytesFormatConfig {})), + }), + other => plan_err!( + "MQTT source connector: format '{}' is not supported", + other.name() + ), + } + } + + fn bad_data_to_proto(bad: &BadData) -> i32 { + match bad { + BadData::Fail {} => BadDataPolicy::BadDataFail as i32, + BadData::Drop {} => BadDataPolicy::BadDataDrop as i32, + } + } + + fn pull_host(options: &mut ConnectorOptions) -> Result { + match options.pull_opt_str(opt::MQTT_HOST)? { + Some(s) => Ok(s), + None => options.pull_opt_str(opt::MQTT_HOST_LEGACY)?.ok_or_else(|| { + plan_datafusion_err!( + "MQTT connector requires 'mqtt.host' (or 'host') in the WITH clause" + ) + }), + } + } + + fn pull_port(options: &mut ConnectorOptions) -> Result { + if let Some(p) = options.pull_opt_u64(opt::MQTT_PORT)? { + return Ok(p.clamp(1, u16::MAX as u64) as u32); + } + if let Some(p) = options.pull_opt_u64(opt::MQTT_PORT_LEGACY)? { + return Ok(p.clamp(1, u16::MAX as u64) as u32); + } + Ok(DEFAULT_MQTT_PORT) + } + + fn pull_qos(options: &mut ConnectorOptions) -> Result { + let qos = match options.pull_opt_u64(opt::MQTT_QOS)? { + Some(v) => v, + None => options + .pull_opt_u64(opt::MQTT_QOS_LEGACY)? + .unwrap_or(DEFAULT_MQTT_QOS as u64), + }; + Ok(qos.clamp(0, 2) as u32) + } + + fn pull_clean_session(options: &mut ConnectorOptions) -> Result { + if let Some(v) = options.pull_opt_bool(opt::MQTT_CLEAN_SESSION)? { + return Ok(v); + } + if let Some(v) = options.pull_opt_bool(opt::MQTT_CLEAN_SESSION_LEGACY)? { + return Ok(v); + } + Ok(true) + } + + fn pull_keep_alive_secs(options: &mut ConnectorOptions) -> Result { + if let Some(v) = options.pull_opt_u64(opt::MQTT_KEEP_ALIVE_SECS)? { + return Ok(v.max(1) as u32); + } + if let Some(v) = options.pull_opt_u64(opt::MQTT_KEEP_ALIVE_SECS_LEGACY)? { + return Ok(v.max(1) as u32); + } + Ok(DEFAULT_MQTT_KEEP_ALIVE_SECS) + } +} + +impl SourceProvider for MqttSourceConnector { + fn name(&self) -> &'static str { + connector_type::MQTT + } + + fn build_source_config( + &self, + options: &mut ConnectorOptions, + format: &Option, + bad_data: BadData, + ) -> Result { + let host = Self::pull_host(options)?; + let port = Self::pull_port(options)?; + + let topic = options.pull_opt_str(opt::MQTT_TOPIC)?.ok_or_else(|| { + plan_datafusion_err!("MQTT connector requires 'topic' in the WITH clause") + })?; + + let sql_format = format.as_ref().ok_or_else(|| { + plan_datafusion_err!( + "MQTT source requires 'format' in the WITH clause (e.g. format = 'json')" + ) + })?; + let proto_format = Self::sql_format_to_proto(sql_format)?; + + let rate_limit = options + .pull_opt_u64(opt::MQTT_RATE_LIMIT_MESSAGES_PER_SECOND)? + .map(|v| v.clamp(0, u32::MAX as u64) as u32) + .unwrap_or(0); + + let client_id = match options.pull_opt_str(opt::MQTT_CLIENT_ID)? { + Some(s) => Some(s), + None => options.pull_opt_str(opt::MQTT_CLIENT_ID_LEGACY)?, + }; + + let username = match options.pull_opt_str(opt::MQTT_USERNAME)? { + Some(s) => Some(s), + None => options.pull_opt_str(opt::MQTT_USERNAME_LEGACY)?, + }; + + let password = match options.pull_opt_str(opt::MQTT_PASSWORD)? { + Some(s) => Some(s), + None => options.pull_opt_str(opt::MQTT_PASSWORD_LEGACY)?, + }; + + let qos = Self::pull_qos(options)?; + let clean_session = Self::pull_clean_session(options)?; + let keep_alive_secs = Self::pull_keep_alive_secs(options)?; + + let _ = options.pull_opt_str(opt::TYPE)?; + let _ = options.pull_opt_str(opt::CONNECTOR)?; + + let mut client_configs = options.drain_remaining_string_values()?; + client_configs.remove(opt::CHECKPOINT_INTERVAL_MS); + client_configs.remove(opt::PIPELINE_PARALLELISM); + client_configs.remove(opt::KEY_BY_PARALLELISM); + client_configs.remove(opt::FORMAT); + + Ok(ConnectorConfig::MqttSource(MqttSourceConfig { + topic, + host, + port, + client_id, + username, + password, + qos, + clean_session, + keep_alive_secs, + format: Some(proto_format), + bad_data_policy: Self::bad_data_to_proto(&bad_data), + rate_limit_msgs_per_sec: rate_limit, + client_configs, + })) + } +} diff --git a/src/streaming_planner/src/connector/source/robot_bag.rs b/src/streaming_planner/src/connector/source/robot_bag.rs new file mode 100644 index 00000000..0f4c7eac --- /dev/null +++ b/src/streaming_planner/src/connector/source/robot_bag.rs @@ -0,0 +1,184 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io::Read; +use std::path::Path; + +use datafusion::common::{Result, plan_datafusion_err, plan_err}; +use protocol::function_stream_graph::{ + RobotBagFormat, RobotBagSourceConfig, +}; + +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::connector_type; +use crate::common::formats::{BadData, Format as SqlFormat}; +use crate::common::with_option_keys as opt; +use crate::connector::config::ConnectorConfig; +use crate::connector::provider::SourceProvider; + +use super::ros_common::{bad_data_to_proto, sql_format_to_proto}; + +const DEFAULT_REPLAY_INTERVAL_MS: u64 = 0; + +pub struct RobotBagSourceConnector; + +impl RobotBagSourceConnector { + fn directory_has_pcd_files(dir: &Path) -> bool { + std::fs::read_dir(dir) + .ok() + .into_iter() + .flatten() + .filter_map(|e| e.ok()) + .any(|e| { + e.path() + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("pcd")) + }) + } + + fn is_ros1_bag_file(path: &Path) -> bool { + let Ok(mut file) = std::fs::File::open(path) else { + return false; + }; + let mut buf = [0u8; 13]; + let Ok(n) = file.read(&mut buf) else { + return false; + }; + if n < 13 { + return false; + } + std::str::from_utf8(&buf).ok().is_some_and(|s| { + s.starts_with("#ROSBAG V2.0") || s.starts_with("#ROSBAG V1.02") || s.starts_with("#ROSBAG V1.0") + }) + } + + fn detect_format(path: &str) -> i32 { + let p = Path::new(path); + if p.is_dir() { + if Self::directory_has_pcd_files(p) { + return RobotBagFormat::RobotBagPcd as i32; + } + return RobotBagFormat::RobotBagRos2 as i32; + } + if Self::is_ros1_bag_file(p) { + return RobotBagFormat::RobotBagRos1 as i32; + } + match p.extension().and_then(|e| e.to_str()).map(|s| s.to_ascii_lowercase()) { + Some(ext) if ext == "mcap" => RobotBagFormat::RobotBagMcap as i32, + Some(ext) if ext == "db3" => RobotBagFormat::RobotBagRos2 as i32, + Some(ext) if ext == "bag" => RobotBagFormat::RobotBagRos1 as i32, + Some(ext) if ext == "pcd" => RobotBagFormat::RobotBagPcd as i32, + Some(ext) if ext == "jsonl" || ext == "ndjson" => RobotBagFormat::RobotBagJsonl as i32, + Some(ext) if ext == "csv" => RobotBagFormat::RobotBagCsv as i32, + _ => RobotBagFormat::RobotBagAuto as i32, + } + } + + fn parse_bag_format(options: &mut ConnectorOptions, path: &str) -> Result { + let fmt = options + .pull_opt_str(opt::ROBOT_BAG_FORMAT)? + .map(|s| s.to_ascii_lowercase()); + Ok(match fmt.as_deref() { + None => Self::detect_format(path), + Some("auto") => Self::detect_format(path), + Some("mcap") => RobotBagFormat::RobotBagMcap as i32, + Some("ros2" | "db3" | "rosbag2") => RobotBagFormat::RobotBagRos2 as i32, + Some("ros1" | "bag" | "rosbag") => RobotBagFormat::RobotBagRos1 as i32, + Some("pcd" | "pointcloud") => RobotBagFormat::RobotBagPcd as i32, + Some("jsonl" | "ndjson" | "json") => RobotBagFormat::RobotBagJsonl as i32, + Some("csv") => RobotBagFormat::RobotBagCsv as i32, + Some(other) => { + return plan_err!( + "invalid bag.format '{other}'; expected auto, mcap, ros1, ros2, pcd, jsonl, or csv" + ); + } + }) + } +} + +impl SourceProvider for RobotBagSourceConnector { + fn name(&self) -> &'static str { + connector_type::ROBOT_BAG + } + + fn build_source_config( + &self, + options: &mut ConnectorOptions, + format: &Option, + bad_data: BadData, + ) -> Result { + let path = options.pull_opt_str(opt::ROBOT_BAG_PATH)?; + let path = if let Some(p) = path { + p + } else { + options.pull_opt_str(opt::PATH)?.ok_or_else(|| { + plan_datafusion_err!("robot-bag source requires 'path' in the WITH clause") + })? + }; + + let bag_format = Self::parse_bag_format(options, &path)?; + + let topic = if let Some(t) = options.pull_opt_str(opt::ROBOT_BAG_TOPIC)? { + Some(t) + } else { + options.pull_opt_str(opt::ROS_TOPIC)? + }; + + let replay_interval_ms = options + .pull_opt_u64(opt::ROBOT_BAG_REPLAY_INTERVAL_MS)? + .unwrap_or(DEFAULT_REPLAY_INTERVAL_MS); + + let loop_replay = options + .pull_opt_bool(opt::ROBOT_BAG_LOOP)? + .unwrap_or(false); + + let pcd_emit_mode = options + .pull_opt_str(opt::ROBOT_BAG_PCD_EMIT_MODE)? + .unwrap_or_else(|| "point".to_string()); + match pcd_emit_mode.to_ascii_lowercase().as_str() { + "point" | "points" | "cloud" | "file" => {} + other => { + return plan_err!( + "invalid pcd.emit.mode '{other}'; expected 'point' or 'cloud'" + ); + } + } + + let sql_format = format.as_ref().ok_or_else(|| { + plan_datafusion_err!( + "robot-bag source requires 'format' (json / raw_bytes / raw_string)" + ) + })?; + let proto_format = sql_format_to_proto(sql_format)?; + + let _ = options.pull_opt_str(opt::TYPE)?; + let _ = options.pull_opt_str(opt::CONNECTOR)?; + + let mut client_configs = options.drain_remaining_string_values()?; + client_configs.remove(opt::CHECKPOINT_INTERVAL_MS); + client_configs.remove(opt::PIPELINE_PARALLELISM); + client_configs.remove(opt::KEY_BY_PARALLELISM); + client_configs.remove(opt::FORMAT); + + Ok(ConnectorConfig::RobotBagSource(RobotBagSourceConfig { + path, + bag_format, + topic, + replay_interval_ms, + loop_replay, + format: Some(proto_format), + bad_data_policy: bad_data_to_proto(&bad_data), + client_configs, + pcd_emit_mode, + })) + } +} diff --git a/src/streaming_planner/src/connector/source/ros.rs b/src/streaming_planner/src/connector/source/ros.rs new file mode 100644 index 00000000..dd7363cc --- /dev/null +++ b/src/streaming_planner/src/connector/source/ros.rs @@ -0,0 +1,44 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use datafusion::common::Result; +use protocol::function_stream_graph::RosVersion; + +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::connector_type; +use crate::common::formats::{BadData, Format}; +use crate::connector::provider::SourceProvider; + +use super::ros_common::{DEFAULT_ROS_URL, build_ros_source_config}; + +pub struct RosSourceConnector; + +impl SourceProvider for RosSourceConnector { + fn name(&self) -> &'static str { + connector_type::ROS + } + + fn build_source_config( + &self, + options: &mut ConnectorOptions, + format: &Option, + bad_data: BadData, + ) -> Result { + build_ros_source_config( + options, + format, + bad_data, + RosVersion::Ros1, + DEFAULT_ROS_URL, + ) + } +} diff --git a/src/streaming_planner/src/connector/source/ros2.rs b/src/streaming_planner/src/connector/source/ros2.rs new file mode 100644 index 00000000..0a4aa62b --- /dev/null +++ b/src/streaming_planner/src/connector/source/ros2.rs @@ -0,0 +1,45 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use datafusion::common::Result; +use protocol::function_stream_graph::RosVersion; + +use crate::common::connector_options::ConnectorOptions; +use crate::common::constants::connector_type; +use crate::common::formats::{BadData, Format}; +use crate::connector::provider::SourceProvider; + +use super::ros_common::{DEFAULT_ROS2_URL, build_ros_source_config}; + +/// ROS2 live source via `rosbridge_server` (same WebSocket protocol as ROS1). +pub struct Ros2SourceConnector; + +impl SourceProvider for Ros2SourceConnector { + fn name(&self) -> &'static str { + connector_type::ROS2 + } + + fn build_source_config( + &self, + options: &mut ConnectorOptions, + format: &Option, + bad_data: BadData, + ) -> Result { + build_ros_source_config( + options, + format, + bad_data, + RosVersion::Ros2, + DEFAULT_ROS2_URL, + ) + } +} diff --git a/src/streaming_planner/src/connector/source/ros_common.rs b/src/streaming_planner/src/connector/source/ros_common.rs new file mode 100644 index 00000000..b470e6e6 --- /dev/null +++ b/src/streaming_planner/src/connector/source/ros_common.rs @@ -0,0 +1,133 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use datafusion::common::{Result, plan_datafusion_err, plan_err}; +use protocol::function_stream_graph::{ + BadDataPolicy, DecimalEncodingProto, FormatConfig, JsonFormatConfig, RawBytesFormatConfig, + RawStringFormatConfig, RosSourceConfig, RosVersion, TimestampFormatProto, format_config, +}; + +use crate::common::connector_options::ConnectorOptions; +use crate::common::formats::{ + BadData, DecimalEncoding as SqlDecimalEncoding, Format as SqlFormat, + TimestampFormat as SqlTimestampFormat, +}; +use crate::common::with_option_keys as opt; +use crate::connector::config::ConnectorConfig; + +pub(crate) const DEFAULT_ROS_URL: &str = "ws://127.0.0.1:9090"; +pub(crate) const DEFAULT_ROS2_URL: &str = "ws://127.0.0.1:9090"; +pub(crate) const DEFAULT_ROS_MESSAGE_FIELD: &str = "msg"; + +pub(crate) fn sql_format_to_proto(fmt: &SqlFormat) -> Result { + match fmt { + SqlFormat::Json(j) => Ok(FormatConfig { + format: Some(format_config::Format::Json(JsonFormatConfig { + timestamp_format: match j.timestamp_format { + SqlTimestampFormat::RFC3339 => TimestampFormatProto::TimestampRfc3339 as i32, + SqlTimestampFormat::UnixMillis => TimestampFormatProto::TimestampUnixMillis as i32, + }, + decimal_encoding: match j.decimal_encoding { + SqlDecimalEncoding::Number => DecimalEncodingProto::DecimalNumber as i32, + SqlDecimalEncoding::String => DecimalEncodingProto::DecimalString as i32, + SqlDecimalEncoding::Bytes => DecimalEncodingProto::DecimalBytes as i32, + }, + include_schema: j.include_schema, + confluent_schema_registry: j.confluent_schema_registry, + schema_id: j.schema_id, + debezium: j.debezium, + unstructured: j.unstructured, + })), + }), + SqlFormat::RawString(_) => Ok(FormatConfig { + format: Some(format_config::Format::RawString(RawStringFormatConfig {})), + }), + SqlFormat::RawBytes(_) => Ok(FormatConfig { + format: Some(format_config::Format::RawBytes(RawBytesFormatConfig {})), + }), + other => plan_err!( + "ROS source connector: format '{}' is not supported", + other.name() + ), + } +} + +pub(crate) fn bad_data_to_proto(bad: &BadData) -> i32 { + match bad { + BadData::Fail {} => BadDataPolicy::BadDataFail as i32, + BadData::Drop {} => BadDataPolicy::BadDataDrop as i32, + } +} + +pub(crate) fn build_ros_source_config( + options: &mut ConnectorOptions, + format: &Option, + bad_data: BadData, + default_version: RosVersion, + default_url: &str, +) -> Result { + let url = match options.pull_opt_str(opt::ROS_URL)? { + Some(s) => s, + None => options + .pull_opt_str(opt::ROS_URL_LEGACY)? + .unwrap_or_else(|| default_url.to_string()), + }; + + let topic = match options.pull_opt_str(opt::ROS_TOPIC)? { + Some(s) => s, + None => options.pull_opt_str(opt::ROS_TOPIC_LEGACY)?.ok_or_else(|| { + plan_datafusion_err!("ROS source requires 'topic' (ROS topic name) in the WITH clause") + })?, + }; + + let message_field = match options.pull_opt_str(opt::ROS_MESSAGE_FIELD)? { + Some(s) => s, + None => options + .pull_opt_str(opt::ROS_MESSAGE_FIELD_LEGACY)? + .unwrap_or_else(|| DEFAULT_ROS_MESSAGE_FIELD.to_string()), + }; + + let ros_version = match options.pull_opt_str(opt::ROS_VERSION)?.as_deref() { + Some("ros1" | "1") => RosVersion::Ros1 as i32, + Some("ros2" | "2") => RosVersion::Ros2 as i32, + None => default_version as i32, + Some(other) => { + return plan_err!("invalid ros.version '{other}'; expected ros1 or ros2"); + } + }; + + let sql_format = format.as_ref().ok_or_else(|| { + plan_datafusion_err!( + "ROS source requires 'format' in the WITH clause (e.g. format = 'json')" + ) + })?; + let proto_format = sql_format_to_proto(sql_format)?; + + let _ = options.pull_opt_str(opt::TYPE)?; + let _ = options.pull_opt_str(opt::CONNECTOR)?; + + let mut client_configs = options.drain_remaining_string_values()?; + client_configs.remove(opt::CHECKPOINT_INTERVAL_MS); + client_configs.remove(opt::PIPELINE_PARALLELISM); + client_configs.remove(opt::KEY_BY_PARALLELISM); + client_configs.remove(opt::FORMAT); + + Ok(ConnectorConfig::RosSource(RosSourceConfig { + url, + topic, + message_field, + format: Some(proto_format), + bad_data_policy: bad_data_to_proto(&bad_data), + client_configs, + ros_version, + })) +} diff --git a/src/sql/functions/mod.rs b/src/streaming_planner/src/functions/mod.rs similarity index 98% rename from src/sql/functions/mod.rs rename to src/streaming_planner/src/functions/mod.rs index b78f5d2a..9ec1007f 100644 --- a/src/sql/functions/mod.rs +++ b/src/streaming_planner/src/functions/mod.rs @@ -10,7 +10,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sql::schema::StreamSchemaProvider; +use crate::schema::StreamSchemaProvider; use datafusion::arrow::array::{ Array, ArrayRef, StringArray, UnionArray, builder::{FixedSizeBinaryBuilder, ListBuilder, StringBuilder}, @@ -34,7 +34,7 @@ use std::collections::HashMap; use std::fmt::{Debug, Write}; use std::sync::{Arc, OnceLock}; -use crate::sql::common::constants::scalar_fn; +use crate::common::constants::scalar_fn; /// Borrowed from DataFusion /// @@ -336,14 +336,14 @@ pub fn extract_json_string(args: &[ColumnarValue]) -> Result { // https://github.com/datafusion-contrib/datafusion-functions-json/blob/main/src/common_union.rs // as the `is_json_union` function is not public. It should be kept in sync with that code so // that we are able to detect JSON unions and rewrite them to serialized JSON for sinks. -pub(crate) fn is_json_union(data_type: &DataType) -> bool { +pub fn is_json_union(data_type: &DataType) -> bool { match data_type { DataType::Union(fields, UnionMode::Sparse) => fields == &union_fields(), _ => false, } } -pub(crate) const TYPE_ID_NULL: i8 = 0; +pub const TYPE_ID_NULL: i8 = 0; const TYPE_ID_BOOL: i8 = 1; const TYPE_ID_INT: i8 = 2; const TYPE_ID_FLOAT: i8 = 3; @@ -456,7 +456,7 @@ fn write_value(b: &mut StringBuilder, id: i8, a: &ArrayRef) -> Result<(), std::f Ok(()) } -pub(crate) fn serialize_outgoing_json( +pub fn serialize_outgoing_json( registry: &StreamSchemaProvider, node: Arc, ) -> LogicalPlan { diff --git a/src/sql/mod.rs b/src/streaming_planner/src/lib.rs similarity index 68% rename from src/sql/mod.rs rename to src/streaming_planner/src/lib.rs index f4a0eef6..2dbc2b47 100644 --- a/src/sql/mod.rs +++ b/src/streaming_planner/src/lib.rs @@ -10,6 +10,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(dead_code)] // Planner keeps helpers/types for upcoming features; strict -D dead_code is noisy. + +//! Streaming SQL planning (logical graph, connectors, schema, physical codec). +//! +//! The `function-stream` binary/library re-exports this crate as `function_stream::sql` for +//! stable `crate::sql::…` paths in in-tree modules. + pub mod api; pub mod common; @@ -20,7 +27,7 @@ pub mod logical_node; pub mod logical_planner; pub mod parse; pub mod physical; -pub(crate) mod planning_runtime; +pub mod planning_runtime; pub mod schema; pub mod types; diff --git a/src/sql/logical_node/aggregate.rs b/src/streaming_planner/src/logical_node/aggregate.rs similarity index 97% rename from src/sql/logical_node/aggregate.rs rename to src/streaming_planner/src/logical_node/aggregate.rs index 1e288ab5..cef337c0 100644 --- a/src/sql/logical_node/aggregate.rs +++ b/src/streaming_planner/src/logical_node/aggregate.rs @@ -30,25 +30,25 @@ use protocol::function_stream_graph::{ SessionWindowAggregateOperator, SlidingWindowAggregateOperator, TumblingWindowAggregateOperator, }; -use crate::multifield_partial_ord; -use crate::sql::common::constants::{extension_node, proto_operator_name}; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; -use crate::sql::logical_node::{ +use crate::common::constants::{extension_node, proto_operator_name}; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; +use crate::logical_node::{ CompiledTopologyNode, StreamingOperatorBlueprint, SystemTimestampInjectorNode, }; -use crate::sql::logical_planner::planner::{NamedNode, Planner, SplitPlanOutput}; -use crate::sql::physical::{StreamingExtensionCodec, window}; -use crate::sql::types::{ +use crate::logical_planner::planner::{NamedNode, Planner, SplitPlanOutput}; +use crate::multifield_partial_ord; +use crate::physical::{StreamingExtensionCodec, window}; +use crate::types::{ QualifiedField, TIMESTAMP_FIELD, WindowBehavior, WindowType, build_df_schema, build_df_schema_with_metadata, extract_qualified_fields, }; -pub(crate) const STREAM_AGG_EXTENSION_NAME: &str = extension_node::STREAM_WINDOW_AGGREGATE; +pub const STREAM_AGG_EXTENSION_NAME: &str = extension_node::STREAM_WINDOW_AGGREGATE; /// Represents a streaming windowed aggregation node in the logical plan. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct StreamWindowAggregateNode { +pub struct StreamWindowAggregateNode { pub(crate) window_spec: WindowBehavior, pub(crate) base_agg_plan: LogicalPlan, pub(crate) output_schema: DFSchemaRef, diff --git a/src/sql/logical_node/async_udf.rs b/src/streaming_planner/src/logical_node/async_udf.rs similarity index 93% rename from src/sql/logical_node/async_udf.rs rename to src/streaming_planner/src/logical_node/async_udf.rs index 1c35398e..a3d1f68c 100644 --- a/src/sql/logical_node/async_udf.rs +++ b/src/streaming_planner/src/logical_node/async_udf.rs @@ -24,25 +24,25 @@ use datafusion_proto::physical_plan::to_proto::serialize_physical_expr; use prost::Message; use protocol::function_stream_graph::{AsyncUdfOperator, AsyncUdfOrdering}; -use crate::multifield_partial_ord; -use crate::sql::common::constants::extension_node; -use crate::sql::common::constants::sql_field; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::logical::{ +use crate::common::constants::extension_node; +use crate::common::constants::sql_field; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::logical::{ DylibUdfConfig, LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName, }; -use crate::sql::logical_node::streaming_operator_blueprint::{ +use crate::logical_node::streaming_operator_blueprint::{ CompiledTopologyNode, StreamingOperatorBlueprint, }; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::types::{QualifiedField, build_df_schema, extract_qualified_fields}; +use crate::logical_planner::planner::{NamedNode, Planner}; +use crate::multifield_partial_ord; +use crate::types::{QualifiedField, build_df_schema, extract_qualified_fields}; -pub(crate) const NODE_TYPE_NAME: &str = extension_node::ASYNC_FUNCTION_EXECUTION; +pub const NODE_TYPE_NAME: &str = extension_node::ASYNC_FUNCTION_EXECUTION; /// Represents a logical node that executes an external asynchronous function (UDF) /// and projects the final results into the streaming pipeline. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct AsyncFunctionExecutionNode { +pub struct AsyncFunctionExecutionNode { pub(crate) upstream_plan: Arc, pub(crate) operator_name: String, pub(crate) function_config: DylibUdfConfig, diff --git a/src/sql/logical_node/debezium.rs b/src/streaming_planner/src/logical_node/debezium.rs similarity index 95% rename from src/sql/logical_node/debezium.rs rename to src/streaming_planner/src/logical_node/debezium.rs index 8d69c6ec..2b13f090 100644 --- a/src/sql/logical_node/debezium.rs +++ b/src/streaming_planner/src/logical_node/debezium.rs @@ -19,12 +19,12 @@ use datafusion::common::{ use datafusion::logical_expr::{Expr, LogicalPlan, UserDefinedLogicalNodeCore}; use datafusion::physical_plan::DisplayAs; +use crate::common::constants::{cdc, extension_node}; +use crate::common::{FsSchema, FsSchemaRef, UPDATING_META_FIELD}; +use crate::logical_planner::planner::{NamedNode, Planner}; use crate::multifield_partial_ord; -use crate::sql::common::constants::{cdc, extension_node}; -use crate::sql::common::{FsSchema, FsSchemaRef, UPDATING_META_FIELD}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::physical::updating_meta_field; -use crate::sql::types::TIMESTAMP_FIELD; +use crate::physical::updating_meta_field; +use crate::types::TIMESTAMP_FIELD; use super::{CompiledTopologyNode, StreamingOperatorBlueprint}; @@ -32,19 +32,19 @@ use super::{CompiledTopologyNode, StreamingOperatorBlueprint}; // Constants & Identifiers // ----------------------------------------------------------------------------- -pub(crate) const UNROLL_NODE_NAME: &str = extension_node::UNROLL_DEBEZIUM_PAYLOAD; -pub(crate) const PACK_NODE_NAME: &str = extension_node::PACK_DEBEZIUM_ENVELOPE; +pub const UNROLL_NODE_NAME: &str = extension_node::UNROLL_DEBEZIUM_PAYLOAD; +pub const PACK_NODE_NAME: &str = extension_node::PACK_DEBEZIUM_ENVELOPE; // ----------------------------------------------------------------------------- // Core Schema Codec // ----------------------------------------------------------------------------- /// Transforms between flat schemas and Debezium CDC envelopes. -pub(crate) struct DebeziumSchemaCodec; +pub struct DebeziumSchemaCodec; impl DebeziumSchemaCodec { /// Wraps a flat physical schema into a Debezium CDC envelope structure. - pub(crate) fn wrap_into_envelope( + pub fn wrap_into_envelope( flat_schema: &DFSchemaRef, qualifier_override: Option, ) -> Result { @@ -299,7 +299,7 @@ impl StreamingOperatorBlueprint for UnrollDebeziumPayloadNode { /// Encodes a flat updating stream back into a Debezium CDC envelope representation. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct PackDebeziumEnvelopeNode { +pub struct PackDebeziumEnvelopeNode { upstream_plan: Arc, envelope_schema: DFSchemaRef, } @@ -307,7 +307,7 @@ pub(crate) struct PackDebeziumEnvelopeNode { multifield_partial_ord!(PackDebeziumEnvelopeNode, upstream_plan); impl PackDebeziumEnvelopeNode { - pub(crate) fn try_new(upstream_plan: LogicalPlan) -> Result { + pub fn try_new(upstream_plan: LogicalPlan) -> Result { let envelope_schema = DebeziumSchemaCodec::wrap_into_envelope(upstream_plan.schema(), None) .map_err(|e| { DataFusionError::Plan(format!("Failed to compile Debezium envelope schema: {e}")) diff --git a/src/sql/logical_node/extension_try_from.rs b/src/streaming_planner/src/logical_node/extension_try_from.rs similarity index 72% rename from src/sql/logical_node/extension_try_from.rs rename to src/streaming_planner/src/logical_node/extension_try_from.rs index 32b12d6c..c13532dc 100644 --- a/src/sql/logical_node/extension_try_from.rs +++ b/src/streaming_planner/src/logical_node/extension_try_from.rs @@ -15,20 +15,20 @@ use std::sync::Arc; use datafusion::common::{DataFusionError, Result}; use datafusion::logical_expr::UserDefinedLogicalNode; -use crate::sql::logical_node::aggregate::StreamWindowAggregateNode; -use crate::sql::logical_node::async_udf::AsyncFunctionExecutionNode; -use crate::sql::logical_node::debezium::{PackDebeziumEnvelopeNode, UnrollDebeziumPayloadNode}; -use crate::sql::logical_node::join::StreamingJoinNode; -use crate::sql::logical_node::key_calculation::KeyExtractionNode; -use crate::sql::logical_node::lookup::StreamReferenceJoinNode; -use crate::sql::logical_node::projection::StreamProjectionNode; -use crate::sql::logical_node::remote_table::RemoteTableBoundaryNode; -use crate::sql::logical_node::sink::StreamEgressNode; -use crate::sql::logical_node::streaming_operator_blueprint::StreamingOperatorBlueprint; -use crate::sql::logical_node::table_source::StreamIngestionNode; -use crate::sql::logical_node::updating_aggregate::ContinuousAggregateNode; -use crate::sql::logical_node::watermark_node::EventTimeWatermarkNode; -use crate::sql::logical_node::windows_function::StreamingWindowFunctionNode; +use crate::logical_node::aggregate::StreamWindowAggregateNode; +use crate::logical_node::async_udf::AsyncFunctionExecutionNode; +use crate::logical_node::debezium::{PackDebeziumEnvelopeNode, UnrollDebeziumPayloadNode}; +use crate::logical_node::join::StreamingJoinNode; +use crate::logical_node::key_calculation::KeyExtractionNode; +use crate::logical_node::lookup::StreamReferenceJoinNode; +use crate::logical_node::projection::StreamProjectionNode; +use crate::logical_node::remote_table::RemoteTableBoundaryNode; +use crate::logical_node::sink::StreamEgressNode; +use crate::logical_node::streaming_operator_blueprint::StreamingOperatorBlueprint; +use crate::logical_node::table_source::StreamIngestionNode; +use crate::logical_node::updating_aggregate::ContinuousAggregateNode; +use crate::logical_node::watermark_node::EventTimeWatermarkNode; +use crate::logical_node::windows_function::StreamingWindowFunctionNode; fn try_from_t( node: &dyn UserDefinedLogicalNode, diff --git a/src/sql/logical_node/is_retract.rs b/src/streaming_planner/src/logical_node/is_retract.rs similarity index 89% rename from src/sql/logical_node/is_retract.rs rename to src/streaming_planner/src/logical_node/is_retract.rs index 4370f6ae..17fad0db 100644 --- a/src/sql/logical_node/is_retract.rs +++ b/src/streaming_planner/src/logical_node/is_retract.rs @@ -17,13 +17,11 @@ use datafusion::common::{DFSchemaRef, Result, TableReference}; use datafusion::logical_expr::{Expr, LogicalPlan, UserDefinedLogicalNodeCore}; use crate::multifield_partial_ord; -use crate::sql::physical::updating_meta_field; -use crate::sql::types::{ - QualifiedField, TIMESTAMP_FIELD, build_df_schema, extract_qualified_fields, -}; +use crate::physical::updating_meta_field; +use crate::types::{QualifiedField, TIMESTAMP_FIELD, build_df_schema, extract_qualified_fields}; #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct IsRetractExtension { +pub struct IsRetractExtension { pub(crate) input: LogicalPlan, pub(crate) schema: DFSchemaRef, pub(crate) timestamp_qualifier: Option, @@ -32,7 +30,7 @@ pub(crate) struct IsRetractExtension { multifield_partial_ord!(IsRetractExtension, input, timestamp_qualifier); impl IsRetractExtension { - pub(crate) fn new(input: LogicalPlan, timestamp_qualifier: Option) -> Self { + pub fn new(input: LogicalPlan, timestamp_qualifier: Option) -> Self { let mut output_fields = extract_qualified_fields(input.schema()); let timestamp_index = output_fields.len() - 1; diff --git a/src/sql/logical_node/join.rs b/src/streaming_planner/src/logical_node/join.rs similarity index 93% rename from src/sql/logical_node/join.rs rename to src/streaming_planner/src/logical_node/join.rs index 15631f1f..35d645f1 100644 --- a/src/sql/logical_node/join.rs +++ b/src/streaming_planner/src/logical_node/join.rs @@ -22,18 +22,18 @@ use datafusion_proto::protobuf::PhysicalPlanNode; use prost::Message; use protocol::function_stream_graph::JoinOperator; -use crate::sql::common::constants::{extension_node, runtime_operator_kind}; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; -use crate::sql::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::physical::StreamingExtensionCodec; +use crate::common::constants::{extension_node, runtime_operator_kind}; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; +use crate::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; +use crate::logical_planner::planner::{NamedNode, Planner}; +use crate::physical::StreamingExtensionCodec; // ----------------------------------------------------------------------------- // Constants // ----------------------------------------------------------------------------- -pub(crate) const STREAM_JOIN_NODE_TYPE: &str = extension_node::STREAMING_JOIN; +pub const STREAM_JOIN_NODE_TYPE: &str = extension_node::STREAMING_JOIN; // ----------------------------------------------------------------------------- // Logical Node Definition diff --git a/src/sql/logical_node/key_calculation.rs b/src/streaming_planner/src/logical_node/key_calculation.rs similarity index 94% rename from src/sql/logical_node/key_calculation.rs rename to src/streaming_planner/src/logical_node/key_calculation.rs index ec83e108..8bd03b6e 100644 --- a/src/sql/logical_node/key_calculation.rs +++ b/src/streaming_planner/src/logical_node/key_calculation.rs @@ -26,16 +26,16 @@ use prost::Message; use protocol::function_stream_graph::{KeyPlanOperator, ProjectionOperator}; +use crate::common::constants::{extension_node, sql_field}; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; +use crate::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; +use crate::logical_planner::planner::{NamedNode, Planner}; use crate::multifield_partial_ord; -use crate::sql::common::constants::{extension_node, sql_field}; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; -use crate::sql::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::physical::StreamingExtensionCodec; -use crate::sql::types::{build_df_schema_with_metadata, extract_qualified_fields}; +use crate::physical::StreamingExtensionCodec; +use crate::types::{build_df_schema_with_metadata, extract_qualified_fields}; -pub(crate) const EXTENSION_NODE_IDENTIFIER: &str = extension_node::KEY_EXTRACTION; +pub const EXTENSION_NODE_IDENTIFIER: &str = extension_node::KEY_EXTRACTION; /// Routing strategy for shuffling data across the stream topology. #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd)] @@ -46,7 +46,7 @@ pub enum KeyExtractionStrategy { /// Logical node that computes or extracts routing keys. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct KeyExtractionNode { +pub struct KeyExtractionNode { pub(crate) operator_label: Option, pub(crate) upstream_plan: LogicalPlan, pub(crate) extraction_strategy: KeyExtractionStrategy, diff --git a/src/sql/logical_node/logical/dylib_udf_config.rs b/src/streaming_planner/src/logical_node/logical/dylib_udf_config.rs similarity index 100% rename from src/sql/logical_node/logical/dylib_udf_config.rs rename to src/streaming_planner/src/logical_node/logical/dylib_udf_config.rs diff --git a/src/sql/logical_node/logical/fs_program_convert.rs b/src/streaming_planner/src/logical_node/logical/fs_program_convert.rs similarity index 98% rename from src/sql/logical_node/logical/fs_program_convert.rs rename to src/streaming_planner/src/logical_node/logical/fs_program_convert.rs index b05d68f5..53316bae 100644 --- a/src/sql/logical_node/logical/fs_program_convert.rs +++ b/src/streaming_planner/src/logical_node/logical/fs_program_convert.rs @@ -23,8 +23,8 @@ use protocol::function_stream_graph::{ FsSchema as ProtoFsSchema, }; -use crate::sql::api::pipelines::{PipelineEdge, PipelineGraph, PipelineNode}; -use crate::sql::common::FsSchema; +use crate::api::pipelines::{PipelineEdge, PipelineGraph, PipelineNode}; +use crate::common::FsSchema; use super::logical_edge::logical_edge_type_from_proto_i32; use super::operator_chain::{ChainedLogicalOperator, OperatorChain}; diff --git a/src/sql/logical_node/logical/logical_edge.rs b/src/streaming_planner/src/logical_node/logical/logical_edge.rs similarity index 96% rename from src/sql/logical_node/logical/logical_edge.rs rename to src/streaming_planner/src/logical_node/logical/logical_edge.rs index 87950e70..d599f7f1 100644 --- a/src/sql/logical_node/logical/logical_edge.rs +++ b/src/streaming_planner/src/logical_node/logical/logical_edge.rs @@ -17,7 +17,7 @@ use datafusion::common::{DataFusionError, Result}; use protocol::function_stream_graph::EdgeType as ProtoEdgeType; use serde::{Deserialize, Serialize}; -use crate::sql::common::FsSchema; +use crate::common::FsSchema; #[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, Serialize, Deserialize)] pub enum LogicalEdgeType { @@ -64,7 +64,7 @@ impl From for ProtoEdgeType { } } -pub(crate) fn logical_edge_type_from_proto_i32(i: i32) -> Result { +pub fn logical_edge_type_from_proto_i32(i: i32) -> Result { let e = ProtoEdgeType::try_from(i).map_err(|_| { DataFusionError::Plan(format!("invalid protobuf EdgeType discriminant {i}")) })?; diff --git a/src/sql/logical_node/logical/logical_graph.rs b/src/streaming_planner/src/logical_node/logical/logical_graph.rs similarity index 100% rename from src/sql/logical_node/logical/logical_graph.rs rename to src/streaming_planner/src/logical_node/logical/logical_graph.rs diff --git a/src/sql/logical_node/logical/logical_node.rs b/src/streaming_planner/src/logical_node/logical/logical_node.rs similarity index 100% rename from src/sql/logical_node/logical/logical_node.rs rename to src/streaming_planner/src/logical_node/logical/logical_node.rs diff --git a/src/sql/logical_node/logical/logical_program.rs b/src/streaming_planner/src/logical_node/logical/logical_program.rs similarity index 100% rename from src/sql/logical_node/logical/logical_program.rs rename to src/streaming_planner/src/logical_node/logical/logical_program.rs diff --git a/src/sql/logical_node/logical/mod.rs b/src/streaming_planner/src/logical_node/logical/mod.rs similarity index 100% rename from src/sql/logical_node/logical/mod.rs rename to src/streaming_planner/src/logical_node/logical/mod.rs diff --git a/src/sql/logical_node/logical/operator_chain.rs b/src/streaming_planner/src/logical_node/logical/operator_chain.rs similarity index 99% rename from src/sql/logical_node/logical/operator_chain.rs rename to src/streaming_planner/src/logical_node/logical/operator_chain.rs index 2aecddd6..34be2f57 100644 --- a/src/sql/logical_node/logical/operator_chain.rs +++ b/src/streaming_planner/src/logical_node/logical/operator_chain.rs @@ -18,7 +18,7 @@ use protocol::function_stream_graph::ConnectorOp; use serde::{Deserialize, Serialize}; use super::operator_name::OperatorName; -use crate::sql::common::FsSchema; +use crate::common::FsSchema; #[derive(Clone, Debug, Serialize, Deserialize)] pub struct ChainedLogicalOperator { diff --git a/src/sql/logical_node/logical/operator_name.rs b/src/streaming_planner/src/logical_node/logical/operator_name.rs similarity index 98% rename from src/sql/logical_node/logical/operator_name.rs rename to src/streaming_planner/src/logical_node/logical/operator_name.rs index 57f53f90..4377b0bd 100644 --- a/src/sql/logical_node/logical/operator_name.rs +++ b/src/streaming_planner/src/logical_node/logical/operator_name.rs @@ -15,7 +15,7 @@ use std::str::FromStr; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use strum::{Display, EnumString, IntoStaticStr}; -use crate::sql::common::constants::operator_feature; +use crate::common::constants::operator_feature; #[derive(Clone, Copy, Debug, Eq, PartialEq, EnumString, Display, IntoStaticStr)] pub enum OperatorName { diff --git a/src/sql/logical_node/logical/program_config.rs b/src/streaming_planner/src/logical_node/logical/program_config.rs similarity index 100% rename from src/sql/logical_node/logical/program_config.rs rename to src/streaming_planner/src/logical_node/logical/program_config.rs diff --git a/src/sql/logical_node/logical/python_udf_config.rs b/src/streaming_planner/src/logical_node/logical/python_udf_config.rs similarity index 100% rename from src/sql/logical_node/logical/python_udf_config.rs rename to src/streaming_planner/src/logical_node/logical/python_udf_config.rs diff --git a/src/sql/logical_node/lookup.rs b/src/streaming_planner/src/logical_node/lookup.rs similarity index 95% rename from src/sql/logical_node/lookup.rs rename to src/streaming_planner/src/logical_node/lookup.rs index d2817c85..ca95c4a6 100644 --- a/src/sql/logical_node/lookup.rs +++ b/src/streaming_planner/src/logical_node/lookup.rs @@ -23,14 +23,14 @@ use prost::Message; use protocol::function_stream_graph; use protocol::function_stream_graph::{ConnectorOp, LookupJoinCondition, LookupJoinOperator}; +use crate::common::constants::extension_node; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; +use crate::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; +use crate::logical_planner::planner::{NamedNode, Planner}; use crate::multifield_partial_ord; -use crate::sql::common::constants::extension_node; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; -use crate::sql::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::schema::LookupTable; -use crate::sql::schema::utils::add_timestamp_field_arrow; +use crate::schema::LookupTable; +use crate::schema::utils::add_timestamp_field_arrow; pub const DICTIONARY_SOURCE_NODE_NAME: &str = extension_node::REFERENCE_TABLE_SOURCE; pub const STREAM_DICTIONARY_JOIN_NODE_NAME: &str = extension_node::STREAM_REFERENCE_JOIN; diff --git a/src/sql/logical_node/macros.rs b/src/streaming_planner/src/logical_node/macros.rs similarity index 100% rename from src/sql/logical_node/macros.rs rename to src/streaming_planner/src/logical_node/macros.rs diff --git a/src/streaming_planner/src/logical_node/mod.rs b/src/streaming_planner/src/logical_node/mod.rs new file mode 100644 index 00000000..3ef02fc8 --- /dev/null +++ b/src/streaming_planner/src/logical_node/mod.rs @@ -0,0 +1,42 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod logical; + +mod macros; + +pub mod streaming_operator_blueprint; +pub use streaming_operator_blueprint::{CompiledTopologyNode, StreamingOperatorBlueprint}; + +pub mod aggregate; +pub mod debezium; +pub mod join; +pub mod key_calculation; +pub mod lookup; +pub mod projection; +pub mod remote_table; +pub mod sink; +pub mod table_source; +pub mod updating_aggregate; +pub mod watermark_node; +pub mod windows_function; + +pub mod timestamp_append; +pub use timestamp_append::SystemTimestampInjectorNode; + +pub mod async_udf; +pub use async_udf::AsyncFunctionExecutionNode; + +pub mod is_retract; +pub use is_retract::IsRetractExtension; + +mod extension_try_from; diff --git a/src/sql/logical_node/projection.rs b/src/streaming_planner/src/logical_node/projection.rs similarity index 92% rename from src/sql/logical_node/projection.rs rename to src/streaming_planner/src/logical_node/projection.rs index 3c5cfccb..f56b109f 100644 --- a/src/sql/logical_node/projection.rs +++ b/src/streaming_planner/src/logical_node/projection.rs @@ -21,19 +21,19 @@ use prost::Message; use protocol::function_stream_graph::ProjectionOperator; +use crate::common::constants::{extension_node, sql_field}; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; +use crate::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; +use crate::logical_planner::planner::{NamedNode, Planner}; use crate::multifield_partial_ord; -use crate::sql::common::constants::{extension_node, sql_field}; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; -use crate::sql::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::types::{QualifiedField, build_df_schema}; +use crate::types::{QualifiedField, build_df_schema}; // ----------------------------------------------------------------------------- // Constants & Identifiers // ----------------------------------------------------------------------------- -pub(crate) const STREAM_PROJECTION_NODE_NAME: &str = extension_node::STREAM_PROJECTION; +pub const STREAM_PROJECTION_NODE_NAME: &str = extension_node::STREAM_PROJECTION; const DEFAULT_PROJECTION_LABEL: &str = sql_field::DEFAULT_PROJECTION_LABEL; // ----------------------------------------------------------------------------- @@ -42,7 +42,7 @@ const DEFAULT_PROJECTION_LABEL: &str = sql_field::DEFAULT_PROJECTION_LABEL; /// Projection within a streaming execution topology. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct StreamProjectionNode { +pub struct StreamProjectionNode { pub(crate) upstream_plans: Vec, pub(crate) operator_label: Option, pub(crate) projection_exprs: Vec, @@ -53,7 +53,7 @@ pub(crate) struct StreamProjectionNode { multifield_partial_ord!(StreamProjectionNode, operator_label, projection_exprs); impl StreamProjectionNode { - pub(crate) fn try_new( + pub fn try_new( upstream_plans: Vec, operator_label: Option, projection_exprs: Vec, @@ -81,7 +81,7 @@ impl StreamProjectionNode { }) } - pub(crate) fn with_shuffle_routing(mut self) -> Self { + pub fn with_shuffle_routing(mut self) -> Self { self.requires_shuffle = true; self } diff --git a/src/sql/logical_node/remote_table.rs b/src/streaming_planner/src/logical_node/remote_table.rs similarity index 92% rename from src/sql/logical_node/remote_table.rs rename to src/streaming_planner/src/logical_node/remote_table.rs index bde1d47f..302228e7 100644 --- a/src/sql/logical_node/remote_table.rs +++ b/src/streaming_planner/src/logical_node/remote_table.rs @@ -21,19 +21,19 @@ use prost::Message; use protocol::function_stream_graph::ValuePlanOperator; +use crate::common::constants::extension_node; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; +use crate::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; +use crate::logical_planner::planner::{NamedNode, Planner}; use crate::multifield_partial_ord; -use crate::sql::common::constants::extension_node; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; -use crate::sql::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::physical::StreamingExtensionCodec; +use crate::physical::StreamingExtensionCodec; // ----------------------------------------------------------------------------- // Constants & Identifiers // ----------------------------------------------------------------------------- -pub(crate) const REMOTE_TABLE_NODE_NAME: &str = extension_node::REMOTE_TABLE_BOUNDARY; +pub const REMOTE_TABLE_NODE_NAME: &str = extension_node::REMOTE_TABLE_BOUNDARY; // ----------------------------------------------------------------------------- // Logical Node Definition @@ -41,7 +41,7 @@ pub(crate) const REMOTE_TABLE_NODE_NAME: &str = extension_node::REMOTE_TABLE_BOU /// Segments the execution graph and merges nodes sharing the same identifier; acts as a boundary. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct RemoteTableBoundaryNode { +pub struct RemoteTableBoundaryNode { pub(crate) upstream_plan: LogicalPlan, pub(crate) table_identifier: TableReference, pub(crate) resolved_schema: DFSchemaRef, diff --git a/src/sql/logical_node/sink.rs b/src/streaming_planner/src/logical_node/sink.rs similarity index 94% rename from src/sql/logical_node/sink.rs rename to src/streaming_planner/src/logical_node/sink.rs index d767afe3..f0b66187 100644 --- a/src/sql/logical_node/sink.rs +++ b/src/streaming_planner/src/logical_node/sink.rs @@ -17,14 +17,14 @@ use datafusion::common::{DFSchemaRef, Result, TableReference, plan_err}; use datafusion::logical_expr::{Expr, Extension, LogicalPlan, UserDefinedLogicalNodeCore}; use prost::Message; +use crate::common::constants::extension_node; +use crate::common::{FsSchema, FsSchemaRef, UPDATING_META_FIELD}; +use crate::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; +use crate::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; +use crate::logical_planner::planner::{NamedNode, Planner}; use crate::multifield_partial_ord; -use crate::sql::common::constants::extension_node; -use crate::sql::common::{FsSchema, FsSchemaRef, UPDATING_META_FIELD}; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; -use crate::sql::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::schema::CatalogEntity; -use crate::sql::schema::catalog::ExternalTable; +use crate::schema::CatalogEntity; +use crate::schema::catalog::ExternalTable; use super::debezium::PackDebeziumEnvelopeNode; use super::remote_table::RemoteTableBoundaryNode; @@ -33,7 +33,7 @@ use super::remote_table::RemoteTableBoundaryNode; // Constants & Identifiers // ----------------------------------------------------------------------------- -pub(crate) const STREAM_EGRESS_NODE_NAME: &str = extension_node::STREAM_EGRESS; +pub const STREAM_EGRESS_NODE_NAME: &str = extension_node::STREAM_EGRESS; // ----------------------------------------------------------------------------- // Logical Node Definition @@ -41,7 +41,7 @@ pub(crate) const STREAM_EGRESS_NODE_NAME: &str = extension_node::STREAM_EGRESS; /// Terminal node routing processed data into an external sink (e.g. Kafka, PostgreSQL). #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct StreamEgressNode { +pub struct StreamEgressNode { pub(crate) target_identifier: TableReference, pub(crate) destination_table: CatalogEntity, pub(crate) egress_schema: DFSchemaRef, diff --git a/src/sql/logical_node/streaming_operator_blueprint.rs b/src/streaming_planner/src/logical_node/streaming_operator_blueprint.rs similarity index 88% rename from src/sql/logical_node/streaming_operator_blueprint.rs rename to src/streaming_planner/src/logical_node/streaming_operator_blueprint.rs index d3f9d459..d9afaeab 100644 --- a/src/sql/logical_node/streaming_operator_blueprint.rs +++ b/src/streaming_planner/src/logical_node/streaming_operator_blueprint.rs @@ -14,16 +14,16 @@ use std::fmt::Debug; use datafusion::common::Result; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalNode}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::logical::{LogicalEdge, LogicalNode}; +use crate::logical_planner::planner::{NamedNode, Planner}; // ----------------------------------------------------------------------------- // Core Execution Blueprint // ----------------------------------------------------------------------------- /// Atomic unit within a streaming execution topology: translates streaming SQL into graph nodes. -pub(crate) trait StreamingOperatorBlueprint: Debug { +pub trait StreamingOperatorBlueprint: Debug { /// Canonical named identity for this operator, if any (sources, sinks, etc.). fn operator_identity(&self) -> Option; @@ -50,7 +50,7 @@ pub(crate) trait StreamingOperatorBlueprint: Debug { /// Compiled vertex: execution unit plus upstream routing edges. #[derive(Debug, Clone)] -pub(crate) struct CompiledTopologyNode { +pub struct CompiledTopologyNode { pub execution_unit: LogicalNode, pub routing_edges: Vec, } diff --git a/src/sql/logical_node/table_source.rs b/src/streaming_planner/src/logical_node/table_source.rs similarity index 91% rename from src/sql/logical_node/table_source.rs rename to src/streaming_planner/src/logical_node/table_source.rs index b1c6bfdd..dad8f921 100644 --- a/src/sql/logical_node/table_source.rs +++ b/src/streaming_planner/src/logical_node/table_source.rs @@ -17,15 +17,15 @@ use datafusion::common::{DFSchemaRef, Result, TableReference, plan_err}; use datafusion::logical_expr::{Expr, LogicalPlan, UserDefinedLogicalNodeCore}; use prost::Message; +use crate::common::constants::extension_node; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::debezium::DebeziumSchemaCodec; +use crate::logical_node::logical::{LogicalNode, OperatorName}; +use crate::logical_planner::planner::{NamedNode, Planner}; use crate::multifield_partial_ord; -use crate::sql::common::constants::extension_node; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::debezium::DebeziumSchemaCodec; -use crate::sql::logical_node::logical::{LogicalNode, OperatorName}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::schema::SourceTable; -use crate::sql::schema::utils::add_timestamp_field; -use crate::sql::types::build_df_schema; +use crate::schema::SourceTable; +use crate::schema::utils::add_timestamp_field; +use crate::types::build_df_schema; use super::{CompiledTopologyNode, StreamingOperatorBlueprint}; @@ -33,7 +33,7 @@ use super::{CompiledTopologyNode, StreamingOperatorBlueprint}; // Constants & Identifiers // ----------------------------------------------------------------------------- -pub(crate) const STREAM_INGESTION_NODE_NAME: &str = extension_node::STREAM_INGESTION; +pub const STREAM_INGESTION_NODE_NAME: &str = extension_node::STREAM_INGESTION; // ----------------------------------------------------------------------------- // Logical Node Definition @@ -41,7 +41,7 @@ pub(crate) const STREAM_INGESTION_NODE_NAME: &str = extension_node::STREAM_INGES /// Foundational ingestion point: connects to external systems and injects raw or CDC data. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct StreamIngestionNode { +pub struct StreamIngestionNode { pub(crate) source_identifier: TableReference, pub(crate) source_definition: SourceTable, pub(crate) resolved_schema: DFSchemaRef, diff --git a/src/sql/logical_node/timestamp_append.rs b/src/streaming_planner/src/logical_node/timestamp_append.rs similarity index 92% rename from src/sql/logical_node/timestamp_append.rs rename to src/streaming_planner/src/logical_node/timestamp_append.rs index 630e5a66..a490243a 100644 --- a/src/sql/logical_node/timestamp_append.rs +++ b/src/streaming_planner/src/logical_node/timestamp_append.rs @@ -15,15 +15,15 @@ use std::fmt::Formatter; use datafusion::common::{DFSchemaRef, Result, TableReference, internal_err}; use datafusion::logical_expr::{Expr, LogicalPlan, UserDefinedLogicalNodeCore}; +use crate::common::constants::extension_node; use crate::multifield_partial_ord; -use crate::sql::common::constants::extension_node; -use crate::sql::schema::utils::{add_timestamp_field, has_timestamp_field}; +use crate::schema::utils::{add_timestamp_field, has_timestamp_field}; // ----------------------------------------------------------------------------- // Constants & Identifiers // ----------------------------------------------------------------------------- -pub(crate) const TIMESTAMP_INJECTOR_NODE_NAME: &str = extension_node::SYSTEM_TIMESTAMP_INJECTOR; +pub const TIMESTAMP_INJECTOR_NODE_NAME: &str = extension_node::SYSTEM_TIMESTAMP_INJECTOR; // ----------------------------------------------------------------------------- // Logical Node Definition @@ -31,7 +31,7 @@ pub(crate) const TIMESTAMP_INJECTOR_NODE_NAME: &str = extension_node::SYSTEM_TIM /// Injects the mandatory system `_timestamp` field into the upstream streaming schema. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct SystemTimestampInjectorNode { +pub struct SystemTimestampInjectorNode { pub(crate) upstream_plan: LogicalPlan, pub(crate) target_qualifier: Option, pub(crate) resolved_schema: DFSchemaRef, @@ -40,7 +40,7 @@ pub(crate) struct SystemTimestampInjectorNode { multifield_partial_ord!(SystemTimestampInjectorNode, upstream_plan, target_qualifier); impl SystemTimestampInjectorNode { - pub(crate) fn try_new( + pub fn try_new( upstream_plan: LogicalPlan, target_qualifier: Option, ) -> Result { diff --git a/src/sql/logical_node/updating_aggregate.rs b/src/streaming_planner/src/logical_node/updating_aggregate.rs similarity index 92% rename from src/sql/logical_node/updating_aggregate.rs rename to src/streaming_planner/src/logical_node/updating_aggregate.rs index 0ddb2b28..14f52ea7 100644 --- a/src/sql/logical_node/updating_aggregate.rs +++ b/src/streaming_planner/src/logical_node/updating_aggregate.rs @@ -25,21 +25,19 @@ use datafusion_proto::protobuf::PhysicalPlanNode; use prost::Message; use protocol::function_stream_graph::UpdatingAggregateOperator; -use crate::sql::common::constants::{extension_node, proto_operator_name, updating_state_field}; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::functions::multi_hash; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; -use crate::sql::logical_node::{ - CompiledTopologyNode, IsRetractExtension, StreamingOperatorBlueprint, -}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::physical::StreamingExtensionCodec; +use crate::common::constants::{extension_node, proto_operator_name, updating_state_field}; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::functions::multi_hash; +use crate::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; +use crate::logical_node::{CompiledTopologyNode, IsRetractExtension, StreamingOperatorBlueprint}; +use crate::logical_planner::planner::{NamedNode, Planner}; +use crate::physical::StreamingExtensionCodec; // ----------------------------------------------------------------------------- // Constants & Configuration // ----------------------------------------------------------------------------- -pub(crate) const CONTINUOUS_AGGREGATE_NODE_NAME: &str = extension_node::CONTINUOUS_AGGREGATE; +pub const CONTINUOUS_AGGREGATE_NODE_NAME: &str = extension_node::CONTINUOUS_AGGREGATE; const DEFAULT_FLUSH_INTERVAL_MICROS: u64 = 10_000_000; @@ -51,7 +49,7 @@ const STATIC_HASH_SIZE_BYTES: i32 = 16; /// Stateful continuous aggregation: running aggregates with updating / retraction semantics. #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd)] -pub(crate) struct ContinuousAggregateNode { +pub struct ContinuousAggregateNode { pub(crate) base_aggregate_plan: LogicalPlan, pub(crate) partition_key_indices: Vec, pub(crate) retract_injected_plan: LogicalPlan, diff --git a/src/sql/logical_node/watermark_node.rs b/src/streaming_planner/src/logical_node/watermark_node.rs similarity index 91% rename from src/sql/logical_node/watermark_node.rs rename to src/streaming_planner/src/logical_node/watermark_node.rs index 9a8fc9d6..8d51fc47 100644 --- a/src/sql/logical_node/watermark_node.rs +++ b/src/streaming_planner/src/logical_node/watermark_node.rs @@ -21,20 +21,20 @@ use datafusion_proto::physical_plan::to_proto::serialize_physical_expr; use prost::Message; use protocol::function_stream_graph::ExpressionWatermarkConfig; +use crate::common::constants::{extension_node, runtime_operator_kind}; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; +use crate::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; +use crate::logical_planner::planner::{NamedNode, Planner}; use crate::multifield_partial_ord; -use crate::sql::common::constants::{extension_node, runtime_operator_kind}; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; -use crate::sql::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::schema::utils::add_timestamp_field; -use crate::sql::types::TIMESTAMP_FIELD; +use crate::schema::utils::add_timestamp_field; +use crate::types::TIMESTAMP_FIELD; // ----------------------------------------------------------------------------- // Constants & Identifiers // ----------------------------------------------------------------------------- -pub(crate) const EVENT_TIME_WATERMARK_NODE_NAME: &str = extension_node::EVENT_TIME_WATERMARK; +pub const EVENT_TIME_WATERMARK_NODE_NAME: &str = extension_node::EVENT_TIME_WATERMARK; const DEFAULT_WATERMARK_EMISSION_PERIOD_MICROS: u64 = 1_000_000; @@ -44,7 +44,7 @@ const DEFAULT_WATERMARK_EMISSION_PERIOD_MICROS: u64 = 1_000_000; /// Event-time watermark from a user strategy; drives time progress in stateful operators. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct EventTimeWatermarkNode { +pub struct EventTimeWatermarkNode { pub(crate) upstream_plan: LogicalPlan, pub(crate) namespace_qualifier: TableReference, pub(crate) watermark_strategy_expr: Expr, @@ -61,7 +61,7 @@ multifield_partial_ord!( ); impl EventTimeWatermarkNode { - pub(crate) fn try_new( + pub fn try_new( upstream_plan: LogicalPlan, namespace_qualifier: TableReference, watermark_strategy_expr: Expr, @@ -89,7 +89,7 @@ impl EventTimeWatermarkNode { }) } - pub(crate) fn generate_fs_schema(&self) -> FsSchema { + pub fn generate_fs_schema(&self) -> FsSchema { FsSchema::new_unkeyed( Arc::new(self.resolved_schema.as_ref().into()), self.internal_timestamp_offset, diff --git a/src/sql/logical_node/windows_function.rs b/src/streaming_planner/src/logical_node/windows_function.rs similarity index 92% rename from src/sql/logical_node/windows_function.rs rename to src/streaming_planner/src/logical_node/windows_function.rs index 9be37382..5fdfb9ce 100644 --- a/src/sql/logical_node/windows_function.rs +++ b/src/streaming_planner/src/logical_node/windows_function.rs @@ -21,12 +21,12 @@ use datafusion_proto::{physical_plan::AsExecutionPlan, protobuf::PhysicalPlanNod use prost::Message; use protocol::function_stream_graph::WindowFunctionOperator; -use crate::sql::common::constants::{extension_node, proto_operator_name, runtime_operator_kind}; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; -use crate::sql::logical_planner::planner::{NamedNode, Planner}; -use crate::sql::physical::StreamingExtensionCodec; -use crate::sql::types::TIMESTAMP_FIELD; +use crate::common::constants::{extension_node, proto_operator_name, runtime_operator_kind}; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::logical::{LogicalEdge, LogicalEdgeType, LogicalNode, OperatorName}; +use crate::logical_planner::planner::{NamedNode, Planner}; +use crate::physical::StreamingExtensionCodec; +use crate::types::TIMESTAMP_FIELD; use super::{CompiledTopologyNode, StreamingOperatorBlueprint}; @@ -34,7 +34,7 @@ use super::{CompiledTopologyNode, StreamingOperatorBlueprint}; // Constants & Identifiers // ----------------------------------------------------------------------------- -pub(crate) const STREAMING_WINDOW_NODE_NAME: &str = extension_node::STREAMING_WINDOW_FUNCTION; +pub const STREAMING_WINDOW_NODE_NAME: &str = extension_node::STREAMING_WINDOW_FUNCTION; // ----------------------------------------------------------------------------- // Logical Node Definition @@ -42,7 +42,7 @@ pub(crate) const STREAMING_WINDOW_NODE_NAME: &str = extension_node::STREAMING_WI /// Stateful streaming window: temporal binning plus underlying window evaluation plan. #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd)] -pub(crate) struct StreamingWindowFunctionNode { +pub struct StreamingWindowFunctionNode { pub(crate) underlying_evaluation_plan: LogicalPlan, pub(crate) partition_key_indices: Vec, } diff --git a/src/sql/logical_planner/mod.rs b/src/streaming_planner/src/logical_planner/mod.rs similarity index 87% rename from src/sql/logical_planner/mod.rs rename to src/streaming_planner/src/logical_planner/mod.rs index 9ecfb676..d04ccce3 100644 --- a/src/sql/logical_planner/mod.rs +++ b/src/streaming_planner/src/logical_planner/mod.rs @@ -12,5 +12,5 @@ pub mod optimizers; -pub(crate) mod streaming_planner; -pub(crate) use streaming_planner as planner; +pub mod streaming_planner; +pub use streaming_planner as planner; diff --git a/src/sql/logical_planner/optimizers/chaining.rs b/src/streaming_planner/src/logical_planner/optimizers/chaining.rs similarity index 97% rename from src/sql/logical_planner/optimizers/chaining.rs rename to src/streaming_planner/src/logical_planner/optimizers/chaining.rs index ea7bd885..60b36115 100644 --- a/src/sql/logical_planner/optimizers/chaining.rs +++ b/src/streaming_planner/src/logical_planner/optimizers/chaining.rs @@ -14,7 +14,7 @@ use petgraph::prelude::*; use petgraph::visit::NodeIndexable; use tracing::debug; -use crate::sql::logical_node::logical::{LogicalEdgeType, LogicalGraph, Optimizer}; +use crate::logical_node::logical::{LogicalEdgeType, LogicalGraph, Optimizer}; pub struct ChainingOptimizer {} @@ -119,8 +119,8 @@ mod tests { use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; - use crate::sql::common::FsSchema; - use crate::sql::logical_node::logical::{ + use crate::common::FsSchema; + use crate::logical_node::logical::{ LogicalEdge, LogicalEdgeType, LogicalGraph, LogicalNode, OperatorName, Optimizer, }; diff --git a/src/sql/logical_planner/optimizers/mod.rs b/src/streaming_planner/src/logical_planner/optimizers/mod.rs similarity index 100% rename from src/sql/logical_planner/optimizers/mod.rs rename to src/streaming_planner/src/logical_planner/optimizers/mod.rs diff --git a/src/sql/logical_planner/optimizers/optimized_plan.rs b/src/streaming_planner/src/logical_planner/optimizers/optimized_plan.rs similarity index 98% rename from src/sql/logical_planner/optimizers/optimized_plan.rs rename to src/streaming_planner/src/logical_planner/optimizers/optimized_plan.rs index fbb64845..df380fe1 100644 --- a/src/sql/logical_planner/optimizers/optimized_plan.rs +++ b/src/streaming_planner/src/logical_planner/optimizers/optimized_plan.rs @@ -41,7 +41,7 @@ use datafusion::optimizer::simplify_expressions::SimplifyExpressions; use datafusion::sql::planner::SqlToRel; use datafusion::sql::sqlparser::ast::Statement; -use crate::sql::schema::StreamSchemaProvider; +use crate::schema::StreamSchemaProvider; /// Converts a SQL statement into an optimized DataFusion logical plan. /// diff --git a/src/sql/logical_planner/streaming_planner.rs b/src/streaming_planner/src/logical_planner/streaming_planner.rs similarity index 90% rename from src/sql/logical_planner/streaming_planner.rs rename to src/streaming_planner/src/logical_planner/streaming_planner.rs index 1e999c2a..a5f3d5f6 100644 --- a/src/sql/logical_planner/streaming_planner.rs +++ b/src/streaming_planner/src/logical_planner/streaming_planner.rs @@ -42,35 +42,33 @@ use datafusion_common::TableReference; use datafusion_proto::physical_plan::DefaultPhysicalExtensionCodec; use datafusion_proto::physical_plan::to_proto::serialize_physical_expr; -use crate::sql::common::constants::sql_planning_default; -use crate::sql::common::{FsSchema, FsSchemaRef}; -use crate::sql::logical_node::debezium::{ - PACK_NODE_NAME, UNROLL_NODE_NAME, UnrollDebeziumPayloadNode, -}; -use crate::sql::logical_node::key_calculation::KeyExtractionNode; -use crate::sql::logical_node::logical::{LogicalEdge, LogicalGraph, LogicalNode}; -use crate::sql::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; -use crate::sql::physical::{ +use crate::common::constants::sql_planning_default; +use crate::common::{FsSchema, FsSchemaRef}; +use crate::logical_node::debezium::{PACK_NODE_NAME, UNROLL_NODE_NAME, UnrollDebeziumPayloadNode}; +use crate::logical_node::key_calculation::KeyExtractionNode; +use crate::logical_node::logical::{LogicalEdge, LogicalGraph, LogicalNode}; +use crate::logical_node::{CompiledTopologyNode, StreamingOperatorBlueprint}; +use crate::physical::{ CdcDebeziumPackExec, CdcDebeziumUnrollExec, FsMemExec, StreamingDecodingContext, StreamingExtensionCodec, }; -use crate::sql::schema::StreamSchemaProvider; -use crate::sql::schema::utils::add_timestamp_field_arrow; +use crate::schema::StreamSchemaProvider; +use crate::schema::utils::add_timestamp_field_arrow; -pub(crate) struct SplitPlanOutput { +pub struct SplitPlanOutput { pub(crate) partial_aggregation_plan: PhysicalPlanNode, pub(crate) partial_schema: FsSchema, pub(crate) finish_plan: PhysicalPlanNode, } #[derive(Eq, Hash, PartialEq, Debug)] -pub(crate) enum NamedNode { +pub enum NamedNode { Source(TableReference), Watermark(TableReference), RemoteTable(TableReference), Sink(TableReference), } -pub(crate) struct PlanToGraphVisitor<'a> { +pub struct PlanToGraphVisitor<'a> { graph: DiGraph, output_schemas: HashMap, named_nodes: HashMap, @@ -90,7 +88,7 @@ impl<'a> PlanToGraphVisitor<'a> { } } -pub(crate) struct Planner<'a> { +pub struct Planner<'a> { schema_provider: &'a StreamSchemaProvider, planner: DefaultPhysicalPlanner, session_state: &'a SessionState, @@ -98,25 +96,22 @@ pub(crate) struct Planner<'a> { impl<'a> Planner<'a> { #[inline] - pub(crate) fn default_parallelism(&self) -> usize { + pub fn default_parallelism(&self) -> usize { self.schema_provider.default_parallelism() } #[inline] - pub(crate) fn key_by_parallelism(&self) -> usize { + pub fn key_by_parallelism(&self) -> usize { self.schema_provider.key_by_parallelism() } /// Parallelism for operators that consume a keyed shuffle (non-empty partition keys). #[inline] - pub(crate) fn keyed_aggregate_parallelism(&self) -> usize { + pub fn keyed_aggregate_parallelism(&self) -> usize { sql_planning_default::KEYED_AGGREGATE_DEFAULT_PARALLELISM } - pub(crate) fn new( - schema_provider: &'a StreamSchemaProvider, - session_state: &'a SessionState, - ) -> Self { + pub fn new(schema_provider: &'a StreamSchemaProvider, session_state: &'a SessionState) -> Self { let planner = DefaultPhysicalPlanner::with_extension_planners(vec![Arc::new(FsExtensionPlanner {})]); Self { @@ -126,7 +121,7 @@ impl<'a> Planner<'a> { } } - pub(crate) fn sync_plan(&self, plan: &LogicalPlan) -> Result> { + pub fn sync_plan(&self, plan: &LogicalPlan) -> Result> { let fut = self.planner.create_physical_plan(plan, self.session_state); let (tx, mut rx) = oneshot::channel(); thread::scope(|s| { @@ -150,7 +145,7 @@ impl<'a> Planner<'a> { rx.try_recv().unwrap() } - pub(crate) fn create_physical_expr( + pub fn create_physical_expr( &self, expr: &Expr, input_dfschema: &DFSchema, @@ -159,17 +154,13 @@ impl<'a> Planner<'a> { .create_physical_expr(expr, input_dfschema, self.session_state) } - pub(crate) fn serialize_as_physical_expr( - &self, - expr: &Expr, - schema: &DFSchema, - ) -> Result> { + pub fn serialize_as_physical_expr(&self, expr: &Expr, schema: &DFSchema) -> Result> { let physical = self.create_physical_expr(expr, schema)?; let proto = serialize_physical_expr(&physical, &DefaultPhysicalExtensionCodec {})?; Ok(proto.encode_to_vec()) } - pub(crate) fn split_physical_plan( + pub fn split_physical_plan( &self, key_indices: Vec, aggregate: &LogicalPlan, @@ -313,7 +304,7 @@ impl PlanToGraphVisitor<'_> { } } - pub(crate) fn add_plan(&mut self, plan: LogicalPlan) -> Result<()> { + pub fn add_plan(&mut self, plan: LogicalPlan) -> Result<()> { self.traversal.clear(); plan.visit(self)?; Ok(()) diff --git a/src/streaming_planner/src/parse.rs b/src/streaming_planner/src/parse.rs new file mode 100644 index 00000000..563214aa --- /dev/null +++ b/src/streaming_planner/src/parse.rs @@ -0,0 +1,82 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! FunctionStream SQL parsing (`parse_sql`). +//! +//! This module only performs lexical/syntactic parsing into sqlparser +//! [`Statement`](datafusion::sql::sqlparser::ast::Statement) values using +//! [`FunctionStreamDialect`]. Mapping those AST nodes to coordinator `Statement` +//! values is done by `classify_statement` in the `function-stream` coordinator module. +//! +//! **Data-definition / pipeline shape (supported forms in the dialect)** +//! - **`CREATE TABLE ... (cols [, WATERMARK FOR ...]) WITH (...)`** — connector-backed source DDL +//! - **`CREATE TABLE ...`** other forms (including `AS SELECT` where the dialect accepts it) +//! - **`CREATE STREAMING TABLE ... WITH (...) AS SELECT ...`** +//! - **`DROP TABLE`** / **`DROP STREAMING TABLE`** +//! - **`SHOW TABLES`**, **`SHOW STREAMING TABLE(S)`**, **`SHOW CREATE TABLE`**, **`SHOW CREATE STREAMING TABLE`** +//! +//! **`INSERT` is not supported** at the coordinator layer — use `CREATE TABLE ... AS SELECT` or +//! `CREATE STREAMING TABLE ... AS SELECT` instead (see coordinator classification). + +use datafusion::common::{Result, plan_err}; +use datafusion::error::DataFusionError; +use datafusion::sql::sqlparser::ast::Statement as DFStatement; +use datafusion::sql::sqlparser::dialect::FunctionStreamDialect; +use datafusion::sql::sqlparser::parser::Parser; + +/// Parse SQL text into zero or more dialect [`Statement`](DFStatement) nodes. +pub fn parse_sql(query: &str) -> Result> { + let trimmed = query.trim(); + if trimmed.is_empty() { + return plan_err!("Query is empty"); + } + + let dialect = FunctionStreamDialect {}; + let statements = Parser::parse_sql(&dialect, trimmed) + .map_err(|e| DataFusionError::Plan(format!("SQL parse error: {e}")))?; + + if statements.is_empty() { + return plan_err!("No SQL statements found"); + } + + Ok(statements) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_multiple_statements_ast() { + let sql = concat!( + "CREATE TABLE t1 (id INT); ", + "CREATE STREAMING TABLE sk WITH ('connector' = 'kafka') AS SELECT id FROM t1", + ); + let stmts = parse_sql(sql).unwrap(); + assert_eq!(stmts.len(), 2); + assert!(matches!(stmts[0], DFStatement::CreateTable(_))); + assert!(matches!(stmts[1], DFStatement::CreateStreamingTable { .. })); + } + + #[test] + fn test_parse_empty() { + assert!(parse_sql("").is_err()); + assert!(parse_sql(" ").is_err()); + } + + #[test] + fn test_parse_select_yields_query_ast() { + let stmts = parse_sql("SELECT 1").unwrap(); + assert_eq!(stmts.len(), 1); + assert!(matches!(stmts[0], DFStatement::Query(_))); + } +} diff --git a/src/sql/physical/cdc/encode.rs b/src/streaming_planner/src/physical/cdc/encode.rs similarity index 97% rename from src/sql/physical/cdc/encode.rs rename to src/streaming_planner/src/physical/cdc/encode.rs index 65ec758d..8457724c 100644 --- a/src/sql/physical/cdc/encode.rs +++ b/src/streaming_planner/src/physical/cdc/encode.rs @@ -28,9 +28,9 @@ use datafusion::execution::{RecordBatchStream, SendableRecordBatchStream, TaskCo use datafusion::physical_plan::{DisplayAs, ExecutionPlan, PlanProperties}; use futures::{StreamExt, ready, stream::Stream}; -use crate::sql::common::constants::{cdc, debezium_op_short, physical_plan_node_name}; -use crate::sql::common::{TIMESTAMP_FIELD, UPDATING_META_FIELD}; -use crate::sql::physical::source_exec::make_stream_properties; +use crate::common::constants::{cdc, debezium_op_short, physical_plan_node_name}; +use crate::common::{TIMESTAMP_FIELD, UPDATING_META_FIELD}; +use crate::physical::source_exec::make_stream_properties; // ============================================================================ // CdcDebeziumPackExec (Execution Plan Node) @@ -85,7 +85,7 @@ impl CdcDebeziumPackExec { }) } - pub(crate) fn from_decoded_parts(input: Arc, schema: SchemaRef) -> Self { + pub fn from_decoded_parts(input: Arc, schema: SchemaRef) -> Self { Self { properties: make_stream_properties(schema.clone()), input, diff --git a/src/sql/physical/cdc/mod.rs b/src/streaming_planner/src/physical/cdc/mod.rs similarity index 100% rename from src/sql/physical/cdc/mod.rs rename to src/streaming_planner/src/physical/cdc/mod.rs diff --git a/src/sql/physical/cdc/unroll.rs b/src/streaming_planner/src/physical/cdc/unroll.rs similarity index 96% rename from src/sql/physical/cdc/unroll.rs rename to src/streaming_planner/src/physical/cdc/unroll.rs index 10c62c6c..963283cc 100644 --- a/src/sql/physical/cdc/unroll.rs +++ b/src/streaming_planner/src/physical/cdc/unroll.rs @@ -29,11 +29,11 @@ use datafusion::logical_expr::ColumnarValue; use datafusion::physical_plan::{DisplayAs, ExecutionPlan, PlanProperties}; use futures::{StreamExt, ready, stream::Stream}; -use crate::sql::common::TIMESTAMP_FIELD; -use crate::sql::common::constants::{cdc, debezium_op_short, physical_plan_node_name}; -use crate::sql::functions::MultiHashFunction; -use crate::sql::physical::meta::{updating_meta_field, updating_meta_fields}; -use crate::sql::physical::source_exec::make_stream_properties; +use crate::common::TIMESTAMP_FIELD; +use crate::common::constants::{cdc, debezium_op_short, physical_plan_node_name}; +use crate::functions::MultiHashFunction; +use crate::physical::meta::{updating_meta_field, updating_meta_fields}; +use crate::physical::source_exec::make_stream_properties; // ============================================================================ // CdcDebeziumUnrollExec (Execution Plan Node) @@ -103,7 +103,7 @@ impl CdcDebeziumUnrollExec { } /// Used when deserializing a plan with a pre-baked output schema (see [`StreamingExtensionCodec`]). - pub(crate) fn from_decoded_parts( + pub fn from_decoded_parts( input: Arc, schema: SchemaRef, primary_key_indices: Vec, diff --git a/src/sql/physical/codec.rs b/src/streaming_planner/src/physical/codec.rs similarity index 97% rename from src/sql/physical/codec.rs rename to src/streaming_planner/src/physical/codec.rs index 1b96a9d6..2fb6fcfc 100644 --- a/src/sql/physical/codec.rs +++ b/src/streaming_planner/src/physical/codec.rs @@ -27,13 +27,13 @@ use protocol::function_stream_graph::{ }; use tokio::sync::mpsc::UnboundedReceiver; -use crate::sql::analysis::UNNESTED_COL; -use crate::sql::common::constants::{mem_exec_join_side, window_function_udf}; -use crate::sql::physical::cdc::{CdcDebeziumPackExec, CdcDebeziumUnrollExec}; -use crate::sql::physical::source_exec::{ +use crate::analysis::UNNESTED_COL; +use crate::common::constants::{mem_exec_join_side, window_function_udf}; +use crate::physical::cdc::{CdcDebeziumPackExec, CdcDebeziumUnrollExec}; +use crate::physical::source_exec::{ BufferedBatchesExec, InjectableSingleBatchExec, MpscReceiverStreamExec, PlanningPlaceholderExec, }; -use crate::sql::physical::udfs::window; +use crate::physical::udfs::window; // ============================================================================ // StreamingExtensionCodec & StreamingDecodingContext diff --git a/src/sql/physical/meta.rs b/src/streaming_planner/src/physical/meta.rs similarity index 93% rename from src/sql/physical/meta.rs rename to src/streaming_planner/src/physical/meta.rs index 1387482c..ced37656 100644 --- a/src/sql/physical/meta.rs +++ b/src/streaming_planner/src/physical/meta.rs @@ -14,8 +14,8 @@ use std::sync::{Arc, OnceLock}; use datafusion::arrow::datatypes::{DataType, Field, Fields}; -use crate::sql::common::UPDATING_META_FIELD; -use crate::sql::common::constants::updating_state_field; +use crate::common::UPDATING_META_FIELD; +use crate::common::constants::updating_state_field; pub fn updating_meta_fields() -> Fields { static FIELDS: OnceLock = OnceLock::new(); diff --git a/src/sql/physical/mod.rs b/src/streaming_planner/src/physical/mod.rs similarity index 100% rename from src/sql/physical/mod.rs rename to src/streaming_planner/src/physical/mod.rs diff --git a/src/sql/physical/source_exec.rs b/src/streaming_planner/src/physical/source_exec.rs similarity index 96% rename from src/sql/physical/source_exec.rs rename to src/streaming_planner/src/physical/source_exec.rs index fa65cbfd..c94c1ca3 100644 --- a/src/sql/physical/source_exec.rs +++ b/src/streaming_planner/src/physical/source_exec.rs @@ -29,11 +29,11 @@ use futures::StreamExt; use tokio::sync::mpsc::UnboundedReceiver; use tokio_stream::wrappers::UnboundedReceiverStream; -use crate::sql::common::constants::physical_plan_node_name; +use crate::common::constants::physical_plan_node_name; /// Standard [`PlanProperties`] for a continuous, unbounded stream: incremental emission, /// unknown partitioning, and unbounded boundedness (without requiring infinite memory). -pub(crate) fn create_unbounded_stream_properties(schema: SchemaRef) -> PlanProperties { +pub fn create_unbounded_stream_properties(schema: SchemaRef) -> PlanProperties { PlanProperties::new( EquivalenceProperties::new(schema), Partitioning::UnknownPartitioning(1), @@ -45,7 +45,7 @@ pub(crate) fn create_unbounded_stream_properties(schema: SchemaRef) -> PlanPrope } /// Alias for call sites that still use the older name. -pub(crate) fn make_stream_properties(schema: SchemaRef) -> PlanProperties { +pub fn make_stream_properties(schema: SchemaRef) -> PlanProperties { create_unbounded_stream_properties(schema) } @@ -58,14 +58,14 @@ pub(crate) fn make_stream_properties(schema: SchemaRef) -> PlanProperties { /// For event-driven loops that receive a single batch from the network and run a DataFusion /// plan over it, the batch is stored in the lock until execution starts. #[derive(Debug)] -pub(crate) struct InjectableSingleBatchExec { +pub struct InjectableSingleBatchExec { schema: SchemaRef, injected_batch: Arc>>, properties: PlanProperties, } impl InjectableSingleBatchExec { - pub(crate) fn new( + pub fn new( schema: SchemaRef, injected_batch: Arc>>, ) -> Self { @@ -152,14 +152,14 @@ impl ExecutionPlan for InjectableSingleBatchExec { /// /// Bridges async producers (e.g. network threads) into a DataFusion pipeline. #[derive(Debug)] -pub(crate) struct MpscReceiverStreamExec { +pub struct MpscReceiverStreamExec { schema: SchemaRef, channel_receiver: Arc>>>, properties: PlanProperties, } impl MpscReceiverStreamExec { - pub(crate) fn new( + pub fn new( schema: SchemaRef, channel_receiver: Arc>>>, ) -> Self { @@ -243,14 +243,14 @@ impl ExecutionPlan for MpscReceiverStreamExec { /// Drains a growable, locked `Vec` when `execute` runs (micro-batching). #[derive(Debug)] -pub(crate) struct BufferedBatchesExec { +pub struct BufferedBatchesExec { schema: SchemaRef, buffered_batches: Arc>>, properties: PlanProperties, } impl BufferedBatchesExec { - pub(crate) fn new( + pub fn new( schema: SchemaRef, buffered_batches: Arc>>, ) -> Self { diff --git a/src/sql/physical/udfs.rs b/src/streaming_planner/src/physical/udfs.rs similarity index 97% rename from src/sql/physical/udfs.rs rename to src/streaming_planner/src/physical/udfs.rs index 767abf06..0a11ba4f 100644 --- a/src/sql/physical/udfs.rs +++ b/src/streaming_planner/src/physical/udfs.rs @@ -20,9 +20,9 @@ use datafusion::logical_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility, }; +use crate::common::constants::window_function_udf; use crate::make_udf_function; -use crate::sql::common::constants::window_function_udf; -use crate::sql::schema::utils::window_arrow_struct; +use crate::schema::utils::window_arrow_struct; // ============================================================================ // WindowFunctionUdf (User-Defined Scalar Function) diff --git a/src/sql/planning_runtime.rs b/src/streaming_planner/src/planning_runtime.rs similarity index 85% rename from src/sql/planning_runtime.rs rename to src/streaming_planner/src/planning_runtime.rs index dc4749ad..cb2ab93e 100644 --- a/src/sql/planning_runtime.rs +++ b/src/streaming_planner/src/planning_runtime.rs @@ -14,9 +14,9 @@ use std::sync::OnceLock; -use crate::config::streaming_job::ResolvedStreamingJobConfig; -use crate::sql::common::constants::sql_planning_default; -use crate::sql::types::SqlConfig; +use crate::common::constants::sql_planning_default; +use crate::types::SqlConfig; +use function_stream_config::streaming_job::ResolvedStreamingJobConfig; static SQL_PLANNING: OnceLock = OnceLock::new(); @@ -30,6 +30,6 @@ pub fn install_sql_planning_from_streaming_job(job: &ResolvedStreamingJobConfig) let _ = SQL_PLANNING.set(cfg).ok(); } -pub(crate) fn sql_planning_snapshot() -> SqlConfig { +pub fn sql_planning_snapshot() -> SqlConfig { SQL_PLANNING.get().cloned().unwrap_or_default() } diff --git a/src/sql/schema/catalog.rs b/src/streaming_planner/src/schema/catalog.rs similarity index 99% rename from src/sql/schema/catalog.rs rename to src/streaming_planner/src/schema/catalog.rs index 479df682..35966289 100644 --- a/src/sql/schema/catalog.rs +++ b/src/streaming_planner/src/schema/catalog.rs @@ -26,11 +26,11 @@ use super::column_descriptor::ColumnDescriptor; use super::data_encoding_format::DataEncodingFormat; use super::table::SqlSource; use super::temporal_pipeline_config::TemporalPipelineConfig; +use crate::common::constants::sql_field; +use crate::common::{Format, FsSchema}; +use crate::connector::config::ConnectorConfig; use crate::multifield_partial_ord; -use crate::sql::common::constants::sql_field; -use crate::sql::common::{Format, FsSchema}; -use crate::sql::connector::config::ConnectorConfig; -use crate::sql::types::ProcessingMode; +use crate::types::ProcessingMode; #[derive(Debug, Clone)] pub struct EngineDescriptor { diff --git a/src/sql/schema/column_descriptor.rs b/src/streaming_planner/src/schema/column_descriptor.rs similarity index 100% rename from src/sql/schema/column_descriptor.rs rename to src/streaming_planner/src/schema/column_descriptor.rs diff --git a/src/sql/schema/connection_type.rs b/src/streaming_planner/src/schema/connection_type.rs similarity index 100% rename from src/sql/schema/connection_type.rs rename to src/streaming_planner/src/schema/connection_type.rs diff --git a/src/sql/schema/data_encoding_format.rs b/src/streaming_planner/src/schema/data_encoding_format.rs similarity index 97% rename from src/sql/schema/data_encoding_format.rs rename to src/streaming_planner/src/schema/data_encoding_format.rs index 0b6f5e1d..b589d683 100644 --- a/src/sql/schema/data_encoding_format.rs +++ b/src/streaming_planner/src/schema/data_encoding_format.rs @@ -14,8 +14,8 @@ use datafusion::arrow::datatypes::{DataType, Field}; use datafusion::common::{Result, plan_err}; use super::column_descriptor::ColumnDescriptor; -use crate::sql::common::Format; -use crate::sql::common::constants::cdc; +use crate::common::Format; +use crate::common::constants::cdc; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] pub enum DataEncodingFormat { diff --git a/src/coordinator/plan/ddl_compiler.rs b/src/streaming_planner/src/schema/introspection/ddl_compiler.rs similarity index 88% rename from src/coordinator/plan/ddl_compiler.rs rename to src/streaming_planner/src/schema/introspection/ddl_compiler.rs index 5c5af39e..ba92da06 100644 --- a/src/coordinator/plan/ddl_compiler.rs +++ b/src/streaming_planner/src/schema/introspection/ddl_compiler.rs @@ -22,21 +22,22 @@ use datafusion::sql::TableReference; use datafusion::sql::planner::{PlannerContext, SqlToRel}; use datafusion::sql::sqlparser::ast; use datafusion::sql::sqlparser::ast::CreateTable as SqlCreateTable; +use datafusion::sql::sqlparser::ast::Statement as DfStatement; use datafusion_expr::ExprSchemable; use tracing::warn; -use super::ast_utils::AstUtils; -use crate::coordinator::tool::ConnectorOptions; -use crate::sql::analysis::StreamSchemaProvider; -use crate::sql::common::constants::{connection_table_role, connector_type, sql_field}; -use crate::sql::common::with_option_keys as opt; -use crate::sql::common::{BadData, Format, Framing, JsonCompression, JsonFormat}; -use crate::sql::connector::registry::REGISTRY; -use crate::sql::schema::ColumnDescriptor; -use crate::sql::schema::catalog::{ExternalTable, LookupTable, SourceTable}; -use crate::sql::schema::data_encoding_format::DataEncodingFormat; -use crate::sql::schema::table_role::{apply_adapter_specific_rules, validate_adapter_availability}; -use crate::sql::schema::temporal_pipeline_config::TemporalPipelineConfig; +use crate::common::ast_utils::AstUtils; +use crate::common::constants::{connection_table_role, connector_type, sql_field}; +use crate::common::with_option_keys as opt; +use crate::common::{BadData, ConnectorOptions, Format, Framing, JsonCompression, JsonFormat}; +use crate::connector::registry::REGISTRY; +use crate::connector::source::kafka::ensure_default_consumer_group; +use crate::schema::ColumnDescriptor; +use crate::schema::catalog::{ExternalTable, LookupTable, SourceTable}; +use crate::schema::data_encoding_format::DataEncodingFormat; +use crate::schema::schema_provider::StreamSchemaProvider; +use crate::schema::table_role::{apply_adapter_specific_rules, validate_adapter_availability}; +use crate::schema::temporal_pipeline_config::TemporalPipelineConfig; pub struct DdlCompiler<'a> { schema_provider: &'a StreamSchemaProvider, @@ -84,6 +85,9 @@ impl<'a> DdlCompiler<'a> { validate_adapter_availability(&adapter_type)?; let pk_constraints = AstUtils::parse_primary_keys(&stmt.constraints)?; + if adapter_type == connector_type::KAFKA { + ensure_default_consumer_group(&target_name, &mut options)?; + } let catalog_with_options = options.snapshot_for_catalog(); let format = Format::from_opts(&mut options)?; @@ -167,6 +171,9 @@ impl<'a> DdlCompiler<'a> { validate_adapter_availability(&adapter_type)?; let pk_constraints = AstUtils::parse_primary_keys(&stmt.constraints)?; + if adapter_type == connector_type::KAFKA { + ensure_default_consumer_group(&target_name, &mut options)?; + } let catalog_with_options = options.snapshot_for_catalog(); let connection_format = Format::from_opts(&mut options)?; @@ -293,6 +300,30 @@ impl<'a> DdlCompiler<'a> { } } +/// If `stmt` is `CREATE TABLE ... WITH (connector = ...)`, compile it to [`ExternalTable`]. +pub fn try_compile_connector_create_table( + schema_provider: &StreamSchemaProvider, + stmt: &DfStatement, +) -> Option> { + let DfStatement::CreateTable(ast_node) = stmt else { + return None; + }; + if ast_node.query.is_some() { + return None; + } + if !AstUtils::contains_connector_property(&ast_node.with_options) { + return None; + } + let declared_role = AstUtils::peek_table_role(&ast_node.with_options); + let if_not_exists = ast_node.if_not_exists; + let compiler = DdlCompiler::new(schema_provider); + Some( + compiler + .compile(ast_node, declared_role.as_deref()) + .map(|table| (table, if_not_exists)), + ) +} + fn resolve_source_watermark( table_identifier: &str, columns: &mut Vec, diff --git a/src/sql/schema/introspection/ddl_formatter.rs b/src/streaming_planner/src/schema/introspection/ddl_formatter.rs similarity index 99% rename from src/sql/schema/introspection/ddl_formatter.rs rename to src/streaming_planner/src/schema/introspection/ddl_formatter.rs index f4ce36e6..15d8f707 100644 --- a/src/sql/schema/introspection/ddl_formatter.rs +++ b/src/streaming_planner/src/schema/introspection/ddl_formatter.rs @@ -15,7 +15,7 @@ use std::fmt::{self, Write}; use datafusion::arrow::datatypes::{DataType, Schema, TimeUnit}; -use crate::sql::common::constants::sql_field; +use crate::common::constants::sql_field; pub struct DdlBuilder<'a> { table_name: &'a str, diff --git a/src/sql/schema/introspection/mod.rs b/src/streaming_planner/src/schema/introspection/mod.rs similarity index 90% rename from src/sql/schema/introspection/mod.rs rename to src/streaming_planner/src/schema/introspection/mod.rs index 1ba9c816..33efdf90 100644 --- a/src/sql/schema/introspection/mod.rs +++ b/src/streaming_planner/src/schema/introspection/mod.rs @@ -10,10 +10,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod ddl_compiler; pub mod ddl_formatter; pub mod show_formatter; pub mod stream_formatter; +pub use ddl_compiler::{DdlCompiler, try_compile_connector_create_table}; #[allow(unused_imports)] pub use ddl_formatter::{DdlBuilder, format_data_type, schema_columns_one_line}; pub use show_formatter::{catalog_table_row_detail, show_create_catalog_table}; diff --git a/src/sql/schema/introspection/show_formatter.rs b/src/streaming_planner/src/schema/introspection/show_formatter.rs similarity index 96% rename from src/sql/schema/introspection/show_formatter.rs rename to src/streaming_planner/src/schema/introspection/show_formatter.rs index 28a81ae9..3857ba21 100644 --- a/src/sql/schema/introspection/show_formatter.rs +++ b/src/streaming_planner/src/schema/introspection/show_formatter.rs @@ -10,9 +10,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sql::common::constants::connection_table_role; -use crate::sql::schema::catalog::ExternalTable; -use crate::sql::schema::table::CatalogEntity; +use crate::common::constants::connection_table_role; +use crate::schema::catalog::ExternalTable; +use crate::schema::table::CatalogEntity; use super::ddl_formatter::DdlBuilder; diff --git a/src/sql/schema/introspection/stream_formatter.rs b/src/streaming_planner/src/schema/introspection/stream_formatter.rs similarity index 95% rename from src/sql/schema/introspection/stream_formatter.rs rename to src/streaming_planner/src/schema/introspection/stream_formatter.rs index ebb02330..e3239ce8 100644 --- a/src/sql/schema/introspection/stream_formatter.rs +++ b/src/streaming_planner/src/schema/introspection/stream_formatter.rs @@ -14,9 +14,9 @@ use std::sync::Arc; use datafusion::arrow::datatypes::Schema; -use crate::sql::common::constants::connection_table_role; -use crate::sql::logical_node::logical::LogicalProgram; -use crate::sql::schema::schema_provider::StreamTable; +use crate::common::constants::connection_table_role; +use crate::logical_node::logical::LogicalProgram; +use crate::schema::schema_provider::StreamTable; use super::ddl_formatter::DdlBuilder; diff --git a/src/sql/schema/mod.rs b/src/streaming_planner/src/schema/mod.rs similarity index 88% rename from src/sql/schema/mod.rs rename to src/streaming_planner/src/schema/mod.rs index e11e4808..d383a205 100644 --- a/src/sql/schema/mod.rs +++ b/src/streaming_planner/src/schema/mod.rs @@ -24,7 +24,8 @@ pub mod utils; pub use catalog::{ExternalTable, LookupTable, SinkTable, SourceTable}; pub use column_descriptor::ColumnDescriptor; pub use introspection::{ - catalog_table_row_detail, schema_columns_one_line, show_create_catalog_table, + DdlCompiler, catalog_table_row_detail, schema_columns_one_line, show_create_catalog_table, + try_compile_connector_create_table, }; pub use schema_provider::{ObjectName, StreamPlanningContext, StreamSchemaProvider, StreamTable}; pub use table::CatalogEntity; diff --git a/src/sql/schema/schema_provider.rs b/src/streaming_planner/src/schema/schema_provider.rs similarity index 96% rename from src/sql/schema/schema_provider.rs rename to src/streaming_planner/src/schema/schema_provider.rs index 15cd58ee..75026c3f 100644 --- a/src/sql/schema/schema_provider.rs +++ b/src/streaming_planner/src/schema/schema_provider.rs @@ -27,11 +27,11 @@ use thiserror::Error; use tracing::{debug, error, info}; use unicase::UniCase; -use crate::sql::common::constants::{planning_placeholder_udf, window_fn}; -use crate::sql::logical_node::logical::{DylibUdfConfig, LogicalProgram}; -use crate::sql::schema::table::CatalogEntity; -use crate::sql::schema::utils::window_arrow_struct; -use crate::sql::types::{PlanningOptions, PlanningPlaceholderUdf, SqlConfig}; +use crate::common::constants::{planning_placeholder_udf, window_fn}; +use crate::logical_node::logical::{DylibUdfConfig, LogicalProgram}; +use crate::schema::table::CatalogEntity; +use crate::schema::utils::window_arrow_struct; +use crate::types::{PlanningOptions, PlanningPlaceholderUdf, SqlConfig}; pub type ObjectName = UniCase; @@ -119,7 +119,7 @@ impl TableProvider for LogicalBatchInput { _filters: &[Expr], _limit: Option, ) -> DataFusionResult> { - Ok(Arc::new(crate::sql::physical::FsMemExec::new( + Ok(Arc::new(crate::physical::FsMemExec::new( self.table_name.clone(), Arc::clone(&self.schema), ))) @@ -181,7 +181,7 @@ impl StreamPlanningContext { } pub fn new() -> Self { - let config = crate::sql::planning_runtime::sql_planning_snapshot(); + let config = crate::planning_runtime::sql_planning_snapshot(); Self::try_new(config).expect("StreamPlanningContext bootstrap") } @@ -264,7 +264,7 @@ impl StreamPlanningContext { .map(Arc::make_mut) } - pub fn get_async_udf_options(&self, _name: &str) -> Option { + pub fn get_async_udf_options(&self, _name: &str) -> Option { None } diff --git a/src/sql/schema/table.rs b/src/streaming_planner/src/schema/table.rs similarity index 94% rename from src/sql/schema/table.rs rename to src/streaming_planner/src/schema/table.rs index 6c001d9c..912e9b00 100644 --- a/src/sql/schema/table.rs +++ b/src/streaming_planner/src/schema/table.rs @@ -10,12 +10,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sql::analysis::rewrite_plan; -use crate::sql::logical_node::remote_table::RemoteTableBoundaryNode; -use crate::sql::logical_planner::optimizers::produce_optimized_plan; -use crate::sql::schema::StreamSchemaProvider; -use crate::sql::schema::catalog::ExternalTable; -use crate::sql::types::{ProcessingMode, QualifiedField}; +use crate::analysis::rewrite_plan; +use crate::logical_node::remote_table::RemoteTableBoundaryNode; +use crate::logical_planner::optimizers::produce_optimized_plan; +use crate::schema::StreamSchemaProvider; +use crate::schema::catalog::ExternalTable; +use crate::types::{ProcessingMode, QualifiedField}; use datafusion::arrow::datatypes::FieldRef; use datafusion::common::{Result, plan_err}; use datafusion::logical_expr::{Extension, LogicalPlan}; diff --git a/src/sql/schema/table_role.rs b/src/streaming_planner/src/schema/table_role.rs similarity index 97% rename from src/sql/schema/table_role.rs rename to src/streaming_planner/src/schema/table_role.rs index 7d301f9d..4574a6ea 100644 --- a/src/sql/schema/table_role.rs +++ b/src/streaming_planner/src/schema/table_role.rs @@ -18,10 +18,10 @@ use datafusion::error::DataFusionError; use super::column_descriptor::ColumnDescriptor; use super::connection_type::ConnectionType; -use crate::sql::common::constants::{ +use crate::common::constants::{ SUPPORTED_CONNECTOR_ADAPTERS, connection_table_role, connector_type, }; -use crate::sql::common::with_option_keys as opt; +use crate::common::with_option_keys as opt; /// Role of a connector-backed table in the pipeline (ingest / egress / lookup). #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] diff --git a/src/sql/schema/temporal_pipeline_config.rs b/src/streaming_planner/src/schema/temporal_pipeline_config.rs similarity index 97% rename from src/sql/schema/temporal_pipeline_config.rs rename to src/streaming_planner/src/schema/temporal_pipeline_config.rs index f672e552..db751b1c 100644 --- a/src/sql/schema/temporal_pipeline_config.rs +++ b/src/streaming_planner/src/schema/temporal_pipeline_config.rs @@ -16,7 +16,7 @@ use datafusion::common::{Result, plan_err}; use datafusion::logical_expr::Expr; use super::column_descriptor::ColumnDescriptor; -use crate::sql::common::constants::sql_field; +use crate::common::constants::sql_field; /// Event-time and watermark configuration for streaming tables. #[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] diff --git a/src/sql/schema/utils.rs b/src/streaming_planner/src/schema/utils.rs similarity index 95% rename from src/sql/schema/utils.rs rename to src/streaming_planner/src/schema/utils.rs index 45254e5f..be5efbab 100644 --- a/src/sql/schema/utils.rs +++ b/src/streaming_planner/src/schema/utils.rs @@ -16,8 +16,8 @@ use std::sync::Arc; use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit}; use datafusion::common::{DFSchema, DFSchemaRef, Result as DFResult, TableReference}; -use crate::sql::common::constants::window_interval_field; -use crate::sql::types::{QualifiedField, TIMESTAMP_FIELD}; +use crate::common::constants::window_interval_field; +use crate::types::{QualifiedField, TIMESTAMP_FIELD}; /// Returns the Arrow struct type for a window (start, end) pair. pub fn window_arrow_struct() -> DataType { diff --git a/src/sql/types/data_type.rs b/src/streaming_planner/src/types/data_type.rs similarity index 98% rename from src/sql/types/data_type.rs rename to src/streaming_planner/src/types/data_type.rs index 387a4190..44bb8087 100644 --- a/src/sql/types/data_type.rs +++ b/src/streaming_planner/src/types/data_type.rs @@ -17,8 +17,8 @@ use datafusion::arrow::datatypes::{ }; use datafusion::common::{Result, plan_datafusion_err, plan_err}; -use crate::sql::common::FsExtensionType; -use crate::sql::common::constants::planning_placeholder_udf; +use crate::common::FsExtensionType; +use crate::common::constants::planning_placeholder_udf; pub fn convert_data_type( sql_type: &datafusion::sql::sqlparser::ast::DataType, diff --git a/src/sql/types/df_field.rs b/src/streaming_planner/src/types/df_field.rs similarity index 100% rename from src/sql/types/df_field.rs rename to src/streaming_planner/src/types/df_field.rs diff --git a/src/sql/types/mod.rs b/src/streaming_planner/src/types/mod.rs similarity index 83% rename from src/sql/types/mod.rs rename to src/streaming_planner/src/types/mod.rs index d5124bcc..a796bfea 100644 --- a/src/sql/types/mod.rs +++ b/src/streaming_planner/src/types/mod.rs @@ -12,22 +12,22 @@ mod data_type; mod df_field; -pub(crate) mod placeholder_udf; +pub mod placeholder_udf; mod stream_schema; mod window; use std::time::Duration; -use crate::sql::common::constants::sql_planning_default; +use crate::common::constants::sql_planning_default; pub use df_field::{ QualifiedField, build_df_schema, build_df_schema_with_metadata, extract_qualified_fields, }; -pub(crate) use placeholder_udf::PlanningPlaceholderUdf; -pub(crate) use window::WindowBehavior; +pub use placeholder_udf::PlanningPlaceholderUdf; +pub use window::WindowBehavior; pub use window::{WindowType, extract_window_type}; -pub use crate::sql::common::constants::sql_field::TIMESTAMP_FIELD; +pub use crate::common::constants::sql_field::TIMESTAMP_FIELD; #[derive(Clone, Debug, Eq, PartialEq)] pub enum ProcessingMode { @@ -38,7 +38,7 @@ pub enum ProcessingMode { #[derive(Clone, Debug)] pub struct SqlConfig { pub default_parallelism: usize, - /// Physical pipeline parallelism for [`KeyExtractionNode`](crate::sql::logical_node::key_calculation::KeyExtractionNode) / KeyBy. + /// Physical pipeline parallelism for [`KeyExtractionNode`](crate::logical_node::key_calculation::KeyExtractionNode) / KeyBy. pub key_by_parallelism: usize, } diff --git a/src/sql/types/placeholder_udf.rs b/src/streaming_planner/src/types/placeholder_udf.rs similarity index 98% rename from src/sql/types/placeholder_udf.rs rename to src/streaming_planner/src/types/placeholder_udf.rs index 059637e9..2f4f3f3c 100644 --- a/src/sql/types/placeholder_udf.rs +++ b/src/streaming_planner/src/types/placeholder_udf.rs @@ -25,7 +25,7 @@ use datafusion::logical_expr::{ // ============================================================================ /// Logical-planning-only UDF: satisfies type checking until real functions are wired in. -pub(crate) struct PlanningPlaceholderUdf { +pub struct PlanningPlaceholderUdf { name: String, signature: Signature, return_type: DataType, diff --git a/src/sql/types/stream_schema.rs b/src/streaming_planner/src/types/stream_schema.rs similarity index 100% rename from src/sql/types/stream_schema.rs rename to src/streaming_planner/src/types/stream_schema.rs diff --git a/src/sql/types/window.rs b/src/streaming_planner/src/types/window.rs similarity index 98% rename from src/sql/types/window.rs rename to src/streaming_planner/src/types/window.rs index 1aa05f42..69401f51 100644 --- a/src/sql/types/window.rs +++ b/src/streaming_planner/src/types/window.rs @@ -16,7 +16,7 @@ use datafusion::common::{Result, ScalarValue, not_impl_err, plan_err}; use datafusion::logical_expr::Expr; use datafusion::logical_expr::expr::{Alias, ScalarFunction}; -use crate::sql::common::constants::window_fn; +use crate::common::constants::window_fn; use super::QualifiedField; @@ -35,7 +35,7 @@ pub enum WindowType { /// How windowing is represented in the physical plan. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) enum WindowBehavior { +pub enum WindowBehavior { FromOperator { window: WindowType, window_field: QualifiedField, diff --git a/src/streaming_runtime/Cargo.toml b/src/streaming_runtime/Cargo.toml new file mode 100644 index 00000000..061c5850 --- /dev/null +++ b/src/streaming_runtime/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "function-stream-streaming-runtime" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_streaming_runtime" +path = "src/lib.rs" + +[dependencies] +function-stream-streaming-runtime-core = { path = "../streaming_runtime_core" } +function-stream-streaming-runtime-job-manager = { path = "../streaming_runtime_job_manager" } +function-stream-streaming-runtime-operators = { path = "../streaming_runtime_operators" } diff --git a/src/lib.rs b/src/streaming_runtime/src/lib.rs similarity index 57% rename from src/lib.rs rename to src/streaming_runtime/src/lib.rs index a6bb4d28..801b25e1 100644 --- a/src/lib.rs +++ b/src/streaming_runtime/src/lib.rs @@ -10,14 +10,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Library crate for function-stream +//! Facade crate for the split streaming runtime libraries. -#![allow(dead_code)] +pub use function_stream_streaming_runtime_core::{ + StreamOutput, api, error, execution, network, protocol, state, +}; +pub use function_stream_streaming_runtime_job_manager::{job, stream_catalog}; +pub use function_stream_streaming_runtime_operators::{factory, format, operators, util}; -pub mod config; -pub mod coordinator; -pub mod logging; -pub mod runtime; -pub mod server; -pub mod sql; -pub mod storage; +pub const CRATE_NAME: &str = "function-stream-streaming-runtime"; diff --git a/src/streaming_runtime_core/Cargo.toml b/src/streaming_runtime_core/Cargo.toml new file mode 100644 index 00000000..16b76104 --- /dev/null +++ b/src/streaming_runtime_core/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "function-stream-streaming-runtime-core" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_streaming_runtime_core" +path = "src/lib.rs" + +[dependencies] +anyhow = "1.0" +async-trait = "0.1" +arrow = { version = "55", default-features = false } +arrow-array = "55" +arrow-schema = { version = "55", features = ["serde"] } +crossbeam-channel = "0.5" +function-stream-runtime-common = { path = "../runtime_common" } +function-stream-streaming-planner = { path = "../streaming_planner" } +parking_lot = "0.12" +parquet = "55" +protocol = { path = "../../protocol" } +serde = { version = "1.0", features = ["derive"] } +thiserror = "2" +tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "sync", "time", "net", "signal"] } +tokio-stream = "0.1.18" +tracing = "0.1" +uuid = { version = "1.0", features = ["v7"] } + +[dev-dependencies] +tempfile = "3.27.0" diff --git a/src/runtime/streaming/api/context.rs b/src/streaming_runtime_core/src/api/context.rs similarity index 95% rename from src/runtime/streaming/api/context.rs rename to src/streaming_runtime_core/src/api/context.rs index f2557e7a..a2db010c 100644 --- a/src/runtime/streaming/api/context.rs +++ b/src/streaming_runtime_core/src/api/context.rs @@ -19,11 +19,11 @@ use arrow_array::RecordBatch; use protocol::storage::SourceCheckpointInfo; use tokio::sync::mpsc; -use crate::runtime::memory::{MemoryBlock, MemoryPool, get_array_memory_size}; -use crate::runtime::streaming::network::endpoint::PhysicalSender; -use crate::runtime::streaming::protocol::control::JobMasterEvent; -use crate::runtime::streaming::protocol::event::{StreamEvent, TrackedEvent}; -use crate::runtime::streaming::state::IoManager; +use crate::memory::{MemoryBlock, MemoryPool, get_array_memory_size}; +use crate::network::endpoint::PhysicalSender; +use crate::protocol::control::JobMasterEvent; +use crate::protocol::event::{StreamEvent, TrackedEvent}; +use crate::state::IoManager; #[derive(Debug, Clone)] pub struct TaskContextConfig { @@ -33,7 +33,7 @@ pub struct TaskContextConfig { impl Default for TaskContextConfig { fn default() -> Self { Self { - source_idle_timeout: Duration::from_millis(50), + source_idle_timeout: Duration::from_millis(5), } } } diff --git a/src/runtime/streaming/api/mod.rs b/src/streaming_runtime_core/src/api/mod.rs similarity index 100% rename from src/runtime/streaming/api/mod.rs rename to src/streaming_runtime_core/src/api/mod.rs diff --git a/src/runtime/streaming/api/operator.rs b/src/streaming_runtime_core/src/api/operator.rs similarity index 93% rename from src/runtime/streaming/api/operator.rs rename to src/streaming_runtime_core/src/api/operator.rs index fc75e475..89e7d9f1 100644 --- a/src/runtime/streaming/api/operator.rs +++ b/src/streaming_runtime_core/src/api/operator.rs @@ -10,9 +10,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::source::SourceOperator; -use crate::runtime::streaming::protocol::event::StreamOutput; +use crate::api::context::TaskContext; +use crate::api::source::SourceOperator; +use crate::protocol::event::StreamOutput; use crate::sql::common::{CheckpointBarrier, Watermark}; use arrow_array::RecordBatch; use async_trait::async_trait; diff --git a/src/runtime/streaming/api/source.rs b/src/streaming_runtime_core/src/api/source.rs similarity index 98% rename from src/runtime/streaming/api/source.rs rename to src/streaming_runtime_core/src/api/source.rs index 9c531f2c..b79dc8c3 100644 --- a/src/runtime/streaming/api/source.rs +++ b/src/streaming_runtime_core/src/api/source.rs @@ -10,7 +10,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::streaming::api::context::TaskContext; +use crate::api::context::TaskContext; use crate::sql::common::{CheckpointBarrier, Watermark}; use arrow_array::RecordBatch; use async_trait::async_trait; diff --git a/src/runtime/streaming/error.rs b/src/streaming_runtime_core/src/error.rs similarity index 100% rename from src/runtime/streaming/error.rs rename to src/streaming_runtime_core/src/error.rs diff --git a/src/runtime/streaming/execution/mod.rs b/src/streaming_runtime_core/src/execution/mod.rs similarity index 100% rename from src/runtime/streaming/execution/mod.rs rename to src/streaming_runtime_core/src/execution/mod.rs diff --git a/src/runtime/streaming/execution/operator_chain.rs b/src/streaming_runtime_core/src/execution/operator_chain.rs similarity index 98% rename from src/runtime/streaming/execution/operator_chain.rs rename to src/streaming_runtime_core/src/execution/operator_chain.rs index 88e8f441..3de774d6 100644 --- a/src/runtime/streaming/execution/operator_chain.rs +++ b/src/streaming_runtime_core/src/execution/operator_chain.rs @@ -13,10 +13,10 @@ use anyhow::anyhow; use async_trait::async_trait; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::error::RunError; -use crate::runtime::streaming::protocol::{ +use crate::api::context::TaskContext; +use crate::api::operator::{Collector, Operator}; +use crate::error::RunError; +use crate::protocol::{ control::{ControlCommand, StopMode}, event::{StreamEvent, StreamOutput, TrackedEvent}, }; diff --git a/src/runtime/streaming/execution/pipeline.rs b/src/streaming_runtime_core/src/execution/pipeline.rs similarity index 94% rename from src/runtime/streaming/execution/pipeline.rs rename to src/streaming_runtime_core/src/execution/pipeline.rs index 7c2ca17a..69cbe6af 100644 --- a/src/runtime/streaming/execution/pipeline.rs +++ b/src/streaming_runtime_core/src/execution/pipeline.rs @@ -14,16 +14,16 @@ use tokio::sync::mpsc::UnboundedReceiver; use tokio_stream::{StreamExt, StreamMap}; use tracing::{Instrument, info, info_span}; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::Operator; -use crate::runtime::streaming::error::RunError; -use crate::runtime::streaming::execution::operator_chain::{ChainBuilder, OperatorDrive}; -use crate::runtime::streaming::execution::tracker::{ +use crate::api::context::TaskContext; +use crate::api::operator::Operator; +use crate::error::RunError; +use crate::execution::operator_chain::{ChainBuilder, OperatorDrive}; +use crate::execution::tracker::{ barrier_aligner::{AlignmentStatus, BarrierAligner}, watermark_tracker::WatermarkTracker, }; -use crate::runtime::streaming::network::endpoint::BoxedEventStream; -use crate::runtime::streaming::protocol::{ +use crate::network::endpoint::BoxedEventStream; +use crate::protocol::{ control::ControlCommand, event::{StreamEvent, TrackedEvent}, }; diff --git a/src/runtime/streaming/execution/source_driver.rs b/src/streaming_runtime_core/src/execution/source_driver.rs similarity index 95% rename from src/runtime/streaming/execution/source_driver.rs rename to src/streaming_runtime_core/src/execution/source_driver.rs index 0118c4ee..6d015f44 100644 --- a/src/runtime/streaming/execution/source_driver.rs +++ b/src/streaming_runtime_core/src/execution/source_driver.rs @@ -14,11 +14,11 @@ use tokio::sync::mpsc::UnboundedReceiver; use tokio::time::{Instant, sleep}; use tracing::{Instrument, info, info_span, warn}; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::source::{SourceCheckpointReport, SourceEvent, SourceOperator}; -use crate::runtime::streaming::error::RunError; -use crate::runtime::streaming::execution::OperatorDrive; -use crate::runtime::streaming::protocol::{ +use crate::api::context::TaskContext; +use crate::api::source::{SourceCheckpointReport, SourceEvent, SourceOperator}; +use crate::error::RunError; +use crate::execution::OperatorDrive; +use crate::protocol::{ control::ControlCommand, event::{StreamEvent, TrackedEvent}, }; diff --git a/src/runtime/streaming/execution/tracker/barrier_aligner.rs b/src/streaming_runtime_core/src/execution/tracker/barrier_aligner.rs similarity index 100% rename from src/runtime/streaming/execution/tracker/barrier_aligner.rs rename to src/streaming_runtime_core/src/execution/tracker/barrier_aligner.rs diff --git a/src/runtime/streaming/execution/tracker/mod.rs b/src/streaming_runtime_core/src/execution/tracker/mod.rs similarity index 100% rename from src/runtime/streaming/execution/tracker/mod.rs rename to src/streaming_runtime_core/src/execution/tracker/mod.rs diff --git a/src/runtime/streaming/execution/tracker/watermark_tracker.rs b/src/streaming_runtime_core/src/execution/tracker/watermark_tracker.rs similarity index 97% rename from src/runtime/streaming/execution/tracker/watermark_tracker.rs rename to src/streaming_runtime_core/src/execution/tracker/watermark_tracker.rs index af6fd0bc..3234e130 100644 --- a/src/runtime/streaming/execution/tracker/watermark_tracker.rs +++ b/src/streaming_runtime_core/src/execution/tracker/watermark_tracker.rs @@ -10,7 +10,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::streaming::protocol::event::{merge_watermarks, watermark_strictly_advances}; +use crate::protocol::event::{merge_watermarks, watermark_strictly_advances}; use crate::sql::common::Watermark; #[derive(Debug)] diff --git a/src/runtime/streaming/mod.rs b/src/streaming_runtime_core/src/lib.rs similarity index 81% rename from src/runtime/streaming/mod.rs rename to src/streaming_runtime_core/src/lib.rs index b092c85d..89f0b07b 100644 --- a/src/runtime/streaming/mod.rs +++ b/src/streaming_runtime_core/src/lib.rs @@ -1,6 +1,5 @@ // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. -// // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 @@ -11,16 +10,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Streaming actor runtime (vendored from Arroyo `arroyo-actor-runtime`). +//! Core streaming runtime building blocks. + +pub use function_stream_runtime_common::{common, memory}; +pub use function_stream_streaming_planner as sql; pub mod api; pub mod error; pub mod execution; -pub mod factory; -pub mod format; -pub mod job; pub mod network; -pub mod operators; pub mod protocol; pub mod state; diff --git a/src/runtime/streaming/network/endpoint.rs b/src/streaming_runtime_core/src/network/endpoint.rs similarity index 94% rename from src/runtime/streaming/network/endpoint.rs rename to src/streaming_runtime_core/src/network/endpoint.rs index ae75e6fc..10958228 100644 --- a/src/runtime/streaming/network/endpoint.rs +++ b/src/streaming_runtime_core/src/network/endpoint.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::streaming::protocol::event::StreamEvent; -use crate::runtime::streaming::protocol::event::TrackedEvent; +use crate::protocol::event::StreamEvent; +use crate::protocol::event::TrackedEvent; use anyhow::{Result, anyhow}; use std::pin::Pin; use tokio::sync::mpsc; diff --git a/src/runtime/streaming/network/environment.rs b/src/streaming_runtime_core/src/network/environment.rs similarity index 100% rename from src/runtime/streaming/network/environment.rs rename to src/streaming_runtime_core/src/network/environment.rs diff --git a/src/runtime/streaming/network/mod.rs b/src/streaming_runtime_core/src/network/mod.rs similarity index 100% rename from src/runtime/streaming/network/mod.rs rename to src/streaming_runtime_core/src/network/mod.rs diff --git a/src/runtime/streaming/protocol/control.rs b/src/streaming_runtime_core/src/protocol/control.rs similarity index 100% rename from src/runtime/streaming/protocol/control.rs rename to src/streaming_runtime_core/src/protocol/control.rs diff --git a/src/streaming_runtime_core/src/protocol/event.rs b/src/streaming_runtime_core/src/protocol/event.rs new file mode 100644 index 00000000..0db2135d --- /dev/null +++ b/src/streaming_runtime_core/src/protocol/event.rs @@ -0,0 +1,59 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use arrow_array::RecordBatch; + +use crate::memory::MemoryTicket; + +pub use function_stream_runtime_common::streaming_protocol::{ + CheckpointBarrier, Watermark, merge_watermarks, watermark_strictly_advances, +}; + +#[derive(Debug, Clone)] +pub enum StreamEvent { + Data(RecordBatch), + Watermark(Watermark), + Barrier(CheckpointBarrier), + EndOfStream, +} + +#[derive(Debug, Clone)] +pub enum StreamOutput { + Forward(RecordBatch), + Keyed(u64, RecordBatch), + Broadcast(RecordBatch), + Watermark(Watermark), +} + +#[derive(Debug, Clone)] +pub struct TrackedEvent { + pub event: StreamEvent, + pub _ticket: Option>, +} + +impl TrackedEvent { + pub fn new(event: StreamEvent, ticket: Option) -> Self { + Self { + event, + _ticket: ticket.map(Arc::new), + } + } + + pub fn control(event: StreamEvent) -> Self { + Self { + event, + _ticket: None, + } + } +} diff --git a/src/runtime/streaming/protocol/mod.rs b/src/streaming_runtime_core/src/protocol/mod.rs similarity index 96% rename from src/runtime/streaming/protocol/mod.rs rename to src/streaming_runtime_core/src/protocol/mod.rs index 28fd85a4..d41d9fcf 100644 --- a/src/runtime/streaming/protocol/mod.rs +++ b/src/streaming_runtime_core/src/protocol/mod.rs @@ -15,4 +15,5 @@ pub mod event; #[allow(unused_imports)] pub use control::{ControlCommand, JobMasterEvent, StopMode}; +#[allow(unused_imports)] pub use event::{CheckpointBarrier, StreamOutput, Watermark}; diff --git a/src/runtime/streaming/state/error.rs b/src/streaming_runtime_core/src/state/error.rs similarity index 97% rename from src/runtime/streaming/state/error.rs rename to src/streaming_runtime_core/src/state/error.rs index 37bc6481..cab033db 100644 --- a/src/runtime/streaming/state/error.rs +++ b/src/streaming_runtime_core/src/state/error.rs @@ -13,7 +13,7 @@ use crossbeam_channel::TrySendError; use thiserror::Error; -use crate::runtime::memory::MemoryAllocationError; +use crate::memory::MemoryAllocationError; #[derive(Error, Debug)] pub enum StateEngineError { diff --git a/src/runtime/streaming/state/io_manager.rs b/src/streaming_runtime_core/src/state/io_manager.rs similarity index 100% rename from src/runtime/streaming/state/io_manager.rs rename to src/streaming_runtime_core/src/state/io_manager.rs diff --git a/src/runtime/streaming/state/metrics.rs b/src/streaming_runtime_core/src/state/metrics.rs similarity index 100% rename from src/runtime/streaming/state/metrics.rs rename to src/streaming_runtime_core/src/state/metrics.rs diff --git a/src/runtime/streaming/state/mod.rs b/src/streaming_runtime_core/src/state/mod.rs similarity index 100% rename from src/runtime/streaming/state/mod.rs rename to src/streaming_runtime_core/src/state/mod.rs diff --git a/src/runtime/streaming/state/operator_state.rs b/src/streaming_runtime_core/src/state/operator_state.rs similarity index 99% rename from src/runtime/streaming/state/operator_state.rs rename to src/streaming_runtime_core/src/state/operator_state.rs index a3514461..224671f2 100644 --- a/src/runtime/streaming/state/operator_state.rs +++ b/src/streaming_runtime_core/src/state/operator_state.rs @@ -13,7 +13,7 @@ use super::error::{Result, StateEngineError}; use super::io_manager::{CompactJob, IoManager, SpillJob}; use super::metrics::StateMetricsCollector; -use crate::runtime::memory::{MemoryBlock, MemoryTicket}; +use crate::memory::{MemoryBlock, MemoryTicket}; use arrow_array::builder::{BinaryBuilder, BooleanBuilder, UInt64Builder}; use arrow_array::{Array, BinaryArray, RecordBatch, UInt64Array}; use arrow_schema::{DataType, Field, Schema}; @@ -813,7 +813,7 @@ mod tests { use super::super::io_manager::IoPool; use super::super::metrics::NoopMetricsCollector; use super::*; - use crate::runtime::memory::{MemoryBlock, MemoryPool, global_memory_pool}; + use crate::memory::{MemoryBlock, MemoryPool, global_memory_pool}; use arrow_array::Int64Array; use tempfile::TempDir; @@ -836,7 +836,7 @@ mod tests { const TEST_OPERATOR_MEMORY: u64 = 2 * 1024 * 1024; fn ensure_global_memory_pool() { - use crate::runtime::memory::{init_global_memory_pool, try_global_memory_pool}; + use crate::memory::{init_global_memory_pool, try_global_memory_pool}; use std::sync::Once; static INIT: Once = Once::new(); INIT.call_once(|| { diff --git a/src/streaming_runtime_job_manager/Cargo.toml b/src/streaming_runtime_job_manager/Cargo.toml new file mode 100644 index 00000000..af80046f --- /dev/null +++ b/src/streaming_runtime_job_manager/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "function-stream-streaming-runtime-job-manager" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_streaming_runtime_job_manager" +path = "src/lib.rs" + +[dependencies] +anyhow = "1.0" +function-stream-config = { path = "../config" } +function-stream-runtime-common = { path = "../runtime_common" } +function-stream-streaming-planner = { path = "../streaming_planner" } +function-stream-streaming-runtime-core = { path = "../streaming_runtime_core" } +function-stream-streaming-runtime-operators = { path = "../streaming_runtime_operators" } +protocol = { path = "../../protocol" } +tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "sync", "time", "net", "signal"] } +tokio-stream = "0.1.18" +tracing = "0.1" diff --git a/src/runtime/streaming/job/edge_manager.rs b/src/streaming_runtime_job_manager/src/job/edge_manager.rs similarity index 98% rename from src/runtime/streaming/job/edge_manager.rs rename to src/streaming_runtime_job_manager/src/job/edge_manager.rs index 00c94485..0cf7e551 100644 --- a/src/runtime/streaming/job/edge_manager.rs +++ b/src/streaming_runtime_job_manager/src/job/edge_manager.rs @@ -16,7 +16,7 @@ use anyhow::{Result, anyhow}; use tokio::sync::mpsc; use tracing::{debug, info, warn}; -use crate::runtime::streaming::protocol::event::TrackedEvent; +use crate::core::protocol::event::TrackedEvent; use protocol::function_stream_graph::{FsEdge, FsNode}; const DEFAULT_CHANNEL_CAPACITY: usize = 2048; diff --git a/src/runtime/streaming/job/job_manager.rs b/src/streaming_runtime_job_manager/src/job/job_manager.rs similarity index 96% rename from src/runtime/streaming/job/job_manager.rs rename to src/streaming_runtime_job_manager/src/job/job_manager.rs index e4b9916b..f6e94118 100644 --- a/src/runtime/streaming/job/job_manager.rs +++ b/src/streaming_runtime_job_manager/src/job/job_manager.rs @@ -30,22 +30,22 @@ use crate::config::{ DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_OPERATOR_STATE_STORE_MEMORY_BYTES, DEFAULT_PIPELINE_PARALLELISM, }; -use crate::runtime::memory::global_memory_pool; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{ConstructedOperator, Operator}; -use crate::runtime::streaming::api::source::SourceOperator; -use crate::runtime::streaming::execution::{ChainBuilder, Pipeline, SourceDriver}; -use crate::runtime::streaming::factory::OperatorFactory; -use crate::runtime::streaming::job::edge_manager::EdgeManager; -use crate::runtime::streaming::job::models::{ +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{ConstructedOperator, Operator}; +use crate::core::api::source::SourceOperator; +use crate::core::execution::{ChainBuilder, Pipeline, SourceDriver}; +use crate::core::network::endpoint::{BoxedEventStream, PhysicalSender}; +use crate::core::protocol::control::{ControlCommand, JobMasterEvent, StopMode}; +use crate::core::protocol::event::CheckpointBarrier; +use crate::core::state::{IoManager, IoPool, NoopMetricsCollector}; +use crate::factory::OperatorFactory; +use crate::job::edge_manager::EdgeManager; +use crate::job::models::{ PhysicalExecutionGraph, PhysicalPipeline, PipelineStatus, StreamingJobRollupStatus, }; -use crate::runtime::streaming::network::endpoint::{BoxedEventStream, PhysicalSender}; -use crate::runtime::streaming::protocol::control::{ControlCommand, JobMasterEvent, StopMode}; -use crate::runtime::streaming::protocol::event::CheckpointBarrier; -use crate::runtime::streaming::state::{IoManager, IoPool, NoopMetricsCollector}; +use crate::memory::global_memory_pool; use crate::sql::logical_node::logical::OperatorName; -use crate::storage::stream_catalog::CatalogManager; +use crate::stream_catalog::CatalogManager; #[derive(Debug, Clone)] pub struct StreamingJobSummary { @@ -80,7 +80,7 @@ pub struct StateConfig { pub pipeline_parallelism: u32, pub job_manager_control_plane_threads: u32, pub job_manager_data_plane_threads: u32, - /// Total bytes shared by all [`crate::runtime::streaming::state::OperatorStateStore`] (global pool). + /// Total bytes shared by all [`crate::core::state::OperatorStateStore`] (global pool). pub per_operator_memory_bytes: u64, } @@ -103,7 +103,7 @@ impl Default for StateConfig { static GLOBAL_JOB_MANAGER: OnceLock> = OnceLock::new(); -/// Operators that create an [`crate::runtime::streaming::state::OperatorStateStore`] at runtime. +/// Operators that create an [`crate::core::state::OperatorStateStore`] at runtime. fn pipeline_state_store_operator_count(operators: &[ChainedOperator]) -> usize { operators .iter() @@ -159,7 +159,7 @@ struct CheckpointCoordinatorConfig { } impl PipelineRunner { - async fn run(self) -> Result<(), crate::runtime::streaming::error::RunError> { + async fn run(self) -> Result<(), crate::core::error::RunError> { match self { PipelineRunner::Source(driver) => driver.run().await, PipelineRunner::Standard(pipeline) => pipeline.run().await, diff --git a/src/runtime/streaming/job/mod.rs b/src/streaming_runtime_job_manager/src/job/mod.rs similarity index 75% rename from src/runtime/streaming/job/mod.rs rename to src/streaming_runtime_job_manager/src/job/mod.rs index 59d5c61f..eaf1cc72 100644 --- a/src/runtime/streaming/job/mod.rs +++ b/src/streaming_runtime_job_manager/src/job/mod.rs @@ -14,4 +14,7 @@ pub mod edge_manager; pub mod job_manager; pub mod models; -pub use job_manager::{JobManager, StateConfig, StreamingJobSummary}; +pub use crate::stream_catalog::{CheckpointCatalog, install_global_checkpoint_catalog}; +pub use job_manager::{ + JobManager, PipelineDetail, StateConfig, StreamingJobDetail, StreamingJobSummary, +}; diff --git a/src/runtime/streaming/job/models.rs b/src/streaming_runtime_job_manager/src/job/models.rs similarity index 98% rename from src/runtime/streaming/job/models.rs rename to src/streaming_runtime_job_manager/src/job/models.rs index e81649f2..3f8ce8c4 100644 --- a/src/runtime/streaming/job/models.rs +++ b/src/streaming_runtime_job_manager/src/job/models.rs @@ -19,7 +19,7 @@ use protocol::function_stream_graph::FsProgram; use tokio::sync::mpsc; use tokio::task::JoinHandle; -use crate::runtime::streaming::protocol::control::ControlCommand; +use crate::core::protocol::control::ControlCommand; #[derive(Debug, Clone, PartialEq)] pub enum PipelineStatus { diff --git a/src/streaming_runtime_job_manager/src/lib.rs b/src/streaming_runtime_job_manager/src/lib.rs new file mode 100644 index 00000000..400b07c2 --- /dev/null +++ b/src/streaming_runtime_job_manager/src/lib.rs @@ -0,0 +1,25 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! JobManager crate for streaming runtime orchestration. + +pub use function_stream_config as config; +pub use function_stream_runtime_common::{common, memory}; +pub use function_stream_streaming_planner as sql; +pub use function_stream_streaming_runtime_core as core; +pub use function_stream_streaming_runtime_core::{ + StreamOutput, api, error, execution, network, protocol, state, +}; +pub use function_stream_streaming_runtime_operators::{factory, format, operators, util}; + +pub mod job; +pub mod stream_catalog; diff --git a/src/streaming_runtime_job_manager/src/stream_catalog.rs b/src/streaming_runtime_job_manager/src/stream_catalog.rs new file mode 100644 index 00000000..2dcab934 --- /dev/null +++ b/src/streaming_runtime_job_manager/src/stream_catalog.rs @@ -0,0 +1,49 @@ +use std::sync::{Arc, OnceLock}; + +use anyhow::{Result, anyhow}; +use protocol::storage::SourceCheckpointInfo; + +pub trait CheckpointCatalog: Send + Sync { + fn commit_job_checkpoint( + &self, + job_id: &str, + epoch: u64, + source_infos: Vec, + ) -> Result<()>; +} + +static GLOBAL_CHECKPOINT_CATALOG: OnceLock> = OnceLock::new(); + +pub fn install_global_checkpoint_catalog(catalog: Arc) -> Result<()> { + GLOBAL_CHECKPOINT_CATALOG + .set(catalog) + .map_err(|_| anyhow!("CheckpointCatalog singleton already initialized")) +} + +#[derive(Clone)] +pub struct CatalogHandle { + inner: Arc, +} + +pub struct CatalogManager; + +impl CatalogManager { + pub fn try_global() -> Option { + GLOBAL_CHECKPOINT_CATALOG + .get() + .cloned() + .map(|inner| CatalogHandle { inner }) + } +} + +impl CatalogHandle { + pub fn commit_job_checkpoint( + &self, + job_id: &str, + epoch: u64, + source_infos: Vec, + ) -> Result<()> { + self.inner + .commit_job_checkpoint(job_id, epoch, source_infos) + } +} diff --git a/src/streaming_runtime_operators/Cargo.toml b/src/streaming_runtime_operators/Cargo.toml new file mode 100644 index 00000000..9445d64b --- /dev/null +++ b/src/streaming_runtime_operators/Cargo.toml @@ -0,0 +1,59 @@ +[package] +name = "function-stream-streaming-runtime-operators" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_streaming_runtime_operators" +path = "src/lib.rs" + +[dependencies] +ahash = "0.8" +anyhow = "1.0" +apache-avro = "0.21" +arrow = { version = "55", default-features = false } +arrow-array = "55" +arrow-array-lance = { package = "arrow-array", version = "57.3.0" } +arrow-ipc = "55" +arrow-ipc-lance = { package = "arrow-ipc", version = "57.3.0" } +arrow-json = { version = "55.2.0" } +arrow-schema = { version = "55", features = ["serde"] } +async-trait = "0.1" +base64 = "0.22" +bincode = { version = "2", features = ["serde"] } +bytes = "1" +datafusion = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-common = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-execution = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-expr = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-physical-expr = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +datafusion-proto = { git = "https://github.com/FunctionStream/datafusion", branch = "48.0.1/fs" } +deltalake = { version = "0.32.3", default-features = false, features = ["rustls", "s3"] } +function-stream-config = { path = "../config" } +function-stream-runtime-common = { path = "../runtime_common" } +function-stream-streaming-planner = { path = "../streaming_planner" } +function-stream-streaming-runtime-core = { path = "../streaming_runtime_core" } +futures = "0.3" +governor = "0.8.0" +itertools = "0.14" +lance = { version = "4.0.0", default-features = false, features = ["aws"] } +object_store = { version = "0.12.5", features = ["aws"] } +parquet = "55" +prost = "0.13" +protocol = { path = "../../protocol" } +rdkafka = { version = "0.38", features = ["cmake-build", "ssl", "gssapi", "curl"] } +rumqttc = "0.24" +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } +axum = "0.7" +tokio-tungstenite = { version = "0.24", features = ["rustls-tls-webpki-roots"] } +mcap = "0.19" +rosbag = "0.6.3" +lzf = "1.0.0" +byteorder = "1.5" +rusqlite = { version = "0.32", features = ["bundled"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +thiserror = "2" +tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "sync", "time", "net", "signal", "fs", "io-util"] } +tracing = "0.1" +url = "2" diff --git a/src/runtime/streaming/factory/connector/delta.rs b/src/streaming_runtime_operators/src/factory/connector/delta.rs similarity index 79% rename from src/runtime/streaming/factory/connector/delta.rs rename to src/streaming_runtime_operators/src/factory/connector/delta.rs index 726f87ef..a44e5464 100644 --- a/src/runtime/streaming/factory/connector/delta.rs +++ b/src/streaming_runtime_operators/src/factory/connector/delta.rs @@ -18,14 +18,17 @@ use prost::Message; use protocol::function_stream_graph::ConnectorOp; use protocol::function_stream_graph::connector_op::Config; -use crate::runtime::streaming::api::operator::ConstructedOperator; -use crate::runtime::streaming::factory::connector::sink_props_codec::{ +use crate::core::api::operator::ConstructedOperator; +use crate::factory::connector::sink_props_codec::{ apply_common_sink_fields, normalized_props, parse_sink_memory_bytes, }; -use crate::runtime::streaming::factory::global::Registry; -use crate::runtime::streaming::factory::operator_constructor::OperatorConstructor; -use crate::runtime::streaming::operators::sink::delta::{DeltaFormat, DeltaSinkOperator}; -use crate::runtime::streaming::operators::sink::filesystem::compression_from_str; +use crate::factory::global::Registry; +use crate::factory::operator_constructor::OperatorConstructor; +use crate::operators::sink::delta::{ + DeltaFormat, DeltaSinkOperator, strip_streaming_system_columns_arc, +}; +use crate::operators::sink::filesystem::compression_from_str; +use crate::sql::common::FsSchema; use crate::sql::common::constants::connection_format_value; use crate::sql::common::with_option_keys as opt; @@ -40,6 +43,14 @@ impl OperatorConstructor for DeltaSinkDispatcher { }; let props = normalized_props(props); + let catalog_schema = op + .fs_schema + .as_ref() + .map(|fs| FsSchema::try_from(fs.clone())) + .transpose() + .map_err(|e| anyhow::anyhow!("invalid fs_schema for delta sink: {e}"))? + .and_then(|fs| strip_streaming_system_columns_arc(fs.schema)); + let format = props .get(opt::FORMAT) .map(String::as_str) @@ -69,6 +80,7 @@ impl OperatorConstructor for DeltaSinkDispatcher { compression, sink_memory_bytes, props, + catalog_schema, )?, ))) } diff --git a/src/runtime/streaming/factory/connector/dispatchers.rs b/src/streaming_runtime_operators/src/factory/connector/dispatchers.rs similarity index 75% rename from src/runtime/streaming/factory/connector/dispatchers.rs rename to src/streaming_runtime_operators/src/factory/connector/dispatchers.rs index 7d626600..c8743355 100644 --- a/src/runtime/streaming/factory/connector/dispatchers.rs +++ b/src/streaming_runtime_operators/src/factory/connector/dispatchers.rs @@ -16,14 +16,16 @@ use anyhow::{Context, Result, bail}; use prost::Message; use protocol::function_stream_graph::ConnectorOp; -use crate::runtime::streaming::api::operator::ConstructedOperator; -use crate::runtime::streaming::factory::global::Registry; -use crate::runtime::streaming::factory::operator_constructor::OperatorConstructor; +use crate::core::api::operator::ConstructedOperator; +use crate::factory::global::Registry; +use crate::factory::operator_constructor::OperatorConstructor; use crate::sql::common::constants::connector_type; use super::{ DeltaSinkDispatcher, FilesystemSinkDispatcher, IcebergSinkDispatcher, LanceDbSinkDispatcher, - S3SinkDispatcher, kafka::KafkaConnectorDispatcher, + S3SinkDispatcher, http::HttpSourceDispatcher, kafka::KafkaConnectorDispatcher, + mqtt::MqttSourceDispatcher, robot_bag::RobotBagSourceDispatcher, + ros::RosSourceDispatcher, }; pub struct ConnectorSourceDispatcher; @@ -34,6 +36,11 @@ impl OperatorConstructor for ConnectorSourceDispatcher { .context("failed decoding connector op for source dispatch")?; match op.connector.to_ascii_lowercase().as_str() { connector_type::KAFKA => KafkaConnectorDispatcher.with_config(config, registry), + connector_type::MQTT => MqttSourceDispatcher.with_config(config, registry), + connector_type::HTTP => HttpSourceDispatcher.with_config(config, registry), + connector_type::ROS => RosSourceDispatcher.with_config(config, registry), + connector_type::ROS2 => RosSourceDispatcher.with_config(config, registry), + connector_type::ROBOT_BAG => RobotBagSourceDispatcher.with_config(config, registry), _ => bail!("unsupported source connector '{}'", op.connector), } } diff --git a/src/runtime/streaming/factory/connector/filesystem.rs b/src/streaming_runtime_operators/src/factory/connector/filesystem.rs similarity index 89% rename from src/runtime/streaming/factory/connector/filesystem.rs rename to src/streaming_runtime_operators/src/factory/connector/filesystem.rs index 94101407..39696a7a 100644 --- a/src/runtime/streaming/factory/connector/filesystem.rs +++ b/src/streaming_runtime_operators/src/factory/connector/filesystem.rs @@ -18,13 +18,13 @@ use prost::Message; use protocol::function_stream_graph::ConnectorOp; use protocol::function_stream_graph::connector_op::Config; -use crate::runtime::streaming::api::operator::ConstructedOperator; -use crate::runtime::streaming::factory::connector::sink_props_codec::{ +use crate::core::api::operator::ConstructedOperator; +use crate::factory::connector::sink_props_codec::{ apply_common_sink_fields, normalized_props, parse_sink_memory_bytes, }; -use crate::runtime::streaming::factory::global::Registry; -use crate::runtime::streaming::factory::operator_constructor::OperatorConstructor; -use crate::runtime::streaming::operators::sink::filesystem::{ +use crate::factory::global::Registry; +use crate::factory::operator_constructor::OperatorConstructor; +use crate::operators::sink::filesystem::{ FilesystemFormat, FilesystemSinkOperator, compression_from_str, }; use crate::sql::common::constants::connection_format_value; diff --git a/src/streaming_runtime_operators/src/factory/connector/http.rs b/src/streaming_runtime_operators/src/factory/connector/http.rs new file mode 100644 index 00000000..37d29347 --- /dev/null +++ b/src/streaming_runtime_operators/src/factory/connector/http.rs @@ -0,0 +1,153 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use anyhow::{Context, Result, bail}; +use prost::Message; +use reqwest::Method; +use std::num::NonZeroU32; +use std::sync::Arc; +use std::time::Duration; +use tracing::info; + +use protocol::function_stream_graph::connector_op::Config; +use protocol::function_stream_graph::{ + BadDataPolicy, ConnectorOp, DecimalEncodingProto, FormatConfig, HttpSourceConfig, + TimestampFormatProto, +}; + +use crate::core::api::operator::ConstructedOperator; +use crate::factory::global::Registry; +use crate::factory::operator_constructor::OperatorConstructor; +use crate::format::{ + BadDataPolicy as RtBadDataPolicy, DecimalEncoding as RtDecimalEncoding, + Format as RuntimeFormat, JsonFormat as RuntimeJsonFormat, TimestampFormat as RtTimestampFormat, +}; +use crate::operators::source::http::{ + HttpSourceOperator, proto_mode_to_runtime, proto_split_to_runtime, +}; +use crate::operators::source::kafka::BufferedDeserializer; +use crate::sql::common::FsSchema; + +const DEFAULT_SOURCE_BATCH_SIZE: usize = 1024; + +fn proto_format_to_runtime(fmt: &Option) -> Result { + let cfg = fmt.as_ref().context("FormatConfig is required")?; + match &cfg.format { + Some(protocol::function_stream_graph::format_config::Format::Json(j)) => { + Ok(RuntimeFormat::Json(RuntimeJsonFormat { + timestamp_format: match j.timestamp_format() { + TimestampFormatProto::TimestampRfc3339 => RtTimestampFormat::RFC3339, + TimestampFormatProto::TimestampUnixMillis => RtTimestampFormat::UnixMillis, + }, + decimal_encoding: match j.decimal_encoding() { + DecimalEncodingProto::DecimalNumber => RtDecimalEncoding::Number, + DecimalEncodingProto::DecimalString => RtDecimalEncoding::String, + DecimalEncodingProto::DecimalBytes => RtDecimalEncoding::Bytes, + }, + include_schema: j.include_schema, + })) + } + Some(protocol::function_stream_graph::format_config::Format::RawString(_)) => { + Ok(RuntimeFormat::RawString) + } + Some(protocol::function_stream_graph::format_config::Format::RawBytes(_)) => { + Ok(RuntimeFormat::RawBytes) + } + None => bail!("FormatConfig has no format variant set"), + } +} + +fn proto_bad_data_to_runtime(policy: i32) -> RtBadDataPolicy { + match BadDataPolicy::try_from(policy) { + Ok(BadDataPolicy::BadDataDrop) => RtBadDataPolicy::Drop, + _ => RtBadDataPolicy::Fail, + } +} + +fn parse_method(method: &str) -> Result { + Method::from_bytes(method.as_bytes()) + .with_context(|| format!("unsupported HTTP method '{method}'")) +} + +pub struct HttpSourceDispatcher; + +impl OperatorConstructor for HttpSourceDispatcher { + fn with_config(&self, payload: &[u8], _registry: Arc) -> Result { + let op = ConnectorOp::decode(payload).context("Failed to decode ConnectorOp protobuf")?; + + let fs_schema = op + .fs_schema + .as_ref() + .map(|fs| FsSchema::try_from(fs.clone())) + .transpose() + .map_err(|e| anyhow::anyhow!("{e}"))?; + + match op.config { + Some(Config::HttpSource(ref cfg)) => { + Self::build_http_source(&op.name, cfg, fs_schema) + } + Some(_) => bail!( + "ConnectorOp '{}': received non-http connector config for HTTP dispatcher", + op.name + ), + None => bail!("ConnectorOp '{}' has no configuration payload", op.name), + } + } +} + +impl HttpSourceDispatcher { + fn build_http_source( + _name: &str, + cfg: &HttpSourceConfig, + fs_schema: Option, + ) -> Result { + let mode = proto_mode_to_runtime(cfg.mode); + info!(?mode, "Constructing HTTP Source"); + + let fs = fs_schema.context("fs_schema is required for HTTP Source")?; + let runtime_format = proto_format_to_runtime(&cfg.format)?; + let bad_data = proto_bad_data_to_runtime(cfg.bad_data_policy); + + let deserializer = Box::new(BufferedDeserializer::new( + runtime_format, + fs.schema.clone(), + bad_data, + DEFAULT_SOURCE_BATCH_SIZE, + )); + + let rate_limit = (cfg.rate_limit_msgs_per_sec > 0) + .then(|| NonZeroU32::new(cfg.rate_limit_msgs_per_sec).expect("positive rate limit")); + + let scan_interval = + Duration::from_millis(cfg.scan_interval_ms.max(1)); + let request_timeout = + Duration::from_millis(cfg.request_timeout_ms.max(1) as u64); + + let source_op = HttpSourceOperator::new( + mode, + cfg.url.clone(), + parse_method(&cfg.method)?, + scan_interval, + cfg.request_body.clone(), + cfg.headers.clone(), + cfg.listen_host.clone(), + cfg.listen_port.clamp(1, u16::MAX as u32) as u16, + cfg.webhook_path.clone(), + request_timeout, + proto_split_to_runtime(cfg.response_split), + rate_limit, + deserializer, + ); + + Ok(ConstructedOperator::Source(Box::new(source_op))) + } +} diff --git a/src/runtime/streaming/factory/connector/iceberg.rs b/src/streaming_runtime_operators/src/factory/connector/iceberg.rs similarity index 86% rename from src/runtime/streaming/factory/connector/iceberg.rs rename to src/streaming_runtime_operators/src/factory/connector/iceberg.rs index 58e0809f..dc8f2cb6 100644 --- a/src/runtime/streaming/factory/connector/iceberg.rs +++ b/src/streaming_runtime_operators/src/factory/connector/iceberg.rs @@ -18,14 +18,14 @@ use prost::Message; use protocol::function_stream_graph::ConnectorOp; use protocol::function_stream_graph::connector_op::Config; -use crate::runtime::streaming::api::operator::ConstructedOperator; -use crate::runtime::streaming::factory::connector::sink_props_codec::{ +use crate::core::api::operator::ConstructedOperator; +use crate::factory::connector::sink_props_codec::{ apply_common_sink_fields, normalized_props, parse_sink_memory_bytes, }; -use crate::runtime::streaming::factory::global::Registry; -use crate::runtime::streaming::factory::operator_constructor::OperatorConstructor; -use crate::runtime::streaming::operators::sink::filesystem::compression_from_str; -use crate::runtime::streaming::operators::sink::iceberg::{IcebergFormat, IcebergSinkOperator}; +use crate::factory::global::Registry; +use crate::factory::operator_constructor::OperatorConstructor; +use crate::operators::sink::filesystem::compression_from_str; +use crate::operators::sink::iceberg::{IcebergFormat, IcebergSinkOperator}; use crate::sql::common::constants::connection_format_value; use crate::sql::common::with_option_keys as opt; diff --git a/src/runtime/streaming/factory/connector/kafka.rs b/src/streaming_runtime_operators/src/factory/connector/kafka.rs similarity index 92% rename from src/runtime/streaming/factory/connector/kafka.rs rename to src/streaming_runtime_operators/src/factory/connector/kafka.rs index 17838e3e..a9747665 100644 --- a/src/runtime/streaming/factory/connector/kafka.rs +++ b/src/streaming_runtime_operators/src/factory/connector/kafka.rs @@ -13,7 +13,6 @@ use anyhow::{Context, Result, bail}; use prost::Message; use std::collections::HashMap; -use std::num::NonZeroU32; use std::sync::Arc; use protocol::function_stream_graph::connector_op::Config; @@ -24,22 +23,19 @@ use protocol::function_stream_graph::{ }; use tracing::info; -use crate::runtime::streaming::api::operator::ConstructedOperator; -use crate::runtime::streaming::api::source::SourceOffset; -use crate::runtime::streaming::factory::global::Registry; -use crate::runtime::streaming::factory::operator_constructor::OperatorConstructor; -use crate::runtime::streaming::format::{ +use crate::core::api::operator::ConstructedOperator; +use crate::core::api::source::SourceOffset; +use crate::factory::global::Registry; +use crate::factory::operator_constructor::OperatorConstructor; +use crate::format::{ BadDataPolicy as RtBadDataPolicy, DataSerializer, DecimalEncoding as RtDecimalEncoding, Format as RuntimeFormat, JsonFormat as RuntimeJsonFormat, TimestampFormat as RtTimestampFormat, }; -use crate::runtime::streaming::operators::sink::kafka::{ConsistencyMode, KafkaSinkOperator}; -use crate::runtime::streaming::operators::source::kafka::{ - BufferedDeserializer, KafkaSourceOperator, -}; +use crate::operators::sink::kafka::{ConsistencyMode, KafkaSinkOperator}; +use crate::operators::source::batch_buffer::DEFAULT_SOURCE_BATCH_SIZE; +use crate::operators::source::kafka::{BufferedDeserializer, KafkaSourceOperator}; use crate::sql::common::FsSchema; -const DEFAULT_SOURCE_BATCH_SIZE: usize = 1024; - // ─────────────── Proto → Runtime type conversions ─────────────── fn proto_format_to_runtime(fmt: &Option) -> Result { @@ -171,9 +167,6 @@ impl KafkaConnectorDispatcher { DEFAULT_SOURCE_BATCH_SIZE, )); - let rate = NonZeroU32::new(cfg.rate_limit_msgs_per_sec.max(1)) - .unwrap_or_else(|| NonZeroU32::new(1_000_000).expect("nonzero")); - let source_op = KafkaSourceOperator::new( cfg.topic.clone(), cfg.bootstrap_servers.clone(), @@ -181,7 +174,6 @@ impl KafkaConnectorDispatcher { cfg.group_id_prefix.clone(), proto_offset_to_runtime(cfg.offset_mode), final_configs, - rate, vec![], deserializer, ); diff --git a/src/runtime/streaming/factory/connector/lancedb.rs b/src/streaming_runtime_operators/src/factory/connector/lancedb.rs similarity index 92% rename from src/runtime/streaming/factory/connector/lancedb.rs rename to src/streaming_runtime_operators/src/factory/connector/lancedb.rs index ad8bc246..3df7ae36 100644 --- a/src/runtime/streaming/factory/connector/lancedb.rs +++ b/src/streaming_runtime_operators/src/factory/connector/lancedb.rs @@ -18,13 +18,11 @@ use prost::Message; use protocol::function_stream_graph::ConnectorOp; use protocol::function_stream_graph::connector_op::Config; -use crate::runtime::streaming::api::operator::ConstructedOperator; -use crate::runtime::streaming::factory::connector::sink_props_codec::{ - apply_common_sink_fields, normalized_props, -}; -use crate::runtime::streaming::factory::global::Registry; -use crate::runtime::streaming::factory::operator_constructor::OperatorConstructor; -use crate::runtime::streaming::operators::sink::lancedb::LanceDbSinkOperator; +use crate::core::api::operator::ConstructedOperator; +use crate::factory::connector::sink_props_codec::{apply_common_sink_fields, normalized_props}; +use crate::factory::global::Registry; +use crate::factory::operator_constructor::OperatorConstructor; +use crate::operators::sink::lancedb::LanceDbSinkOperator; use crate::sql::common::constants::connection_format_value; use crate::sql::common::with_option_keys as opt; diff --git a/src/runtime/streaming/factory/connector/mod.rs b/src/streaming_runtime_operators/src/factory/connector/mod.rs similarity index 82% rename from src/runtime/streaming/factory/connector/mod.rs rename to src/streaming_runtime_operators/src/factory/connector/mod.rs index d1a6d7f3..cddb9344 100644 --- a/src/runtime/streaming/factory/connector/mod.rs +++ b/src/streaming_runtime_operators/src/factory/connector/mod.rs @@ -14,7 +14,11 @@ pub mod delta; mod dispatchers; pub mod filesystem; pub mod iceberg; +pub mod http; pub mod kafka; +pub mod mqtt; +pub mod robot_bag; +pub mod ros; pub mod lancedb; pub mod s3; pub mod sink_props_codec; @@ -24,5 +28,9 @@ pub use dispatchers::{ConnectorSinkDispatcher, ConnectorSourceDispatcher}; pub use filesystem::FilesystemSinkDispatcher; pub use iceberg::IcebergSinkDispatcher; pub use kafka::KafkaConnectorDispatcher; +pub use http::HttpSourceDispatcher; +pub use mqtt::MqttSourceDispatcher; +pub use robot_bag::RobotBagSourceDispatcher; +pub use ros::RosSourceDispatcher; pub use lancedb::LanceDbSinkDispatcher; pub use s3::S3SinkDispatcher; diff --git a/src/streaming_runtime_operators/src/factory/connector/mqtt.rs b/src/streaming_runtime_operators/src/factory/connector/mqtt.rs new file mode 100644 index 00000000..e5fc3107 --- /dev/null +++ b/src/streaming_runtime_operators/src/factory/connector/mqtt.rs @@ -0,0 +1,138 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use anyhow::{Context, Result, bail}; +use prost::Message; +use std::sync::Arc; +use tracing::info; + +use protocol::function_stream_graph::connector_op::Config; +use protocol::function_stream_graph::{ + BadDataPolicy, ConnectorOp, DecimalEncodingProto, FormatConfig, MqttSourceConfig, + TimestampFormatProto, +}; + +use crate::core::api::operator::ConstructedOperator; +use crate::factory::global::Registry; +use crate::factory::operator_constructor::OperatorConstructor; +use crate::format::{ + BadDataPolicy as RtBadDataPolicy, DecimalEncoding as RtDecimalEncoding, + Format as RuntimeFormat, JsonFormat as RuntimeJsonFormat, TimestampFormat as RtTimestampFormat, +}; +use crate::operators::source::batch_buffer::DEFAULT_SOURCE_BATCH_SIZE; +use crate::operators::source::kafka::BufferedDeserializer; +use crate::operators::source::mqtt::{MqttSourceOperator, proto_qos_to_rumqtt}; +use crate::sql::common::FsSchema; + +fn proto_format_to_runtime(fmt: &Option) -> Result { + let cfg = fmt.as_ref().context("FormatConfig is required")?; + match &cfg.format { + Some(protocol::function_stream_graph::format_config::Format::Json(j)) => { + Ok(RuntimeFormat::Json(RuntimeJsonFormat { + timestamp_format: match j.timestamp_format() { + TimestampFormatProto::TimestampRfc3339 => RtTimestampFormat::RFC3339, + TimestampFormatProto::TimestampUnixMillis => RtTimestampFormat::UnixMillis, + }, + decimal_encoding: match j.decimal_encoding() { + DecimalEncodingProto::DecimalNumber => RtDecimalEncoding::Number, + DecimalEncodingProto::DecimalString => RtDecimalEncoding::String, + DecimalEncodingProto::DecimalBytes => RtDecimalEncoding::Bytes, + }, + include_schema: j.include_schema, + })) + } + Some(protocol::function_stream_graph::format_config::Format::RawString(_)) => { + Ok(RuntimeFormat::RawString) + } + Some(protocol::function_stream_graph::format_config::Format::RawBytes(_)) => { + Ok(RuntimeFormat::RawBytes) + } + None => bail!("FormatConfig has no format variant set"), + } +} + +fn proto_bad_data_to_runtime(policy: i32) -> RtBadDataPolicy { + match BadDataPolicy::try_from(policy) { + Ok(BadDataPolicy::BadDataDrop) => RtBadDataPolicy::Drop, + _ => RtBadDataPolicy::Fail, + } +} + +pub struct MqttSourceDispatcher; + +impl OperatorConstructor for MqttSourceDispatcher { + fn with_config(&self, payload: &[u8], _registry: Arc) -> Result { + let op = ConnectorOp::decode(payload).context("Failed to decode ConnectorOp protobuf")?; + + let fs_schema = op + .fs_schema + .as_ref() + .map(|fs| FsSchema::try_from(fs.clone())) + .transpose() + .map_err(|e| anyhow::anyhow!("{e}"))?; + + match op.config { + Some(Config::MqttSource(ref cfg)) => { + Self::build_mqtt_source(&op.name, cfg, fs_schema) + } + Some(_) => bail!( + "ConnectorOp '{}': received non-mqtt connector config for MQTT dispatcher", + op.name + ), + None => bail!("ConnectorOp '{}' has no configuration payload", op.name), + } + } +} + +impl MqttSourceDispatcher { + fn build_mqtt_source( + _name: &str, + cfg: &MqttSourceConfig, + fs_schema: Option, + ) -> Result { + info!( + topic = %cfg.topic, + host = %cfg.host, + port = cfg.port, + "Constructing MQTT Source" + ); + + let fs = fs_schema.context("fs_schema is required for MQTT Source")?; + let runtime_format = proto_format_to_runtime(&cfg.format)?; + let bad_data = proto_bad_data_to_runtime(cfg.bad_data_policy); + + let deserializer = Box::new(BufferedDeserializer::new( + runtime_format, + fs.schema.clone(), + bad_data, + DEFAULT_SOURCE_BATCH_SIZE, + )); + + let port = cfg.port.clamp(1, u16::MAX as u32) as u16; + + let source_op = MqttSourceOperator::new( + cfg.topic.clone(), + cfg.host.clone(), + port, + cfg.client_id.clone(), + cfg.username.clone(), + cfg.password.clone(), + proto_qos_to_rumqtt(cfg.qos), + cfg.clean_session, + cfg.keep_alive_secs as u64, + vec![], + deserializer, + ); + + Ok(ConstructedOperator::Source(Box::new(source_op))) + } +} diff --git a/src/streaming_runtime_operators/src/factory/connector/robot_bag.rs b/src/streaming_runtime_operators/src/factory/connector/robot_bag.rs new file mode 100644 index 00000000..85c78894 --- /dev/null +++ b/src/streaming_runtime_operators/src/factory/connector/robot_bag.rs @@ -0,0 +1,134 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use anyhow::{Context, Result, bail}; +use prost::Message; +use std::sync::Arc; +use std::time::Duration; +use tracing::info; + +use protocol::function_stream_graph::connector_op::Config; +use protocol::function_stream_graph::{ + BadDataPolicy, ConnectorOp, DecimalEncodingProto, FormatConfig, RobotBagSourceConfig, + TimestampFormatProto, +}; + +use crate::core::api::operator::ConstructedOperator; +use crate::factory::global::Registry; +use crate::factory::operator_constructor::OperatorConstructor; +use crate::format::{ + BadDataPolicy as RtBadDataPolicy, DecimalEncoding as RtDecimalEncoding, + Format as RuntimeFormat, JsonFormat as RuntimeJsonFormat, TimestampFormat as RtTimestampFormat, +}; +use crate::operators::source::batch_buffer::DEFAULT_SOURCE_BATCH_SIZE; +use crate::operators::source::kafka::BufferedDeserializer; +use crate::operators::source::robot_bag::{ + RobotBagSourceOperator, parse_pcd_emit_mode, proto_bag_format_to_runtime, +}; +use crate::sql::common::FsSchema; + +fn proto_format_to_runtime(fmt: &Option) -> Result { + let cfg = fmt.as_ref().context("FormatConfig is required")?; + match &cfg.format { + Some(protocol::function_stream_graph::format_config::Format::Json(j)) => { + Ok(RuntimeFormat::Json(RuntimeJsonFormat { + timestamp_format: match j.timestamp_format() { + TimestampFormatProto::TimestampRfc3339 => RtTimestampFormat::RFC3339, + TimestampFormatProto::TimestampUnixMillis => RtTimestampFormat::UnixMillis, + }, + decimal_encoding: match j.decimal_encoding() { + DecimalEncodingProto::DecimalNumber => RtDecimalEncoding::Number, + DecimalEncodingProto::DecimalString => RtDecimalEncoding::String, + DecimalEncodingProto::DecimalBytes => RtDecimalEncoding::Bytes, + }, + include_schema: j.include_schema, + })) + } + Some(protocol::function_stream_graph::format_config::Format::RawString(_)) => { + Ok(RuntimeFormat::RawString) + } + Some(protocol::function_stream_graph::format_config::Format::RawBytes(_)) => { + Ok(RuntimeFormat::RawBytes) + } + None => bail!("FormatConfig has no format variant set"), + } +} + +fn proto_bad_data_to_runtime(policy: i32) -> RtBadDataPolicy { + match BadDataPolicy::try_from(policy) { + Ok(BadDataPolicy::BadDataDrop) => RtBadDataPolicy::Drop, + _ => RtBadDataPolicy::Fail, + } +} + +pub struct RobotBagSourceDispatcher; + +impl OperatorConstructor for RobotBagSourceDispatcher { + fn with_config(&self, payload: &[u8], _registry: Arc) -> Result { + let op = ConnectorOp::decode(payload).context("Failed to decode ConnectorOp protobuf")?; + + let fs_schema = op + .fs_schema + .as_ref() + .map(|fs| FsSchema::try_from(fs.clone())) + .transpose() + .map_err(|e| anyhow::anyhow!("{e}"))?; + + match op.config { + Some(Config::RobotBagSource(ref cfg)) => { + Self::build_robot_bag_source(&op.name, cfg, fs_schema) + } + Some(_) => bail!( + "ConnectorOp '{}': received non-robot-bag config for robot-bag dispatcher", + op.name + ), + None => bail!("ConnectorOp '{}' has no configuration payload", op.name), + } + } +} + +impl RobotBagSourceDispatcher { + fn build_robot_bag_source( + _name: &str, + cfg: &RobotBagSourceConfig, + fs_schema: Option, + ) -> Result { + info!(path = %cfg.path, "Constructing robot-bag Source"); + + let fs = fs_schema.context("fs_schema is required for robot-bag Source")?; + let runtime_format = proto_format_to_runtime(&cfg.format)?; + let bad_data = proto_bad_data_to_runtime(cfg.bad_data_policy); + + let deserializer = Box::new(BufferedDeserializer::new( + runtime_format, + fs.schema.clone(), + bad_data, + DEFAULT_SOURCE_BATCH_SIZE, + )); + + let source_op = RobotBagSourceOperator::new( + cfg.path.clone(), + proto_bag_format_to_runtime(cfg.bag_format), + cfg.topic.clone(), + parse_pcd_emit_mode(if cfg.pcd_emit_mode.is_empty() { + None + } else { + Some(cfg.pcd_emit_mode.as_str()) + })?, + Duration::from_millis(cfg.replay_interval_ms), + cfg.loop_replay, + deserializer, + ); + + Ok(ConstructedOperator::Source(Box::new(source_op))) + } +} diff --git a/src/streaming_runtime_operators/src/factory/connector/ros.rs b/src/streaming_runtime_operators/src/factory/connector/ros.rs new file mode 100644 index 00000000..ab4725bb --- /dev/null +++ b/src/streaming_runtime_operators/src/factory/connector/ros.rs @@ -0,0 +1,126 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use anyhow::{Context, Result, bail}; +use prost::Message; +use std::sync::Arc; +use tracing::info; + +use protocol::function_stream_graph::connector_op::Config; +use protocol::function_stream_graph::{ + BadDataPolicy, ConnectorOp, DecimalEncodingProto, FormatConfig, RosSourceConfig, + TimestampFormatProto, +}; + +use crate::core::api::operator::ConstructedOperator; +use crate::factory::global::Registry; +use crate::factory::operator_constructor::OperatorConstructor; +use crate::format::{ + BadDataPolicy as RtBadDataPolicy, DecimalEncoding as RtDecimalEncoding, + Format as RuntimeFormat, JsonFormat as RuntimeJsonFormat, TimestampFormat as RtTimestampFormat, +}; +use crate::operators::source::batch_buffer::DEFAULT_SOURCE_BATCH_SIZE; +use crate::operators::source::kafka::BufferedDeserializer; +use crate::operators::source::ros::RosSourceOperator; +use crate::sql::common::FsSchema; + +fn proto_format_to_runtime(fmt: &Option) -> Result { + let cfg = fmt.as_ref().context("FormatConfig is required")?; + match &cfg.format { + Some(protocol::function_stream_graph::format_config::Format::Json(j)) => { + Ok(RuntimeFormat::Json(RuntimeJsonFormat { + timestamp_format: match j.timestamp_format() { + TimestampFormatProto::TimestampRfc3339 => RtTimestampFormat::RFC3339, + TimestampFormatProto::TimestampUnixMillis => RtTimestampFormat::UnixMillis, + }, + decimal_encoding: match j.decimal_encoding() { + DecimalEncodingProto::DecimalNumber => RtDecimalEncoding::Number, + DecimalEncodingProto::DecimalString => RtDecimalEncoding::String, + DecimalEncodingProto::DecimalBytes => RtDecimalEncoding::Bytes, + }, + include_schema: j.include_schema, + })) + } + Some(protocol::function_stream_graph::format_config::Format::RawString(_)) => { + Ok(RuntimeFormat::RawString) + } + Some(protocol::function_stream_graph::format_config::Format::RawBytes(_)) => { + Ok(RuntimeFormat::RawBytes) + } + None => bail!("FormatConfig has no format variant set"), + } +} + +fn proto_bad_data_to_runtime(policy: i32) -> RtBadDataPolicy { + match BadDataPolicy::try_from(policy) { + Ok(BadDataPolicy::BadDataDrop) => RtBadDataPolicy::Drop, + _ => RtBadDataPolicy::Fail, + } +} + +pub struct RosSourceDispatcher; + +impl OperatorConstructor for RosSourceDispatcher { + fn with_config(&self, payload: &[u8], _registry: Arc) -> Result { + let op = ConnectorOp::decode(payload).context("Failed to decode ConnectorOp protobuf")?; + + let fs_schema = op + .fs_schema + .as_ref() + .map(|fs| FsSchema::try_from(fs.clone())) + .transpose() + .map_err(|e| anyhow::anyhow!("{e}"))?; + + match op.config { + Some(Config::RosSource(ref cfg)) => Self::build_ros_source(&op.name, cfg, fs_schema), + Some(_) => bail!( + "ConnectorOp '{}': received non-ros connector config for ROS dispatcher", + op.name + ), + None => bail!("ConnectorOp '{}' has no configuration payload", op.name), + } + } +} + +impl RosSourceDispatcher { + fn build_ros_source( + _name: &str, + cfg: &RosSourceConfig, + fs_schema: Option, + ) -> Result { + info!( + url = %cfg.url, + topic = %cfg.topic, + "Constructing ROS Source (rosbridge)" + ); + + let fs = fs_schema.context("fs_schema is required for ROS Source")?; + let runtime_format = proto_format_to_runtime(&cfg.format)?; + let bad_data = proto_bad_data_to_runtime(cfg.bad_data_policy); + + let deserializer = Box::new(BufferedDeserializer::new( + runtime_format, + fs.schema.clone(), + bad_data, + DEFAULT_SOURCE_BATCH_SIZE, + )); + + let source_op = RosSourceOperator::new( + cfg.url.clone(), + cfg.topic.clone(), + cfg.message_field.clone(), + deserializer, + ); + + Ok(ConstructedOperator::Source(Box::new(source_op))) + } +} diff --git a/src/runtime/streaming/factory/connector/s3.rs b/src/streaming_runtime_operators/src/factory/connector/s3.rs similarity index 87% rename from src/runtime/streaming/factory/connector/s3.rs rename to src/streaming_runtime_operators/src/factory/connector/s3.rs index 4b67fb9e..0947394a 100644 --- a/src/runtime/streaming/factory/connector/s3.rs +++ b/src/streaming_runtime_operators/src/factory/connector/s3.rs @@ -18,15 +18,11 @@ use prost::Message; use protocol::function_stream_graph::ConnectorOp; use protocol::function_stream_graph::connector_op::Config; -use crate::runtime::streaming::api::operator::ConstructedOperator; -use crate::runtime::streaming::factory::connector::sink_props_codec::{ - apply_common_sink_fields, normalized_props, -}; -use crate::runtime::streaming::factory::global::Registry; -use crate::runtime::streaming::factory::operator_constructor::OperatorConstructor; -use crate::runtime::streaming::operators::sink::s3::{ - S3Format, S3SinkOperator, compression_from_str, -}; +use crate::core::api::operator::ConstructedOperator; +use crate::factory::connector::sink_props_codec::{apply_common_sink_fields, normalized_props}; +use crate::factory::global::Registry; +use crate::factory::operator_constructor::OperatorConstructor; +use crate::operators::sink::s3::{S3Format, S3SinkOperator, compression_from_str}; use crate::sql::common::constants::connection_format_value; use crate::sql::common::with_option_keys as opt; diff --git a/src/runtime/streaming/factory/connector/sink_props_codec.rs b/src/streaming_runtime_operators/src/factory/connector/sink_props_codec.rs similarity index 100% rename from src/runtime/streaming/factory/connector/sink_props_codec.rs rename to src/streaming_runtime_operators/src/factory/connector/sink_props_codec.rs diff --git a/src/runtime/streaming/factory/global/mod.rs b/src/streaming_runtime_operators/src/factory/global/mod.rs similarity index 100% rename from src/runtime/streaming/factory/global/mod.rs rename to src/streaming_runtime_operators/src/factory/global/mod.rs diff --git a/src/runtime/streaming/factory/global/session_registry.rs b/src/streaming_runtime_operators/src/factory/global/session_registry.rs similarity index 100% rename from src/runtime/streaming/factory/global/session_registry.rs rename to src/streaming_runtime_operators/src/factory/global/session_registry.rs diff --git a/src/runtime/streaming/factory/mod.rs b/src/streaming_runtime_operators/src/factory/mod.rs similarity index 100% rename from src/runtime/streaming/factory/mod.rs rename to src/streaming_runtime_operators/src/factory/mod.rs diff --git a/src/runtime/streaming/factory/operator_constructor.rs b/src/streaming_runtime_operators/src/factory/operator_constructor.rs similarity index 86% rename from src/runtime/streaming/factory/operator_constructor.rs rename to src/streaming_runtime_operators/src/factory/operator_constructor.rs index 5d0ff7d7..54788b7d 100644 --- a/src/runtime/streaming/factory/operator_constructor.rs +++ b/src/streaming_runtime_operators/src/factory/operator_constructor.rs @@ -13,8 +13,8 @@ use anyhow::Result; use std::sync::Arc; -use crate::runtime::streaming::api::operator::ConstructedOperator; -use crate::runtime::streaming::factory::global::Registry; +use crate::core::api::operator::ConstructedOperator; +use crate::factory::global::Registry; /// Builds a [`ConstructedOperator`] from serialized configuration and a [`Registry`]. pub trait OperatorConstructor: Send + Sync { diff --git a/src/runtime/streaming/factory/operator_factory.rs b/src/streaming_runtime_operators/src/factory/operator_factory.rs similarity index 94% rename from src/runtime/streaming/factory/operator_factory.rs rename to src/streaming_runtime_operators/src/factory/operator_factory.rs index 1ce04eeb..e695dd51 100644 --- a/src/runtime/streaming/factory/operator_factory.rs +++ b/src/streaming_runtime_operators/src/factory/operator_factory.rs @@ -11,27 +11,23 @@ // limitations under the License. use super::operator_constructor::OperatorConstructor; -use crate::runtime::streaming::api::operator::ConstructedOperator; -use crate::runtime::streaming::factory::connector::{ - ConnectorSinkDispatcher, ConnectorSourceDispatcher, -}; -use crate::runtime::streaming::factory::global::Registry; -use crate::runtime::streaming::operators::grouping::IncrementalAggregatingConstructor; -use crate::runtime::streaming::operators::joins::{ - InstantJoinConstructor, JoinWithExpirationConstructor, -}; +use crate::core::api::operator::ConstructedOperator; +use crate::factory::connector::{ConnectorSinkDispatcher, ConnectorSourceDispatcher}; +use crate::factory::global::Registry; +use crate::operators::grouping::IncrementalAggregatingConstructor; +use crate::operators::joins::{InstantJoinConstructor, JoinWithExpirationConstructor}; use anyhow::{Result, anyhow}; use prost::Message; use protocol::function_stream_graph::ProjectionOperator as ProjectionOperatorProto; use std::collections::HashMap; use std::sync::Arc; -use crate::runtime::streaming::operators::watermark::WatermarkGeneratorConstructor; -use crate::runtime::streaming::operators::windows::{ +use crate::operators::watermark::WatermarkGeneratorConstructor; +use crate::operators::windows::{ SessionAggregatingWindowConstructor, SlidingAggregatingWindowConstructor, TumblingAggregateWindowConstructor, WindowFunctionConstructor, }; -use crate::runtime::streaming::operators::{ +use crate::operators::{ KeyExecutionOperator, ProjectionOperator, StatelessPhysicalExecutor, ValueExecutionOperator, }; use protocol::function_stream_graph::{ @@ -125,7 +121,7 @@ impl OperatorFactory { ); self.register_named(OperatorName::ConnectorSink, Box::new(ConnectorSinkBridge)); - crate::runtime::streaming::factory::register_kafka_connector_plugins(self); + crate::factory::register_kafka_connector_plugins(self); } } diff --git a/src/runtime/streaming/format/config.rs b/src/streaming_runtime_operators/src/format/config.rs similarity index 100% rename from src/runtime/streaming/format/config.rs rename to src/streaming_runtime_operators/src/format/config.rs diff --git a/src/runtime/streaming/format/deserializer.rs b/src/streaming_runtime_operators/src/format/deserializer.rs similarity index 63% rename from src/runtime/streaming/format/deserializer.rs rename to src/streaming_runtime_operators/src/format/deserializer.rs index 4058908f..5056b4df 100644 --- a/src/runtime/streaming/format/deserializer.rs +++ b/src/streaming_runtime_operators/src/format/deserializer.rs @@ -11,6 +11,7 @@ // limitations under the License. use anyhow::{Context, Result, anyhow}; +use arrow::compute::concat_batches; use arrow_array::builder::{BinaryBuilder, StringBuilder, TimestampNanosecondBuilder}; use arrow_array::{ArrayRef, RecordBatch}; use arrow_json::reader::ReaderBuilder; @@ -108,18 +109,8 @@ impl DataDeserializer { buffer.push(b'\n'); } - let mut decoder = ReaderBuilder::new(self.decoder_schema.clone()) - .with_strict_mode(false) - .build_decoder() - .context("Failed to build Arrow JSON decoder")?; - - decoder - .decode(&buffer) - .context("Arrow batch decoding failed")?; - - let decoded_batch = decoder - .flush()? - .ok_or_else(|| anyhow!("Decoder returned empty batch after successful validation"))?; + let decoded_batch = + decode_ndjson_buffer(&self.decoder_schema, &buffer, valid_indices.len())?; // Step 3: Re-inject Event-Time Column self.rebuild_with_timestamp(decoded_batch, kafka_timestamps_ms, &valid_indices) @@ -217,6 +208,53 @@ impl DataDeserializer { } } +/// Decodes NDJSON bytes into a single [`RecordBatch`], looping decode/flush until the +/// buffer is fully consumed. Arrow's JSON decoder defaults to `batch_size = 1024`, so a +/// single decode+flush is insufficient when the input has more rows than that. +fn decode_ndjson_buffer( + decoder_schema: &SchemaRef, + buffer: &[u8], + expected_rows: usize, +) -> Result { + let mut decoder = ReaderBuilder::new(decoder_schema.clone()) + .with_strict_mode(false) + .with_batch_size(expected_rows.max(1)) + .build_decoder() + .context("Failed to build Arrow JSON decoder")?; + + let mut offset = 0; + let mut batches = Vec::new(); + while offset < buffer.len() { + let decoded = decoder + .decode(&buffer[offset..]) + .context("Arrow batch decoding failed")?; + if decoded == 0 { + return Err(anyhow!( + "Arrow JSON decoder stalled at offset {}/{}", + offset, + buffer.len() + )); + } + offset += decoded; + + while let Some(batch) = decoder.flush()? { + batches.push(batch); + } + } + + if batches.is_empty() { + return Err(anyhow!( + "Decoder returned empty batch after successful validation" + )); + } + + if batches.len() == 1 { + Ok(batches.into_iter().next().unwrap()) + } else { + concat_batches(decoder_schema, &batches).context("Failed to concat decoded JSON batches") + } +} + /// Helper: Strips the specialized timestamp field to allow the raw decoder /// to focus only on payload data. fn schema_without_timestamp(schema: &Schema) -> SchemaRef { @@ -228,3 +266,88 @@ fn schema_without_timestamp(schema: &Schema) -> SchemaRef { .collect::>(); Arc::new(Schema::new_with_metadata(fields, schema.metadata().clone())) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::config::{BadDataPolicy, DecimalEncoding, Format, JsonFormat, TimestampFormat}; + use arrow_schema::{DataType, Field, TimeUnit}; + + fn test_deserializer() -> DataDeserializer { + let schema = Arc::new(Schema::new(vec![ + Field::new( + TIMESTAMP_FIELD, + DataType::Timestamp(TimeUnit::Nanosecond, None), + false, + ), + Field::new("id", DataType::Int64, false), + Field::new("value", DataType::Utf8, false), + ])); + DataDeserializer::new( + Format::Json(JsonFormat { + timestamp_format: TimestampFormat::UnixMillis, + decimal_encoding: DecimalEncoding::String, + include_schema: false, + }), + schema, + BadDataPolicy::Fail, + ) + } + + #[test] + fn decode_json_batch_exceeds_arrow_default_batch_size() { + let deserializer = test_deserializer(); + let row_count = 2500; + let messages: Vec> = (0..row_count) + .map(|i| format!(r#"{{"id":{i},"value":"row-{i}"}}"#).into_bytes()) + .collect(); + let msg_refs: Vec<&[u8]> = messages.iter().map(|m| m.as_slice()).collect(); + let kafka_ts: Vec = (0..row_count).map(|i| 1_700_000_000_000 + i as u64).collect(); + + let batch = deserializer + .deserialize_batch_with_kafka_timestamps(&msg_refs, &kafka_ts) + .expect("batch larger than 1024 rows should decode"); + + assert_eq!(batch.num_rows(), row_count); + } + + #[test] + fn decode_json_batch_at_kafka_source_batch_size() { + let deserializer = test_deserializer(); + let row_count = 4096; + let messages: Vec> = (0..row_count) + .map(|i| format!(r#"{{"id":{i},"value":"row-{i}"}}"#).into_bytes()) + .collect(); + let msg_refs: Vec<&[u8]> = messages.iter().map(|m| m.as_slice()).collect(); + let kafka_ts: Vec = (0..row_count).map(|i| i as u64).collect(); + + let batch = deserializer + .deserialize_batch_with_kafka_timestamps(&msg_refs, &kafka_ts) + .expect("4096-row batch should decode without alignment error"); + + assert_eq!(batch.num_rows(), row_count); + } + + #[test] + fn decode_ndjson_buffer_loops_past_default_batch_size() { + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); + let row_count = 3647; + let mut buffer = Vec::new(); + for i in 0..row_count { + buffer.extend_from_slice(format!(r#"{{"id":{i}}}"#).as_bytes()); + buffer.push(b'\n'); + } + + // Pre-fix behavior: one decode + one flush only yields 1024 rows. + let mut decoder = ReaderBuilder::new(schema.clone()) + .with_strict_mode(false) + .build_decoder() + .unwrap(); + decoder.decode(&buffer).unwrap(); + let single_flush = decoder.flush().unwrap().unwrap(); + assert_eq!(single_flush.num_rows(), 1024); + + let batch = decode_ndjson_buffer(&schema, &buffer, row_count).expect("3647 rows"); + assert_eq!(batch.num_rows(), row_count); + } +} diff --git a/src/runtime/streaming/format/encoder.rs b/src/streaming_runtime_operators/src/format/encoder.rs similarity index 97% rename from src/runtime/streaming/format/encoder.rs rename to src/streaming_runtime_operators/src/format/encoder.rs index 198b06b2..5e6eb780 100644 --- a/src/runtime/streaming/format/encoder.rs +++ b/src/streaming_runtime_operators/src/format/encoder.rs @@ -57,10 +57,22 @@ impl FormatEncoder { if batches.is_empty() { return Ok(Vec::new()); } + Self::encode_parquet_with_schema(batches, batches[0].schema(), compression) + } + + /// Encode with an explicit schema and larger row groups for throughput. + pub fn encode_parquet_with_schema( + batches: &[RecordBatch], + schema: std::sync::Arc, + compression: Compression, + ) -> Result> { + if batches.is_empty() { + return Ok(Vec::new()); + } - let schema = batches[0].schema(); let props = WriterProperties::builder() .set_compression(compression) + .set_max_row_group_size(128 * 1024 * 1024) .build(); let mut cursor = Cursor::new(Vec::new()); let mut writer = ArrowWriter::try_new(&mut cursor, schema, Some(props)) diff --git a/src/runtime/streaming/format/json_encoder.rs b/src/streaming_runtime_operators/src/format/json_encoder.rs similarity index 100% rename from src/runtime/streaming/format/json_encoder.rs rename to src/streaming_runtime_operators/src/format/json_encoder.rs diff --git a/src/runtime/streaming/format/mod.rs b/src/streaming_runtime_operators/src/format/mod.rs similarity index 100% rename from src/runtime/streaming/format/mod.rs rename to src/streaming_runtime_operators/src/format/mod.rs diff --git a/src/runtime/streaming/format/serializer.rs b/src/streaming_runtime_operators/src/format/serializer.rs similarity index 100% rename from src/runtime/streaming/format/serializer.rs rename to src/streaming_runtime_operators/src/format/serializer.rs diff --git a/src/streaming_runtime_operators/src/lib.rs b/src/streaming_runtime_operators/src/lib.rs new file mode 100644 index 00000000..d2c11639 --- /dev/null +++ b/src/streaming_runtime_operators/src/lib.rs @@ -0,0 +1,26 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Streaming operators, formats, and operator factory. + +pub use function_stream_config as config; +pub use function_stream_runtime_common::{common, memory}; +pub use function_stream_streaming_planner as sql; +pub use function_stream_streaming_runtime_core as core; +pub use function_stream_streaming_runtime_core::{ + StreamOutput, api, error, execution, network, protocol, state, +}; + +pub mod factory; +pub mod format; +pub mod operators; +pub mod util; diff --git a/src/runtime/streaming/operators/grouping/incremental_aggregate.rs b/src/streaming_runtime_operators/src/operators/grouping/incremental_aggregate.rs similarity index 99% rename from src/runtime/streaming/operators/grouping/incremental_aggregate.rs rename to src/streaming_runtime_operators/src/operators/grouping/incremental_aggregate.rs index ff997c11..c1b34b71 100644 --- a/src/runtime/streaming/operators/grouping/incremental_aggregate.rs +++ b/src/streaming_runtime_operators/src/operators/grouping/incremental_aggregate.rs @@ -41,17 +41,17 @@ use std::{collections::HashMap, mem, sync::Arc}; use tracing::{debug, info, warn}; // ========================================================================= // ========================================================================= -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::factory::Registry; -use crate::runtime::streaming::operators::{Key, UpdatingCache}; -use crate::runtime::streaming::state::OperatorStateStore; -use crate::runtime::util::decode_aggregate; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::core::state::OperatorStateStore; +use crate::factory::Registry; +use crate::operators::{Key, UpdatingCache}; use crate::sql::common::{ CheckpointBarrier, FsSchema, TIMESTAMP_FIELD, UPDATING_META_FIELD, Watermark, to_nanos, }; use crate::sql::physical::updating_meta_fields; +use crate::util::decode_aggregate; #[derive(Debug, Copy, Clone)] struct BatchData { diff --git a/src/runtime/streaming/operators/grouping/mod.rs b/src/streaming_runtime_operators/src/operators/grouping/mod.rs similarity index 100% rename from src/runtime/streaming/operators/grouping/mod.rs rename to src/streaming_runtime_operators/src/operators/grouping/mod.rs diff --git a/src/runtime/streaming/operators/grouping/updating_cache.rs b/src/streaming_runtime_operators/src/operators/grouping/updating_cache.rs similarity index 100% rename from src/runtime/streaming/operators/grouping/updating_cache.rs rename to src/streaming_runtime_operators/src/operators/grouping/updating_cache.rs diff --git a/src/runtime/streaming/operators/joins/join_instance.rs b/src/streaming_runtime_operators/src/operators/joins/join_instance.rs similarity index 98% rename from src/runtime/streaming/operators/joins/join_instance.rs rename to src/streaming_runtime_operators/src/operators/joins/join_instance.rs index 5c04c1fb..3c9c55fc 100644 --- a/src/runtime/streaming/operators/joins/join_instance.rs +++ b/src/streaming_runtime_operators/src/operators/joins/join_instance.rs @@ -25,11 +25,11 @@ use std::sync::{Arc, RwLock}; use std::time::UNIX_EPOCH; use tracing::{info, warn}; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::factory::Registry; -use crate::runtime::streaming::state::OperatorStateStore; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::core::state::OperatorStateStore; +use crate::factory::Registry; use crate::sql::common::{CheckpointBarrier, FsSchema, FsSchemaRef, Watermark}; use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; use async_trait::async_trait; diff --git a/src/runtime/streaming/operators/joins/join_with_expiration.rs b/src/streaming_runtime_operators/src/operators/joins/join_with_expiration.rs similarity index 97% rename from src/runtime/streaming/operators/joins/join_with_expiration.rs rename to src/streaming_runtime_operators/src/operators/joins/join_with_expiration.rs index 5ed8dfa3..f0e5eec5 100644 --- a/src/runtime/streaming/operators/joins/join_with_expiration.rs +++ b/src/streaming_runtime_operators/src/operators/joins/join_with_expiration.rs @@ -24,11 +24,11 @@ use std::sync::{Arc, RwLock}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tracing::{info, warn}; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::factory::Registry; -use crate::runtime::streaming::state::OperatorStateStore; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::core::state::OperatorStateStore; +use crate::factory::Registry; use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark}; use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; use async_trait::async_trait; diff --git a/src/runtime/streaming/operators/joins/mod.rs b/src/streaming_runtime_operators/src/operators/joins/mod.rs similarity index 100% rename from src/runtime/streaming/operators/joins/mod.rs rename to src/streaming_runtime_operators/src/operators/joins/mod.rs diff --git a/src/runtime/streaming/operators/key_by.rs b/src/streaming_runtime_operators/src/operators/key_by.rs similarity index 96% rename from src/runtime/streaming/operators/key_by.rs rename to src/streaming_runtime_operators/src/operators/key_by.rs index 90c55d08..63ac06cd 100644 --- a/src/runtime/streaming/operators/key_by.rs +++ b/src/streaming_runtime_operators/src/operators/key_by.rs @@ -19,9 +19,9 @@ use datafusion_common::hash_utils::create_hashes; use datafusion_physical_expr::expressions::Column; use std::sync::Arc; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; use crate::sql::common::{CheckpointBarrier, Watermark}; use protocol::function_stream_graph::KeyPlanOperator; diff --git a/src/runtime/streaming/operators/key_operator.rs b/src/streaming_runtime_operators/src/operators/key_operator.rs similarity index 94% rename from src/runtime/streaming/operators/key_operator.rs rename to src/streaming_runtime_operators/src/operators/key_operator.rs index 7a89d2f2..a485bf9e 100644 --- a/src/runtime/streaming/operators/key_operator.rs +++ b/src/streaming_runtime_operators/src/operators/key_operator.rs @@ -13,12 +13,12 @@ //! Key-by over the physical plan output: key column(s) are **values** projected by the plan //! (e.g. `_key_user_id`); **shuffle / `StreamOutput::Keyed` uses `u64` hashes** computed by //! [`datafusion_common::hash_utils::create_hashes`] on those columns — same mechanism as -//! [`crate::runtime::streaming::operators::key_by::KeyByOperator`]. +//! [`crate::operators::key_by::KeyByOperator`]. -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::operators::StatelessPhysicalExecutor; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::operators::StatelessPhysicalExecutor; use crate::sql::common::{CheckpointBarrier, Watermark}; use ahash::RandomState; use anyhow::{Result, anyhow}; diff --git a/src/runtime/streaming/operators/mod.rs b/src/streaming_runtime_operators/src/operators/mod.rs similarity index 100% rename from src/runtime/streaming/operators/mod.rs rename to src/streaming_runtime_operators/src/operators/mod.rs diff --git a/src/runtime/streaming/operators/projection.rs b/src/streaming_runtime_operators/src/operators/projection.rs similarity index 94% rename from src/runtime/streaming/operators/projection.rs rename to src/streaming_runtime_operators/src/operators/projection.rs index b84d74aa..b2aa0af3 100644 --- a/src/runtime/streaming/operators/projection.rs +++ b/src/streaming_runtime_operators/src/operators/projection.rs @@ -22,10 +22,10 @@ use std::sync::Arc; use protocol::function_stream_graph::ProjectionOperator as ProjectionOperatorProto; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::factory::global::Registry; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::factory::global::Registry; use crate::sql::common::{CheckpointBarrier, FsSchema, FsSchemaRef, Watermark}; use crate::sql::logical_node::logical::OperatorName; diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs b/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs new file mode 100644 index 00000000..e73d5058 --- /dev/null +++ b/src/streaming_runtime_operators/src/operators/sink/delta/delta_commit.rs @@ -0,0 +1,589 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +use arrow::compute::cast; +use arrow_array::RecordBatch; +use arrow_ipc::writer::StreamWriter; +use arrow_schema::{DataType, Field, FieldRef, Schema as ArrowSchema, TimeUnit}; +use deltalake::errors::DeltaTableError; +use deltalake::kernel::engine::arrow_conversion::TryIntoKernel as _; +use deltalake::kernel::schema::cast::normalize_for_delta; +use deltalake::kernel::transaction::CommitBuilder; +use deltalake::kernel::{Action, Add, StructField, StructType}; +use deltalake::protocol::{DeltaOperation, SaveMode}; +use deltalake::{DeltaTable, open_table_with_storage_options}; +use tracing::{info, instrument}; +use url::Url; + +use crate::sql::common::{TIMESTAMP_FIELD, UPDATING_META_FIELD}; + +use super::DeltaSinkError; + +/// Coordination strategy for Delta `_delta_log` commits on S3-compatible stores. +/// +/// `deltalake-aws` 0.15 only ships two safe primitives for S3 commits: +/// - a DynamoDB-backed lock client, or +/// - a single-writer flag (`AWS_S3_ALLOW_UNSAFE_RENAME=true`) that +/// guarantees correctness only when there is exactly one writer per table. +/// +/// Function-Stream's planner+scheduler guarantees a single writer per Delta +/// table, so `SingleWriter` is the safe default for all S3-compatible backends +/// including MinIO, R2 and GCS-S3. For multi-process deployments writing to the +/// same table, switch to `DynamoDb`. +#[derive(Debug, Clone)] +pub enum DeltaCommitStrategy { + /// Single-writer per table (default). Function-Stream's scheduler enforces + /// this invariant, so direct PUTs to `_delta_log/*.json` are race-free. + SingleWriter, + + /// DynamoDB-backed multi-writer concurrency control via deltalake-aws. + DynamoDb(DynamoDbLockConfig), +} + +#[derive(Debug, Clone, Default)] +pub struct DynamoDbLockConfig { + pub table_name: Option, + pub region: Option, + pub endpoint: Option, + pub access_key_id: Option, + pub secret_access_key: Option, +} + +impl DeltaCommitStrategy { + /// Resolve the commit strategy from sink options. + /// + /// `delta.commit.strategy`: + /// - `single-writer` / `single_writer` / `none` / unset → SingleWriter + /// - `dynamodb` / `dynamo` → DynamoDb + pub fn from_options(options: &HashMap) -> Result { + use crate::sql::common::with_option_keys as opt; + + let raw = options + .get(opt::DELTA_COMMIT_STRATEGY) + .map(|v| v.trim().to_ascii_lowercase()) + .unwrap_or_default(); + + match raw.as_str() { + "" | "single-writer" | "single_writer" | "singlewriter" | "none" => { + Ok(Self::SingleWriter) + } + "dynamodb" | "dynamo" => { + let cfg = DynamoDbLockConfig { + table_name: options.get(opt::DELTA_DYNAMODB_TABLE).cloned(), + region: options.get(opt::DELTA_DYNAMODB_REGION).cloned(), + endpoint: options.get(opt::DELTA_DYNAMODB_ENDPOINT).cloned(), + access_key_id: options.get(opt::DELTA_DYNAMODB_ACCESS_KEY_ID).cloned(), + secret_access_key: options.get(opt::DELTA_DYNAMODB_SECRET_ACCESS_KEY).cloned(), + }; + if cfg.table_name.is_none() { + return Err(DeltaSinkError::Config( + "delta.commit.strategy=dynamodb requires delta.dynamodb.table".into(), + )); + } + Ok(Self::DynamoDb(cfg)) + } + other => Err(DeltaSinkError::Config(format!( + "unknown delta.commit.strategy '{other}'; expected single-writer or dynamodb" + ))), + } + } + + pub fn label(&self) -> &'static str { + match self { + Self::SingleWriter => "single-writer", + Self::DynamoDb(_) => "dynamodb", + } + } +} + +/// Streaming-internal columns that must not be persisted to external sinks. +pub fn is_streaming_system_column(name: &str) -> bool { + name == TIMESTAMP_FIELD || name == UPDATING_META_FIELD +} + +/// Remove `_timestamp` / `_updating_meta` from a schema (e.g. connector `fs_schema` may inject them). +pub fn strip_streaming_system_columns(schema: &ArrowSchema) -> ArrowSchema { + let fields: Vec = schema + .fields() + .iter() + .filter(|f| !is_streaming_system_column(f.name())) + .cloned() + .collect(); + ArrowSchema::new(fields) +} + +pub fn strip_streaming_system_columns_arc(schema: Arc) -> Option> { + let stripped = strip_streaming_system_columns(schema.as_ref()); + if stripped.fields().is_empty() { + return None; + } + let had_system = schema + .fields() + .iter() + .any(|f| is_streaming_system_column(f.name())); + if had_system { + Some(Arc::new(stripped)) + } else { + Some(schema) + } +} + +pub struct UncommittedDataFile { + pub path: String, + pub size_bytes: u64, + pub record_count: u64, +} + +pub struct DeltaTableCommitter { + table_uri: Url, + storage_options: HashMap, + uncommitted: Vec, + table: Option, + /// Precomputed from catalog `fs_schema` at startup, or lazily from the first batch. + delta_columns: Option>, + /// Arrow 55 schema for Parquet writes (timestamps normalized to microsecond). + write_schema: Option>, +} + +impl DeltaTableCommitter { + pub fn try_new( + table_uri: Url, + storage_options: HashMap, + catalog_schema: Option>, + ) -> Result { + let user_schema = catalog_schema.and_then(strip_streaming_system_columns_arc); + let write_schema = user_schema + .as_ref() + .map(|s| Arc::new(normalize_arrow_schema_for_delta(s))); + let delta_columns = user_schema + .as_deref() + .map(arrow_schema_to_delta_columns) + .transpose()?; + + Ok(Self { + table_uri, + storage_options, + uncommitted: Vec::new(), + table: None, + delta_columns, + write_schema, + }) + } + + pub fn write_schema(&self) -> Option> { + self.write_schema.clone() + } + + /// Fallback when catalog schema is absent: derive user columns from the first flushed batch. + pub fn update_schema(&mut self, schema: Arc) -> Result<(), DeltaSinkError> { + let user_schema = strip_streaming_system_columns_arc(schema).ok_or_else(|| { + DeltaSinkError::CommitterFailed( + "cannot derive delta table schema: no user columns after removing streaming \ + system columns (_timestamp, _updating_meta)" + .into(), + ) + })?; + if self.delta_columns.is_none() { + self.delta_columns = Some(arrow_schema_to_delta_columns(user_schema.as_ref())?); + } + if self.write_schema.is_none() { + self.write_schema = Some(Arc::new(normalize_arrow_schema_for_delta( + user_schema.as_ref(), + ))); + } + Ok(()) + } + + pub fn register_uncommitted(&mut self, file: UncommittedDataFile) { + self.uncommitted.push(file); + } + + pub fn has_uncommitted(&self) -> bool { + !self.uncommitted.is_empty() + } + + async fn ensure_table(&mut self) -> Result<(), DeltaSinkError> { + if self.table.is_some() { + return Ok(()); + } + + let open_result = if self.storage_options.is_empty() { + deltalake::open_table(self.table_uri.clone()).await + } else { + open_table_with_storage_options(self.table_uri.clone(), self.storage_options.clone()) + .await + }; + + match open_result { + Ok(table) => { + let version = table.version(); + info!( + table_uri = %self.table_uri, + ?version, + "opened existing delta table" + ); + self.table = Some(table); + return Ok(()); + } + Err(DeltaTableError::NotATable(_)) => {} + Err(e) => { + return Err(DeltaSinkError::CommitterFailed(e.to_string())); + } + } + + let columns = self.delta_columns.as_ref().ok_or_else(|| { + DeltaSinkError::Config( + "no catalog schema and no batch schema available before commit".into(), + ) + })?; + + let table = if self.storage_options.is_empty() { + DeltaTable::try_from_url(self.table_uri.clone()) + .await + .map_err(|e| DeltaSinkError::CommitterFailed(e.to_string()))? + } else { + DeltaTable::try_from_url_with_storage_options( + self.table_uri.clone(), + self.storage_options.clone(), + ) + .await + .map_err(|e| DeltaSinkError::CommitterFailed(e.to_string()))? + }; + + let table = table + .create() + .with_columns(columns.clone()) + .await + .map_err(|e| DeltaSinkError::CommitterFailed(e.to_string()))?; + + info!( + table_uri = %self.table_uri, + column_count = columns.len(), + version = table.version(), + "created new delta table" + ); + self.table = Some(table); + Ok(()) + } + + #[instrument(skip(self), fields(table_uri = %self.table_uri, epoch))] + pub async fn commit_checkpoint(&mut self, epoch: u64) -> Result<(), DeltaSinkError> { + if self.uncommitted.is_empty() { + return Ok(()); + } + + let files = std::mem::take(&mut self.uncommitted); + let file_count = files.len(); + let total_bytes: u64 = files.iter().map(|f| f.size_bytes).sum(); + let total_records: u64 = files.iter().map(|f| f.record_count).sum(); + let paths: Vec<&str> = files.iter().map(|f| f.path.as_str()).collect(); + + let prev_version = self.table.as_ref().and_then(|t| t.version()); + + info!( + file_count, + total_bytes, + total_records, + ?prev_version, + paths = ?paths, + "committing delta checkpoint" + ); + + self.ensure_table().await?; + + let table = self + .table + .as_mut() + .ok_or_else(|| DeltaSinkError::CommitterFailed("delta table not loaded".into()))?; + + let now_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as i64; + + let adds: Vec = files + .iter() + .map(|f| { + Action::Add(Add { + path: f.path.clone(), + size: f.size_bytes as i64, + modification_time: now_ms, + data_change: true, + partition_values: HashMap::new(), + stats: simple_stats_json(f.record_count), + ..Default::default() + }) + }) + .collect(); + + let snapshot = table + .snapshot() + .map_err(|e| DeltaSinkError::CommitterFailed(e.to_string()))?; + + let operation = DeltaOperation::Write { + mode: SaveMode::Append, + partition_by: None, + predicate: None, + }; + + let finalized = CommitBuilder::default() + .with_actions(adds) + .build(Some(snapshot), table.log_store().clone(), operation) + .await + .map_err(|e| DeltaSinkError::CommitterFailed(e.to_string()))?; + + let version = finalized.version(); + table.state = Some(finalized.snapshot()); + table + .load() + .await + .map_err(|e| DeltaSinkError::CommitterFailed(e.to_string()))?; + + info!( + epoch, + version, + file_count, + total_bytes, + total_records, + ?prev_version, + paths = ?paths, + "delta checkpoint committed" + ); + Ok(()) + } +} + +/// Normalize Arrow 55 schema for Delta Parquet writes (align with deltalake `normalize_for_delta`). +pub fn normalize_arrow_schema_for_delta(schema: &ArrowSchema) -> ArrowSchema { + let fields: Vec = schema + .fields() + .iter() + .map(|f| Arc::new(normalize_field_for_delta(f.as_ref()))) + .collect(); + ArrowSchema::new(fields) +} + +fn normalize_field_for_delta(field: &Field) -> Field { + let data_type = normalize_datatype_for_delta(field.data_type()); + if data_type == *field.data_type() { + field.clone() + } else { + field.clone().with_data_type(data_type) + } +} + +fn normalize_datatype_for_delta(dt: &DataType) -> DataType { + match dt { + DataType::Date64 => DataType::Date32, + DataType::Timestamp(TimeUnit::Second, tz) + | DataType::Timestamp(TimeUnit::Millisecond, tz) + | DataType::Timestamp(TimeUnit::Nanosecond, tz) => { + DataType::Timestamp(TimeUnit::Microsecond, tz.clone()) + } + DataType::Struct(fields) => { + let normalized: Vec = fields + .iter() + .map(|f| Arc::new(normalize_field_for_delta(f.as_ref()))) + .collect(); + DataType::Struct(normalized.into()) + } + DataType::List(inner) => { + DataType::List(Arc::new(normalize_field_for_delta(inner.as_ref()))) + } + DataType::LargeList(inner) => { + DataType::LargeList(Arc::new(normalize_field_for_delta(inner.as_ref()))) + } + DataType::FixedSizeList(inner, len) => { + DataType::FixedSizeList(Arc::new(normalize_field_for_delta(inner.as_ref())), *len) + } + DataType::Map(entries, sorted) => DataType::Map( + Arc::new(normalize_field_for_delta(entries.as_ref())), + *sorted, + ), + _ => dt.clone(), + } +} + +/// Cast record batches so on-disk Parquet matches the Delta table schema. +pub fn cast_batches_for_delta_write( + batches: &[RecordBatch], + target_schema: &ArrowSchema, +) -> Result, DeltaSinkError> { + let target = Arc::new(target_schema.clone()); + batches + .iter() + .map(|batch| cast_batch_for_delta_write(batch, &target)) + .collect() +} + +fn cast_batch_for_delta_write( + batch: &RecordBatch, + target_schema: &Arc, +) -> Result { + if batch.schema().as_ref() == target_schema.as_ref() { + return Ok(batch.clone()); + } + + let mut columns = Vec::with_capacity(target_schema.fields().len()); + for field in target_schema.fields() { + let col = batch.column_by_name(field.name()).ok_or_else(|| { + DeltaSinkError::CommitterFailed(format!( + "batch missing column '{}' required by delta write schema", + field.name() + )) + })?; + let casted = if col.data_type() == field.data_type() { + col.clone() + } else { + cast(col, field.data_type()).map_err(|e| { + DeltaSinkError::CommitterFailed(format!( + "failed to cast column '{}' from {:?} to {:?}: {e}", + field.name(), + col.data_type(), + field.data_type() + )) + })? + }; + columns.push(casted); + } + + RecordBatch::try_new(target_schema.clone(), columns).map_err(|e| { + DeltaSinkError::CommitterFailed(format!("failed to build delta write batch: {e}")) + }) +} + +/// Bridge arrow 55 (runtime) schema to deltalake kernel schema via IPC. +fn arrow_schema_to_delta_columns(schema: &ArrowSchema) -> Result, DeltaSinkError> { + let empty = RecordBatch::new_empty(Arc::new(schema.clone())); + let mut buf = Vec::new(); + { + let mut writer = StreamWriter::try_new(&mut buf, &empty.schema()).map_err(|e| { + DeltaSinkError::CommitterFailed(format!("ipc schema encode failed: {e}")) + })?; + writer.write(&empty).map_err(|e| { + DeltaSinkError::CommitterFailed(format!("ipc schema encode failed: {e}")) + })?; + writer.finish().map_err(|e| { + DeltaSinkError::CommitterFailed(format!("ipc schema encode failed: {e}")) + })?; + } + + let cursor = std::io::Cursor::new(buf); + let reader = deltalake::arrow::ipc::reader::StreamReader::try_new(cursor, None) + .map_err(|e| DeltaSinkError::CommitterFailed(format!("ipc schema decode failed: {e}")))?; + let delta_schema = reader.schema(); + // Delta kernel rejects Timestamp(ms); normalize to Delta-compatible types (e.g. us). + let normalized = normalize_for_delta(&delta_schema); + let struct_type: StructType = normalized + .as_ref() + .try_into_kernel() + .map_err(|e| DeltaSinkError::CommitterFailed(e.to_string()))?; + Ok(struct_type.fields().cloned().collect()) +} + +fn simple_stats_json(record_count: u64) -> Option { + Some(format!(r#"{{"numRecords":{record_count}}}"#)) +} + +/// Build the storage options handed to deltalake. +/// +/// Returns both the assembled HashMap and the resolved commit strategy so the +/// caller can log/expose it. +pub fn build_delta_storage_options( + options: &HashMap, +) -> Result<(HashMap, DeltaCommitStrategy), DeltaSinkError> { + use crate::sql::common::with_option_keys as opt; + + let mut storage = HashMap::new(); + + if let Some(v) = options.get(opt::S3_ACCESS_KEY_ID) { + storage.insert("AWS_ACCESS_KEY_ID".to_string(), v.clone()); + } + if let Some(v) = options.get(opt::S3_SECRET_ACCESS_KEY) { + storage.insert("AWS_SECRET_ACCESS_KEY".to_string(), v.clone()); + } + if let Some(v) = options.get(opt::S3_REGION) { + storage.insert("AWS_REGION".to_string(), v.clone()); + } + if let Some(v) = options.get(opt::S3_ENDPOINT) { + storage.insert("AWS_ENDPOINT_URL".to_string(), v.clone()); + if !v.starts_with("https://") { + storage.insert("AWS_ALLOW_HTTP".to_string(), "true".to_string()); + } + // Custom endpoint implies MinIO / Ceph / R2 / private S3 gateway — these + // backends route via path style; virtual-hosted style requires per-bucket + // DNS that they typically do not provide. + storage.insert( + "AWS_VIRTUAL_HOSTED_STYLE_REQUEST".to_string(), + "false".to_string(), + ); + } + if let Some(v) = options.get(opt::S3_SESSION_TOKEN) { + storage.insert("AWS_SESSION_TOKEN".to_string(), v.clone()); + } + + let strategy = DeltaCommitStrategy::from_options(options)?; + match &strategy { + DeltaCommitStrategy::SingleWriter => { + // Function-Stream guarantees a single writer per Delta table via its + // pipeline scheduler. deltalake-aws still gates S3 commits unless one + // of the two supported flags is set, so we set the single-writer flag + // here; the architectural invariant makes this safe. + storage.insert("AWS_S3_ALLOW_UNSAFE_RENAME".to_string(), "true".to_string()); + } + DeltaCommitStrategy::DynamoDb(cfg) => { + storage.insert( + "AWS_S3_LOCKING_PROVIDER".to_string(), + "dynamodb".to_string(), + ); + if let Some(table) = &cfg.table_name { + storage.insert("DELTA_DYNAMO_TABLE_NAME".to_string(), table.clone()); + } + if let Some(region) = &cfg.region { + storage.insert("AWS_REGION_DYNAMODB".to_string(), region.clone()); + } + if let Some(endpoint) = &cfg.endpoint { + storage.insert("AWS_ENDPOINT_URL_DYNAMODB".to_string(), endpoint.clone()); + } + if let Some(v) = &cfg.access_key_id { + storage.insert("AWS_ACCESS_KEY_ID_DYNAMODB".to_string(), v.clone()); + } + if let Some(v) = &cfg.secret_access_key { + storage.insert("AWS_SECRET_ACCESS_KEY_DYNAMODB".to_string(), v.clone()); + } + } + } + + Ok((storage, strategy)) +} + +pub fn resolve_delta_table_uri(path: &str, bucket: Option<&str>) -> Result { + if path.contains("://") { + return Url::parse(path).map_err(|e| DeltaSinkError::Config(e.to_string())); + } + if let Some(bucket) = bucket { + let trimmed = path.trim_matches('/'); + let uri = if trimmed.is_empty() { + format!("s3://{bucket}") + } else { + format!("s3://{bucket}/{trimmed}") + }; + return Url::parse(&uri).map_err(|e| DeltaSinkError::Config(e.to_string())); + } + let canonical = std::fs::canonicalize(path).map_err(DeltaSinkError::Io)?; + Url::from_directory_path(canonical) + .map_err(|_| DeltaSinkError::Config(format!("invalid local delta table path: {path}"))) +} diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/delta_writer.rs b/src/streaming_runtime_operators/src/operators/sink/delta/delta_writer.rs new file mode 100644 index 00000000..cefe08cd --- /dev/null +++ b/src/streaming_runtime_operators/src/operators/sink/delta/delta_writer.rs @@ -0,0 +1,392 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Delta data-file writer: encode batches and PUT `part-*` objects (S3/MinIO or local). +//! Does not write `_delta_log`; [`super::delta_commit::DeltaTableCommitter`] handles that +//! at checkpoint. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use arrow_array::RecordBatch; +use arrow_schema::Schema as ArrowSchema; +use bytes::Bytes; +use object_store::aws::AmazonS3Builder; +use object_store::path::Path as ObjectStorePath; +use object_store::{BackoffConfig, ObjectStore, PutPayload, RetryConfig}; +use parquet::basic::Compression; +use tokio::io::AsyncWriteExt; +use tracing::{debug, info, instrument}; + +use crate::format::encoder::FormatEncoder; +use crate::sql::common::with_option_keys as opt; + +use super::delta_commit::cast_batches_for_delta_write; +use super::{DeltaFormat, DeltaSinkError}; + +/// Descriptor for a completed data-file write (two-phase commit: register at checkpoint). +#[derive(Debug, Clone)] +pub struct FinishedFile { + pub filename: String, + pub size: usize, + pub record_count: u64, +} + +impl FinishedFile { + pub fn into_uncommitted(self) -> super::delta_commit::UncommittedDataFile { + super::delta_commit::UncommittedDataFile { + path: self.filename, + size_bytes: self.size as u64, + record_count: self.record_count, + } + } +} + +/// S3-compatible data-file writer (AWS S3, MinIO, R2). No `_delta_log` I/O on this path. +pub struct DeltaSink { + bucket_name: String, + prefix_path: String, + object_client: Arc, + file_counter: u64, +} + +impl DeltaSink { + pub fn try_new( + bucket: String, + prefix: String, + options: &HashMap, + ) -> Result { + let client = build_s3_object_store(bucket.as_str(), options)?; + Ok(Self { + bucket_name: bucket, + prefix_path: prefix.trim_matches('/').to_string(), + object_client: client, + file_counter: 0, + }) + } + + pub fn client(&self) -> Arc { + self.object_client.clone() + } + + pub fn bucket_name(&self) -> &str { + &self.bucket_name + } + + pub fn prefix_path(&self) -> &str { + &self.prefix_path + } + + #[instrument(skip(self, batches, parquet_write_schema), fields(epoch, subtask = subtask_idx))] + pub async fn write_batches( + &mut self, + batches: &[RecordBatch], + format: DeltaFormat, + compression: Compression, + parquet_write_schema: Option>, + epoch: u64, + subtask_idx: usize, + ) -> Result, DeltaSinkError> { + if batches.is_empty() { + return Ok(None); + } + + let record_count: u64 = batches.iter().map(|b| b.num_rows() as u64).sum(); + if record_count == 0 { + return Ok(None); + } + + let encoded = + encode_in_background(batches.to_vec(), format, compression, parquet_write_schema) + .await?; + + if encoded.is_empty() { + return Ok(None); + } + + self.file_counter += 1; + let ext = format.file_extension(); + let file_name = format!( + "part-{subtask_idx:05}-epoch-{epoch:010}-{counter:06}.{ext}", + counter = self.file_counter + ); + let object_key = if self.prefix_path.is_empty() { + file_name + } else { + format!("{}/{}", self.prefix_path, file_name) + }; + + let size = encoded.len(); + debug!( + target_key = %object_key, + size_bytes = size, + record_count, + "uploading delta data file to object storage" + ); + + let start_io = Instant::now(); + self.object_client + .put( + &ObjectStorePath::from(object_key.clone()), + PutPayload::from(Bytes::from(encoded)), + ) + .await?; + let io_duration = start_io.elapsed(); + + info!( + file = %object_key, + size_bytes = size, + records = record_count, + io_latency_ms = io_duration.as_millis(), + "delta data file persisted to object storage" + ); + + Ok(Some(FinishedFile { + filename: object_key, + size, + record_count, + })) + } +} + +/// Local filesystem data-file writer with tmp + fsync + atomic rename. +pub struct DeltaLocalSink { + root: PathBuf, + file_counter: u64, +} + +impl DeltaLocalSink { + pub fn new(root: PathBuf) -> Self { + Self { + root, + file_counter: 0, + } + } + + #[instrument(skip(self, batches, parquet_write_schema), fields(epoch, subtask = subtask_idx))] + pub async fn write_batches( + &mut self, + batches: &[RecordBatch], + format: DeltaFormat, + compression: Compression, + parquet_write_schema: Option>, + epoch: u64, + subtask_idx: usize, + ) -> Result, DeltaSinkError> { + if batches.is_empty() { + return Ok(None); + } + + let record_count: u64 = batches.iter().map(|b| b.num_rows() as u64).sum(); + if record_count == 0 { + return Ok(None); + } + + let encoded = + encode_in_background(batches.to_vec(), format, compression, parquet_write_schema) + .await?; + + if encoded.is_empty() { + return Ok(None); + } + + self.file_counter += 1; + let ext = format.file_extension(); + let file_name = format!( + "part-{subtask_idx:05}-epoch-{epoch:010}-{counter:06}.{ext}", + counter = self.file_counter + ); + let final_out = self.root.join(&file_name); + let tmp_out = self.root.join(format!("{file_name}.tmp")); + let size = encoded.len(); + + let start_io = Instant::now(); + { + let mut f = tokio::fs::File::create(&tmp_out).await?; + f.write_all(&encoded).await?; + f.sync_all().await?; + } + + if let Err(e) = tokio::fs::rename(&tmp_out, &final_out).await { + let _ = tokio::fs::remove_file(&tmp_out).await; + return Err(DeltaSinkError::Io(e)); + } + let io_duration = start_io.elapsed(); + + info!( + file = %file_name, + size_bytes = size, + records = record_count, + io_latency_ms = io_duration.as_millis(), + "delta data file atomically persisted locally" + ); + + Ok(Some(FinishedFile { + filename: file_name, + size, + record_count, + })) + } +} + +/// Offload CPU-heavy encoding to the blocking thread pool. +async fn encode_in_background( + owned_batches: Vec, + format: DeltaFormat, + compression: Compression, + parquet_write_schema: Option>, +) -> Result, DeltaSinkError> { + let start_cpu = Instant::now(); + let encoded = tokio::task::spawn_blocking(move || { + let batches = if format == DeltaFormat::Parquet { + if let Some(ref schema) = parquet_write_schema { + cast_batches_for_delta_write(&owned_batches, schema.as_ref()) + .map_err(|e| e.to_string())? + } else { + owned_batches + } + } else { + owned_batches + }; + + match format { + DeltaFormat::Csv => FormatEncoder::encode_csv(&batches).map_err(|e| e.to_string()), + DeltaFormat::Parquet => { + if let Some(ref schema) = parquet_write_schema { + FormatEncoder::encode_parquet_with_schema( + &batches, + Arc::clone(schema), + compression, + ) + .map_err(|e| e.to_string()) + } else { + FormatEncoder::encode_parquet(&batches, compression).map_err(|e| e.to_string()) + } + } + DeltaFormat::JsonL => FormatEncoder::encode_jsonl(&batches).map_err(|e| e.to_string()), + DeltaFormat::Avro => FormatEncoder::encode_avro(&batches).map_err(|e| e.to_string()), + DeltaFormat::Orc => FormatEncoder::encode_orc(&batches).map_err(|e| e.to_string()), + } + }) + .await + .map_err(|e| DeltaSinkError::SerializationPanic(format!("worker thread panicked: {e}")))? + .map_err(DeltaSinkError::SerializationPanic)?; + + debug!( + cpu_latency_ms = start_cpu.elapsed().as_millis(), + "record batch serialization completed" + ); + Ok(encoded) +} + +pub fn build_s3_object_store( + bucket: &str, + options: &HashMap, +) -> Result, DeltaSinkError> { + let region = options + .get(opt::S3_REGION) + .map(|s| s.as_str()) + .unwrap_or("us-east-1"); + + let retry_config = RetryConfig { + backoff: BackoffConfig { + init_backoff: Duration::from_millis(100), + max_backoff: Duration::from_secs(3), + base: 2.0, + }, + max_retries: 3, + retry_timeout: Duration::from_secs(120), + }; + + let mut builder = AmazonS3Builder::new() + .with_bucket_name(bucket) + .with_region(region) + .with_retry(retry_config); + + if let Some(endpoint) = options.get(opt::S3_ENDPOINT) { + builder = builder.with_endpoint(endpoint); + if endpoint.to_ascii_lowercase().starts_with("http://") { + builder = builder.with_allow_http(true); + } + builder = builder.with_virtual_hosted_style_request(false); + } + if let Some(v) = options.get(opt::S3_ACCESS_KEY_ID) { + builder = builder.with_access_key_id(v); + } + if let Some(v) = options.get(opt::S3_SECRET_ACCESS_KEY) { + builder = builder.with_secret_access_key(v); + } + if let Some(v) = options.get(opt::S3_SESSION_TOKEN) { + builder = builder.with_token(v); + } + + Ok(Arc::new( + builder.build().map_err(DeltaSinkError::ObjectStore)?, + )) +} + +pub async fn preflight_s3_bucket( + client: &dyn ObjectStore, + prefix: &str, + bucket: Option<&str>, +) -> Result<(), DeltaSinkError> { + let probe_path = if prefix.is_empty() { + ObjectStorePath::from("") + } else { + ObjectStorePath::from(prefix.to_string()) + }; + + let mut stream = client.list(Some(&probe_path)); + match futures::StreamExt::next(&mut stream).await { + None | Some(Ok(_)) => Ok(()), + Some(Err(object_store::Error::NotFound { .. })) => Ok(()), + Some(Err(e)) => { + let msg = e.to_string(); + let lower = msg.to_ascii_lowercase(); + if lower.contains("nosuchbucket") || lower.contains("the specified bucket") { + return Err(DeltaSinkError::Config(format!( + "S3 bucket '{}' does not exist", + bucket.unwrap_or("") + ))); + } + if lower.contains("invalidaccesskeyid") + || lower.contains("signaturedoesnotmatch") + || lower.contains("access denied") + || lower.contains("accessdenied") + { + return Err(DeltaSinkError::Config(format!( + "S3 authentication failed for bucket '{}': {msg}", + bucket.unwrap_or("") + ))); + } + Err(DeltaSinkError::Config(format!( + "failed to access S3 bucket '{}' (prefix '{}'): {msg}", + bucket.unwrap_or(""), + prefix + ))) + } + } +} + +impl DeltaFormat { + pub(crate) fn file_extension(self) -> &'static str { + match self { + DeltaFormat::Csv => "csv", + DeltaFormat::Parquet => "parquet", + DeltaFormat::JsonL => "jsonl", + DeltaFormat::Avro => "avro", + DeltaFormat::Orc => "orc", + } + } +} diff --git a/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs new file mode 100644 index 00000000..d4f2fcdd --- /dev/null +++ b/src/streaming_runtime_operators/src/operators/sink/delta/mod.rs @@ -0,0 +1,367 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Delta Lake sink with Arroyo-style separation: +//! - [`delta_writer`]: [`DeltaSink`] / [`DeltaLocalSink`] write `part-*` data files +//! - [`delta_commit`]: checkpoint-time `_delta_log` transaction commit +//! - [`DeltaSinkOperator`]: stream operator coordinator + +mod delta_commit; +mod delta_writer; + +pub use delta_commit::strip_streaming_system_columns_arc; +pub use delta_writer::FinishedFile; + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Once; + +use arrow_array::RecordBatch; +use arrow_schema::Schema as ArrowSchema; +use async_trait::async_trait; +use arrow::compute::concat_batches; +use delta_commit::{ + DeltaCommitStrategy, DeltaTableCommitter, build_delta_storage_options, resolve_delta_table_uri, +}; +use parquet::basic::Compression; +use tracing::{error, info, instrument, warn}; +use url::Url; + +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::memory::{MemoryBlock, try_global_memory_pool}; +use crate::sql::common::constants::factory_operator_name; +use crate::sql::common::with_option_keys as opt; +use crate::sql::common::{CheckpointBarrier, Watermark}; + +use delta_writer::{DeltaLocalSink, DeltaSink, preflight_s3_bucket}; + +const DEFAULT_MAX_BUFFER_BYTES: usize = 256 * 1024 * 1024; + +fn pending_row_count(batches: &[RecordBatch]) -> usize { + batches.iter().map(|b| b.num_rows()).sum() +} + +/// Merge many small stream batches into one Parquet row group where possible. +fn merge_pending_for_write(batches: Vec) -> Result, DeltaSinkError> { + if batches.len() <= 1 { + return Ok(batches); + } + let schema = batches[0].schema(); + let merged = concat_batches(&schema, &batches) + .map_err(|e| DeltaSinkError::Config(format!("failed to merge pending batches: {e}")))?; + Ok(vec![merged]) +} + +fn ensure_delta_handlers_registered() { + static REGISTER: Once = Once::new(); + REGISTER.call_once(|| { + deltalake::aws::register_handlers(None); + }); +} + +#[derive(thiserror::Error, Debug)] +pub enum DeltaSinkError { + #[error("local filesystem I/O error: {0}")] + Io(#[from] std::io::Error), + + #[error("object store error: {0}")] + ObjectStore(#[from] object_store::Error), + + #[error("serialization task panicked: {0}")] + SerializationPanic(String), + + #[error("delta committer failed: {0}")] + CommitterFailed(String), + + #[error("configuration error: {0}")] + Config(String), +} + +enum DeltaSinkBackend { + Local(DeltaLocalSink), + ObjectStore(DeltaSink), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DeltaFormat { + Csv, + Parquet, + JsonL, + Avro, + Orc, +} + +/// Stream operator: buffers batches, delegates data-file writes to [`DeltaSinkBackend`], +/// and commits `_delta_log` at checkpoint via [`DeltaTableCommitter`]. +pub struct DeltaSinkOperator { + table_name: String, + sink: DeltaSinkBackend, + parquet_compression: Compression, + pending: Vec, + pending_bytes: usize, + sink_memory_block: Option>, + early_flush_threshold_bytes: usize, + format: DeltaFormat, + committer: Option, + table_uri: Option, + storage_options: HashMap, + commit_strategy: DeltaCommitStrategy, + s3_bucket: Option, + sink_path: String, + catalog_schema: Option>, + parquet_write_schema: Option>, + /// Retained for local path creation in [`Operator::on_start`]. + local_root: Option, +} + +impl DeltaSinkOperator { + pub fn try_new( + table_name: String, + path: String, + format: DeltaFormat, + parquet_compression: Compression, + sink_memory_bytes: u64, + options: HashMap, + catalog_schema: Option>, + ) -> Result { + let s3_bucket = options.get(opt::S3_BUCKET).cloned(); + let (storage_options, commit_strategy) = build_delta_storage_options(&options)?; + + let (sink, local_root) = if let Some(bucket) = &s3_bucket { + let s3 = DeltaSink::try_new(bucket.clone(), path.clone(), &options)?; + (DeltaSinkBackend::ObjectStore(s3), None) + } else { + let root = PathBuf::from(path.clone()); + ( + DeltaSinkBackend::Local(DeltaLocalSink::new(root.clone())), + Some(root), + ) + }; + + let mut sink_memory_block = None; + let reserve_bytes = usize::try_from(sink_memory_bytes).unwrap_or(DEFAULT_MAX_BUFFER_BYTES); + let mut early_flush_threshold_bytes = reserve_bytes; + + if let Ok(pool) = try_global_memory_pool() { + if let Ok(block) = pool.try_request_block(reserve_bytes as u64) { + early_flush_threshold_bytes = ((block.capacity() as usize) * 8) / 10; + sink_memory_block = Some(block); + } + } + + Ok(Self { + table_name, + sink, + parquet_compression, + pending: Vec::with_capacity(64), + pending_bytes: 0, + sink_memory_block, + early_flush_threshold_bytes, + format, + committer: None, + table_uri: None, + storage_options, + commit_strategy, + s3_bucket, + sink_path: path, + catalog_schema: catalog_schema.and_then(strip_streaming_system_columns_arc), + parquet_write_schema: None, + local_root, + }) + } + + #[instrument(skip(self), fields(table = %self.table_name))] + async fn flush_data_file( + &mut self, + epoch: u64, + subtask_idx: usize, + ) -> Result<(), DeltaSinkError> { + if self.pending.is_empty() || pending_row_count(&self.pending) == 0 { + self.pending.clear(); + self.pending_bytes = 0; + return Ok(()); + } + + let fallback_schema = self.catalog_schema.is_none().then(|| { + strip_streaming_system_columns_arc(self.pending[0].schema()) + .expect("batch has no user columns after removing streaming system columns") + }); + + let batches = merge_pending_for_write(std::mem::take(&mut self.pending))?; + self.pending_bytes = 0; + + let compression = self.parquet_compression; + let format = self.format; + let schema = self.parquet_write_schema.clone(); + + let finished = match &mut self.sink { + DeltaSinkBackend::ObjectStore(sink) => { + sink.write_batches(&batches, format, compression, schema, epoch, subtask_idx) + .await? + } + DeltaSinkBackend::Local(sink) => { + sink.write_batches(&batches, format, compression, schema, epoch, subtask_idx) + .await? + } + }; + + let Some(finished) = finished else { + return Ok(()); + }; + + if let Some(committer) = self.committer.as_mut() { + if let Some(schema) = fallback_schema { + committer.update_schema(schema)?; + } + committer.register_uncommitted(finished.into_uncommitted()); + } + + Ok(()) + } + + async fn flush_and_commit_checkpoint( + &mut self, + epoch: u64, + subtask_idx: usize, + ) -> Result<(), DeltaSinkError> { + self.flush_data_file(epoch, subtask_idx).await?; + + if let Some(committer) = self.committer.as_mut() { + if committer.has_uncommitted() { + info!( + table = %self.table_name, + epoch, + subtask_idx, + "delta sink triggering checkpoint commit" + ); + committer.commit_checkpoint(epoch).await?; + } + } + Ok(()) + } +} + +#[async_trait] +impl Operator for DeltaSinkOperator { + fn name(&self) -> &str { + factory_operator_name::CONNECTOR_SINK + } + + async fn on_start(&mut self, _ctx: &mut TaskContext) -> anyhow::Result<()> { + ensure_delta_handlers_registered(); + + if let Some(root) = &self.local_root { + tokio::fs::create_dir_all(root).await?; + } + + if let DeltaSinkBackend::ObjectStore(sink) = &self.sink { + preflight_s3_bucket( + sink.client().as_ref(), + sink.prefix_path(), + self.s3_bucket.as_deref(), + ) + .await?; + } + + let table_uri = resolve_delta_table_uri(&self.sink_path, self.s3_bucket.as_deref())?; + self.table_uri = Some(table_uri.clone()); + + if self.format == DeltaFormat::Parquet { + let committer = DeltaTableCommitter::try_new( + table_uri, + self.storage_options.clone(), + self.catalog_schema.clone(), + )?; + self.parquet_write_schema = committer.write_schema(); + self.committer = Some(committer); + } else { + warn!( + format = ?self.format, + "format is not Parquet; writing raw files WITHOUT Delta transaction logs" + ); + } + + info!( + table = %self.table_name, + threshold_bytes = self.early_flush_threshold_bytes, + is_true_delta = self.committer.is_some(), + commit_strategy = %self.commit_strategy.label(), + "delta sink operator started (physical write + checkpoint commit)" + ); + Ok(()) + } + + async fn process_data( + &mut self, + _input_idx: usize, + batch: RecordBatch, + ctx: &mut TaskContext, + _collector: &mut dyn Collector, + ) -> anyhow::Result<()> { + if batch.num_rows() == 0 { + return Ok(()); + } + + self.pending_bytes += batch.get_array_memory_size(); + self.pending.push(batch); + + if self.pending_bytes > self.early_flush_threshold_bytes { + self.flush_data_file(0, ctx.subtask_index as usize).await?; + } + Ok(()) + } + + async fn process_watermark( + &mut self, + _watermark: Watermark, + _ctx: &mut TaskContext, + _collector: &mut dyn Collector, + ) -> anyhow::Result<()> { + Ok(()) + } + + #[instrument(skip(self, ctx), fields(epoch = barrier.epoch))] + async fn snapshot_state( + &mut self, + barrier: CheckpointBarrier, + ctx: &mut TaskContext, + ) -> anyhow::Result<()> { + self.flush_and_commit_checkpoint(barrier.epoch, ctx.subtask_index as usize) + .await + .map_err(anyhow::Error::from) + } + + async fn on_close(&mut self, ctx: &mut TaskContext) -> anyhow::Result> { + if !self.pending.is_empty() || self.committer.as_ref().is_some_and(|c| c.has_uncommitted()) + { + warn!( + table = %self.table_name, + "operator closing, forcing final flush and commit" + ); + if let Err(e) = self + .flush_and_commit_checkpoint(u64::MAX, ctx.subtask_index as usize) + .await + { + error!( + table = %self.table_name, + error = %e, + "fatal: final flush/commit on close failed; data may be lost" + ); + return Err(e.into()); + } + } + Ok(vec![]) + } +} diff --git a/src/runtime/streaming/operators/sink/filesystem/mod.rs b/src/streaming_runtime_operators/src/operators/sink/filesystem/mod.rs similarity index 96% rename from src/runtime/streaming/operators/sink/filesystem/mod.rs rename to src/streaming_runtime_operators/src/operators/sink/filesystem/mod.rs index a865a752..7aabfc3c 100644 --- a/src/runtime/streaming/operators/sink/filesystem/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/filesystem/mod.rs @@ -20,11 +20,11 @@ use parquet::basic::Compression; use tokio::io::AsyncWriteExt; use tracing::{debug, info, warn}; -use crate::runtime::memory::{MemoryBlock, try_global_memory_pool}; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::format::encoder::FormatEncoder; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::format::encoder::FormatEncoder; +use crate::memory::{MemoryBlock, try_global_memory_pool}; use crate::sql::common::constants::factory_operator_name; use crate::sql::common::{CheckpointBarrier, Watermark}; diff --git a/src/runtime/streaming/operators/sink/iceberg/mod.rs b/src/streaming_runtime_operators/src/operators/sink/iceberg/mod.rs similarity index 96% rename from src/runtime/streaming/operators/sink/iceberg/mod.rs rename to src/streaming_runtime_operators/src/operators/sink/iceberg/mod.rs index b6c17414..c89df325 100644 --- a/src/runtime/streaming/operators/sink/iceberg/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/iceberg/mod.rs @@ -26,11 +26,11 @@ use parquet::basic::Compression; use tokio::io::AsyncWriteExt; use tracing::{debug, info, warn}; -use crate::runtime::memory::{MemoryBlock, try_global_memory_pool}; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::format::encoder::FormatEncoder; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::format::encoder::FormatEncoder; +use crate::memory::{MemoryBlock, try_global_memory_pool}; use crate::sql::common::constants::factory_operator_name; use crate::sql::common::with_option_keys as opt; use crate::sql::common::{CheckpointBarrier, Watermark}; diff --git a/src/runtime/streaming/operators/sink/kafka/mod.rs b/src/streaming_runtime_operators/src/operators/sink/kafka/mod.rs similarity index 98% rename from src/runtime/streaming/operators/sink/kafka/mod.rs rename to src/streaming_runtime_operators/src/operators/sink/kafka/mod.rs index b30bc572..e973244f 100644 --- a/src/runtime/streaming/operators/sink/kafka/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/kafka/mod.rs @@ -36,10 +36,10 @@ use std::time::Duration; use tokio::time::sleep; use tracing::{info, warn}; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::format::DataSerializer; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::format::DataSerializer; use crate::sql::common::constants::factory_operator_name; use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark}; // ============================================================================ diff --git a/src/runtime/streaming/operators/sink/lancedb/mod.rs b/src/streaming_runtime_operators/src/operators/sink/lancedb/mod.rs similarity index 96% rename from src/runtime/streaming/operators/sink/lancedb/mod.rs rename to src/streaming_runtime_operators/src/operators/sink/lancedb/mod.rs index 4e8d8309..4c9ed7a9 100644 --- a/src/runtime/streaming/operators/sink/lancedb/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/lancedb/mod.rs @@ -23,9 +23,9 @@ use lance::dataset::{WriteMode, WriteParams}; use lance::io::{ObjectStoreParams, StorageOptionsAccessor}; use tracing::{info, warn}; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; use crate::sql::common::constants::factory_operator_name; use crate::sql::common::{CheckpointBarrier, Watermark}; diff --git a/src/runtime/streaming/operators/sink/mod.rs b/src/streaming_runtime_operators/src/operators/sink/mod.rs similarity index 100% rename from src/runtime/streaming/operators/sink/mod.rs rename to src/streaming_runtime_operators/src/operators/sink/mod.rs diff --git a/src/runtime/streaming/operators/sink/s3/mod.rs b/src/streaming_runtime_operators/src/operators/sink/s3/mod.rs similarity index 97% rename from src/runtime/streaming/operators/sink/s3/mod.rs rename to src/streaming_runtime_operators/src/operators/sink/s3/mod.rs index 715b5b86..1a454c17 100644 --- a/src/runtime/streaming/operators/sink/s3/mod.rs +++ b/src/streaming_runtime_operators/src/operators/sink/s3/mod.rs @@ -26,9 +26,9 @@ use parquet::basic::Compression; use parquet::file::properties::WriterProperties; use tracing::{info, warn}; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; use crate::sql::common::constants::factory_operator_name; use crate::sql::common::with_option_keys as opt; use crate::sql::common::{CheckpointBarrier, Watermark}; diff --git a/src/streaming_runtime_operators/src/operators/source/batch_buffer.rs b/src/streaming_runtime_operators/src/operators/source/batch_buffer.rs new file mode 100644 index 00000000..b958ad34 --- /dev/null +++ b/src/streaming_runtime_operators/src/operators/source/batch_buffer.rs @@ -0,0 +1,24 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Shared source batching defaults (Kafka, MQTT, HTTP, ROS, …). + +use std::time::Duration; + +/// Rows buffered before emitting a `RecordBatch`. +pub const DEFAULT_SOURCE_BATCH_SIZE: usize = 4096; + +/// Max wait when the buffer is non-empty but below `DEFAULT_SOURCE_BATCH_SIZE`. +pub const MAX_BATCH_LINGER_TIME: Duration = Duration::from_millis(20); + +/// Poll/read timeout in the hot loop; keep small to avoid idle sleeps between messages. +pub const SOURCE_POLL_TIMEOUT: Duration = Duration::from_millis(10); diff --git a/src/streaming_runtime_operators/src/operators/source/http/mod.rs b/src/streaming_runtime_operators/src/operators/source/http/mod.rs new file mode 100644 index 00000000..c2c701a1 --- /dev/null +++ b/src/streaming_runtime_operators/src/operators/source/http/mod.rs @@ -0,0 +1,449 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! HTTP source for robot / REST telemetry. +//! +//! - **poll** (Flink `url` + `scan.interval`): periodic GET/POST to robot APIs. +//! - **webhook** (`http.listen.port`): robots POST JSON payloads to Function Stream. + +use anyhow::{Context as _, Result, anyhow}; +use async_trait::async_trait; +use axum::{ + Router, + body::Bytes, + extract::State, + http::StatusCode, + routing::post, +}; +use governor::{DefaultDirectRateLimiter, Quota, RateLimiter as GovernorRateLimiter}; +use reqwest::{Client, Method}; +use std::collections::HashMap; +use std::num::NonZeroU32; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use tokio::sync::mpsc; +use tracing::{debug, error, info, warn}; + +use crate::core::api::context::TaskContext; +use crate::core::api::source::{SourceCheckpointReport, SourceEvent, SourceOperator}; +use crate::operators::source::kafka::BatchDeserializer; +use crate::sql::common::CheckpointBarrier; + +const HTTP_WAIT_TIMEOUT: Duration = Duration::from_millis(100); +const MAX_BATCH_LINGER_TIME: Duration = Duration::from_millis(500); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HttpSourceMode { + Poll, + Webhook, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HttpResponseSplit { + Single, + Ndjson, + JsonArray, +} + +pub fn proto_mode_to_runtime(mode: i32) -> HttpSourceMode { + use protocol::function_stream_graph::HttpSourceMode as ProtoMode; + match ProtoMode::try_from(mode) { + Ok(ProtoMode::HttpSourceWebhook) => HttpSourceMode::Webhook, + _ => HttpSourceMode::Poll, + } +} + +pub fn proto_split_to_runtime(split: i32) -> HttpResponseSplit { + use protocol::function_stream_graph::HttpResponseSplit as ProtoSplit; + match ProtoSplit::try_from(split) { + Ok(ProtoSplit::HttpResponseNdjson) => HttpResponseSplit::Ndjson, + Ok(ProtoSplit::HttpResponseJsonArray) => HttpResponseSplit::JsonArray, + _ => HttpResponseSplit::Single, + } +} + +fn wall_clock_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +struct InboundMessage { + payload: Vec, + timestamp_ms: u64, +} + +fn normalize_webhook_path(path: &str) -> String { + if path.is_empty() || path == "/" { + return "/".to_string(); + } + if path.starts_with('/') { + path.to_string() + } else { + format!("/{path}") + } +} + +fn split_response_body(body: &[u8], split: HttpResponseSplit) -> Result>> { + if body.is_empty() { + return Ok(vec![]); + } + match split { + HttpResponseSplit::Single => Ok(vec![body.to_vec()]), + HttpResponseSplit::Ndjson => Ok(body + .split(|&b| b == b'\n') + .filter(|line| !line.is_empty()) + .map(|line| line.to_vec()) + .collect()), + HttpResponseSplit::JsonArray => { + let value: serde_json::Value = + serde_json::from_slice(body).context("HTTP response is not valid JSON")?; + match value { + serde_json::Value::Array(items) => items + .into_iter() + .map(|v| serde_json::to_vec(&v)) + .collect::, _>>() + .context("failed to encode JSON array element"), + _ => Ok(vec![body.to_vec()]), + } + } + } +} + +async fn webhook_handler( + State(tx): State>, + body: Bytes, +) -> StatusCode { + if body.is_empty() { + return StatusCode::OK; + } + let msg = InboundMessage { + payload: body.to_vec(), + timestamp_ms: wall_clock_ms(), + }; + if tx.send(msg).await.is_err() { + warn!("HTTP webhook channel closed; dropping payload"); + } + StatusCode::OK +} + +pub struct HttpSourceOperator { + pub mode: HttpSourceMode, + pub url: String, + pub method: Method, + pub scan_interval: Duration, + pub request_body: Option, + pub headers: HashMap, + pub listen_host: String, + pub listen_port: u16, + pub webhook_path: String, + pub request_timeout: Duration, + pub response_split: HttpResponseSplit, + pub messages_per_second: Option, + + client: Option, + rate_limiter: Option, + deserializer: Box, + last_flush_time: Instant, + last_poll_time: Instant, + + inbound_rx: Option>, + _server_task: Option>, +} + +impl HttpSourceOperator { + #[allow(clippy::too_many_arguments)] + pub fn new( + mode: HttpSourceMode, + url: String, + method: Method, + scan_interval: Duration, + request_body: Option, + headers: HashMap, + listen_host: String, + listen_port: u16, + webhook_path: String, + request_timeout: Duration, + response_split: HttpResponseSplit, + messages_per_second: Option, + deserializer: Box, + ) -> Self { + Self { + mode, + url, + method, + scan_interval, + request_body, + headers, + listen_host, + listen_port, + webhook_path, + request_timeout, + response_split, + messages_per_second, + client: None, + rate_limiter: None, + deserializer, + last_flush_time: Instant::now(), + last_poll_time: Instant::now() + .checked_sub(scan_interval) + .unwrap_or_else(Instant::now), + inbound_rx: None, + _server_task: None, + } + } + + fn try_emit_buffered_batch(&mut self, reason: &str) -> Result> { + let should_flush_by_size = self.deserializer.should_flush(); + let should_flush_by_time = self.last_flush_time.elapsed() > MAX_BATCH_LINGER_TIME; + + if !self.deserializer.is_empty() + && (should_flush_by_size || should_flush_by_time) + && let Some(batch) = self.deserializer.flush_buffer()? + { + self.last_flush_time = Instant::now(); + debug!( + num_rows = batch.num_rows(), + flush_by_size = should_flush_by_size, + flush_by_time = should_flush_by_time, + reason, + "http source emitting record batch" + ); + return Ok(Some(SourceEvent::Data(batch))); + } + Ok(None) + } + + fn ingest_payloads(&mut self, payloads: Vec>, timestamp_ms: u64) -> Result<()> { + for payload in payloads { + if payload.is_empty() { + continue; + } + self.deserializer + .deserialize_slice(&payload, timestamp_ms, None)?; + } + Ok(()) + } + + async fn poll_http_once(&mut self) -> Result<()> { + let client = self + .client + .as_ref() + .ok_or_else(|| anyhow!("HTTP client not initialized"))?; + + let mut req = client.request(self.method.clone(), &self.url); + for (k, v) in &self.headers { + req = req.header(k.as_str(), v.as_str()); + } + if let Some(body) = &self.request_body { + req = req.body(body.clone()); + } + + let response = req + .send() + .await + .with_context(|| format!("HTTP {} {}", self.method, self.url))?; + + let status = response.status(); + let bytes = response + .bytes() + .await + .context("failed to read HTTP response body")?; + + if !status.is_success() { + warn!( + status = %status, + url = %self.url, + body_len = bytes.len(), + "HTTP poll returned non-success status" + ); + return Ok(()); + } + + debug!( + url = %self.url, + status = %status, + body_len = bytes.len(), + "http source poll response" + ); + + let chunks = split_response_body(&bytes, self.response_split)?; + self.ingest_payloads(chunks, wall_clock_ms())?; + Ok(()) + } + + async fn start_webhook_server(&mut self, ctx: &TaskContext) -> Result<()> { + if ctx.parallelism > 1 { + warn!( + job_id = %ctx.job_id, + subtask = ctx.subtask_index, + parallelism = ctx.parallelism, + "HTTP webhook source with parallelism > 1 will bind duplicate ports; use parallelism = 1" + ); + } + + let path = normalize_webhook_path(&self.webhook_path); + let addr = format!("{}:{}", self.listen_host, self.listen_port); + info!( + addr = %addr, + path = %path, + "Starting HTTP webhook server for robot telemetry" + ); + + let (tx, rx) = mpsc::channel::(4096); + let app = Router::new() + .route(&path, post(webhook_handler)) + .with_state(tx); + + let listener = tokio::net::TcpListener::bind(&addr) + .await + .with_context(|| format!("failed to bind HTTP webhook on {addr}"))?; + + let server = tokio::spawn(async move { + if let Err(e) = axum::serve(listener, app).await { + error!("HTTP webhook server error: {e}"); + } + }); + + self.inbound_rx = Some(rx); + self._server_task = Some(server); + Ok(()) + } + + async fn start_poll_client(&mut self) -> Result<()> { + info!( + url = %self.url, + method = %self.method, + interval_ms = self.scan_interval.as_millis(), + "Starting HTTP poll source for robot API" + ); + let client = Client::builder() + .timeout(self.request_timeout) + .build() + .context("failed to build HTTP client")?; + self.client = Some(client); + Ok(()) + } +} + +#[async_trait] +impl SourceOperator for HttpSourceOperator { + fn name(&self) -> &str { + match self.mode { + HttpSourceMode::Poll => &self.url, + HttpSourceMode::Webhook => &self.webhook_path, + } + } + + async fn on_start(&mut self, ctx: &mut TaskContext) -> Result<()> { + match self.mode { + HttpSourceMode::Poll => self.start_poll_client().await?, + HttpSourceMode::Webhook => self.start_webhook_server(ctx).await?, + } + self.rate_limiter = self.messages_per_second.map(|qps| { + GovernorRateLimiter::direct(Quota::per_second(qps)) + }); + Ok(()) + } + + async fn fetch_next(&mut self, _ctx: &mut TaskContext) -> Result { + loop { + if let Some(event) = self.try_emit_buffered_batch("linger")? { + return Ok(event); + } + + match self.mode { + HttpSourceMode::Webhook => { + let rx = self + .inbound_rx + .as_mut() + .ok_or_else(|| anyhow!("HTTP webhook receiver not initialized"))?; + + match tokio::time::timeout(HTTP_WAIT_TIMEOUT, rx.recv()).await { + Ok(Some(msg)) => { + let chunks = + split_response_body(&msg.payload, self.response_split)?; + self.ingest_payloads(chunks, msg.timestamp_ms)?; + + if let Some(rl) = &self.rate_limiter { + rl.until_ready().await; + } + + if let Some(event) = self.try_emit_buffered_batch("webhook")? { + return Ok(event); + } + } + Ok(None) => { + return Err(anyhow!("HTTP webhook channel closed")); + } + Err(_) => { + if let Some(event) = + self.try_emit_buffered_batch("webhook timeout")? + { + return Ok(event); + } + return Ok(SourceEvent::Idle); + } + } + } + HttpSourceMode::Poll => { + if self.last_poll_time.elapsed() >= self.scan_interval { + if let Err(e) = self.poll_http_once().await { + error!("HTTP poll error: {e:#}"); + return Err(e); + } + self.last_poll_time = Instant::now(); + + if let Some(rl) = &self.rate_limiter { + rl.until_ready().await; + } + + if let Some(event) = self.try_emit_buffered_batch("poll")? { + return Ok(event); + } + } else { + if let Some(event) = + self.try_emit_buffered_batch("poll wait")? + { + return Ok(event); + } + return Ok(SourceEvent::Idle); + } + } + } + } + } + + async fn snapshot_state( + &mut self, + barrier: CheckpointBarrier, + ctx: &mut TaskContext, + ) -> Result { + debug!( + subtask = ctx.subtask_index, + epoch = barrier.epoch, + mode = ?self.mode, + "HTTP source checkpoint (no offset state in v1)" + ); + Ok(SourceCheckpointReport::default()) + } + + async fn on_close(&mut self, _ctx: &mut TaskContext) -> Result<()> { + info!("HTTP source shutting down"); + if let Some(task) = self._server_task.take() { + task.abort(); + } + self.client.take(); + self.inbound_rx.take(); + Ok(()) + } +} diff --git a/src/runtime/streaming/operators/source/kafka/mod.rs b/src/streaming_runtime_operators/src/operators/source/kafka/mod.rs similarity index 72% rename from src/runtime/streaming/operators/source/kafka/mod.rs rename to src/streaming_runtime_operators/src/operators/source/kafka/mod.rs index 75edb968..67fcf647 100644 --- a/src/runtime/streaming/operators/source/kafka/mod.rs +++ b/src/streaming_runtime_operators/src/operators/source/kafka/mod.rs @@ -19,7 +19,6 @@ use anyhow::{Context as _, Result, anyhow}; use arrow_array::RecordBatch; use arrow_schema::SchemaRef; use async_trait::async_trait; -use governor::{DefaultDirectRateLimiter, Quota, RateLimiter as GovernorRateLimiter}; use protocol::storage::{ KafkaPartitionOffset, KafkaSourceSubtaskCheckpoint, SourceCheckpointInfo, source_checkpoint_info, @@ -27,15 +26,14 @@ use protocol::storage::{ use rdkafka::consumer::{CommitMode, Consumer, StreamConsumer}; use rdkafka::{ClientConfig, Message as KMessage, Offset, TopicPartitionList}; use std::collections::HashMap; -use std::num::NonZeroU32; use std::time::{Duration, Instant}; use tracing::{debug, error, info, warn}; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::source::{ - SourceCheckpointReport, SourceEvent, SourceOffset, SourceOperator, -}; -use crate::runtime::streaming::format::{BadDataPolicy, DataDeserializer, Format}; +use crate::operators::source::batch_buffer::{MAX_BATCH_LINGER_TIME, SOURCE_POLL_TIMEOUT}; + +use crate::core::api::context::TaskContext; +use crate::core::api::source::{SourceCheckpointReport, SourceEvent, SourceOffset, SourceOperator}; +use crate::format::{BadDataPolicy, DataDeserializer, Format}; use crate::sql::common::fs_schema::FieldValueType; use crate::sql::common::{CheckpointBarrier, MetadataField}; @@ -116,7 +114,11 @@ impl BatchDeserializer for BufferedDeserializer { } } -impl SourceOffset { +trait SourceOffsetExt { + fn rdkafka_offset(self) -> Offset; +} + +impl SourceOffsetExt for SourceOffset { fn rdkafka_offset(self) -> Offset { match self { SourceOffset::Earliest => Offset::Beginning, @@ -129,9 +131,6 @@ impl SourceOffset { // ============================================================================ // ============================================================================ -const KAFKA_POLL_TIMEOUT: Duration = Duration::from_millis(100); -const MAX_BATCH_LINGER_TIME: Duration = Duration::from_millis(500); - pub struct KafkaSourceOperator { pub topic: String, pub bootstrap_servers: String, @@ -140,11 +139,9 @@ pub struct KafkaSourceOperator { pub offset_mode: SourceOffset, pub client_configs: HashMap, - pub messages_per_second: NonZeroU32, pub metadata_fields: Vec, consumer: Option, - rate_limiter: Option, deserializer: Box, current_offsets: HashMap, @@ -164,7 +161,6 @@ impl KafkaSourceOperator { group_id_prefix: Option, offset_mode: SourceOffset, client_configs: HashMap, - messages_per_second: NonZeroU32, metadata_fields: Vec, deserializer: Box, ) -> Self { @@ -175,10 +171,8 @@ impl KafkaSourceOperator { group_id_prefix, offset_mode, client_configs, - messages_per_second, metadata_fields, consumer: None, - rate_limiter: None, deserializer, current_offsets: HashMap::new(), is_empty_assignment: false, @@ -236,6 +230,19 @@ impl KafkaSourceOperator { client_config.set(key, value); } + // High-throughput fetch defaults (override via client_configs in WITH clause). + for (k, v) in [ + ("fetch.min.bytes", "65536"), + ("fetch.max.bytes", "52428800"), + ("max.partition.fetch.bytes", "1048576"), + ("queued.min.messages", "500000"), + ("queued.max.messages.kbytes", "65536"), + ] { + if !self.client_configs.contains_key(k) { + client_config.set(k, v); + } + } + let consumer: StreamConsumer = client_config .set("bootstrap.servers", &self.bootstrap_servers) .set("enable.partition.eof", "false") @@ -290,6 +297,20 @@ impl KafkaSourceOperator { self.consumer = Some(consumer); Ok(()) } + + fn try_emit_buffered_batch(&mut self, _reason: &str) -> Result> { + let should_flush_by_size = self.deserializer.should_flush(); + let should_flush_by_time = self.last_flush_time.elapsed() > MAX_BATCH_LINGER_TIME; + + if !self.deserializer.is_empty() + && (should_flush_by_size || should_flush_by_time) + && let Some(batch) = self.deserializer.flush_buffer()? + { + self.last_flush_time = Instant::now(); + return Ok(Some(SourceEvent::Data(batch))); + } + Ok(None) + } } // ============================================================================ @@ -307,9 +328,6 @@ impl SourceOperator for KafkaSourceOperator { async fn on_start(&mut self, ctx: &mut TaskContext) -> Result<()> { self.init_and_assign_consumer(ctx).await?; - self.rate_limiter = Some(GovernorRateLimiter::direct(Quota::per_second( - self.messages_per_second, - ))); Ok(()) } @@ -318,87 +336,80 @@ impl SourceOperator for KafkaSourceOperator { return Ok(SourceEvent::Idle); } - let consumer = self - .consumer - .as_ref() - .ok_or_else(|| anyhow!("Kafka consumer not initialized"))?; - let rate_limiter = self - .rate_limiter - .as_ref() - .ok_or_else(|| anyhow!("rate limiter not initialized"))?; - - match tokio::time::timeout(KAFKA_POLL_TIMEOUT, consumer.recv()).await { - Ok(Ok(msg)) => { - let partition = msg.partition(); - let offset = msg.offset(); - let timestamp = msg.timestamp().to_millis().ok_or_else(|| { - anyhow!("Failed to read timestamp from Kafka record: message has no timestamp") - })?; - - self.current_offsets.insert(partition, offset); - - if let Some(payload) = msg.payload() { - let topic = msg.topic(); - - let connector_metadata = if !self.metadata_fields.is_empty() { - let mut meta = HashMap::new(); - for f in &self.metadata_fields { - meta.insert( - f.field_name.as_str(), - match f.key.as_str() { - "key" => FieldValueType::Bytes(msg.key()), - "offset_id" => FieldValueType::Int64(Some(msg.offset())), - "partition" => FieldValueType::Int32(Some(msg.partition())), - "topic" => FieldValueType::String(Some(topic)), - "timestamp" => FieldValueType::Int64(Some(timestamp)), - _ => continue, - }, - ); - } - Some(meta) - } else { - None - }; - - self.deserializer.deserialize_slice( - payload, - timestamp.max(0) as u64, - connector_metadata, - )?; - } else { - debug!( - "Received tombstone message at partition {} offset {}", - partition, offset - ); - } - - rate_limiter.until_ready().await; - - let should_flush_by_size = self.deserializer.should_flush(); - let should_flush_by_time = self.last_flush_time.elapsed() > MAX_BATCH_LINGER_TIME; + // Keep polling while the in-memory buffer is filling; returning Idle here would make + // SourceDriver sleep `source_idle_timeout` (50ms) between every single message. + loop { + if let Some(event) = self.try_emit_buffered_batch("linger")? { + return Ok(event); + } - if !self.deserializer.is_empty() - && (should_flush_by_size || should_flush_by_time) - && let Some(batch) = self.deserializer.flush_buffer()? - { - self.last_flush_time = Instant::now(); - return Ok(SourceEvent::Data(batch)); + let recv_result = { + let consumer = self + .consumer + .as_ref() + .ok_or_else(|| anyhow!("Kafka consumer not initialized"))?; + tokio::time::timeout(SOURCE_POLL_TIMEOUT, consumer.recv()).await + }; + + match recv_result { + Ok(Ok(msg)) => { + let partition = msg.partition(); + let offset = msg.offset(); + let timestamp = msg.timestamp().to_millis().ok_or_else(|| { + anyhow!( + "Failed to read timestamp from Kafka record: message has no timestamp" + ) + })?; + + self.current_offsets.insert(partition, offset); + + if let Some(payload) = msg.payload() { + let topic = msg.topic(); + + let connector_metadata = if !self.metadata_fields.is_empty() { + let mut meta = HashMap::new(); + for f in &self.metadata_fields { + meta.insert( + f.field_name.as_str(), + match f.key.as_str() { + "key" => FieldValueType::Bytes(msg.key()), + "offset_id" => FieldValueType::Int64(Some(msg.offset())), + "partition" => FieldValueType::Int32(Some(msg.partition())), + "topic" => FieldValueType::String(Some(topic)), + "timestamp" => FieldValueType::Int64(Some(timestamp)), + _ => continue, + }, + ); + } + Some(meta) + } else { + None + }; + + self.deserializer.deserialize_slice( + payload, + timestamp.max(0) as u64, + connector_metadata, + )?; + } + + if let Some(event) = self.try_emit_buffered_batch("batch full")? { + return Ok(event); + } + // Buffer not ready: poll again immediately (do not return Idle). } - - Ok(SourceEvent::Idle) - } - Ok(Err(e)) => { - error!("Kafka recv error: {}", e); - Err(anyhow!("Kafka error: {}", e)) - } - Err(_) => { - if !self.deserializer.is_empty() - && let Some(batch) = self.deserializer.flush_buffer()? - { - self.last_flush_time = Instant::now(); - return Ok(SourceEvent::Data(batch)); + Ok(Err(e)) => { + error!("Kafka recv error: {}", e); + return Err(anyhow!("Kafka error: {}", e)); + } + Err(_) => { + if let Some(event) = + self.try_emit_buffered_batch("poll timeout")? + { + return Ok(event); + } + return Ok(SourceEvent::Idle); } - Ok(SourceEvent::Idle) } } } diff --git a/src/runtime/wasm/input/protocol/mod.rs b/src/streaming_runtime_operators/src/operators/source/mod.rs similarity index 87% rename from src/runtime/wasm/input/protocol/mod.rs rename to src/streaming_runtime_operators/src/operators/source/mod.rs index b9574391..b9eb1d5f 100644 --- a/src/runtime/wasm/input/protocol/mod.rs +++ b/src/streaming_runtime_operators/src/operators/source/mod.rs @@ -10,4 +10,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod batch_buffer; +pub mod http; pub mod kafka; +pub mod mqtt; +pub mod robot_bag; +pub mod ros; diff --git a/src/streaming_runtime_operators/src/operators/source/mqtt/mod.rs b/src/streaming_runtime_operators/src/operators/source/mqtt/mod.rs new file mode 100644 index 00000000..36fdbc11 --- /dev/null +++ b/src/streaming_runtime_operators/src/operators/source/mqtt/mod.rs @@ -0,0 +1,243 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! MQTT source: subscribe on connect, batch deserialize like Kafka source. +//! Checkpoint v1 reports no offset state (at-least-once; use stable `client_id` +//! and `clean_session=false` on the broker if you need redelivery after restart). + +use anyhow::{Context as _, Result, anyhow}; +use async_trait::async_trait; +use rumqttc::{AsyncClient, Event, EventLoop, MqttOptions, Packet, QoS}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; +use tracing::{debug, error, info, warn}; + +use crate::core::api::context::TaskContext; +use crate::core::api::source::{SourceCheckpointReport, SourceEvent, SourceOperator}; +use crate::operators::source::batch_buffer::{MAX_BATCH_LINGER_TIME, SOURCE_POLL_TIMEOUT}; +use crate::operators::source::kafka::BatchDeserializer; +use crate::sql::common::{CheckpointBarrier, MetadataField}; + +pub fn proto_qos_to_rumqtt(qos: u32) -> QoS { + match qos { + 0 => QoS::AtMostOnce, + 2 => QoS::ExactlyOnce, + _ => QoS::AtLeastOnce, + } +} + +fn wall_clock_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +pub struct MqttSourceOperator { + pub topic: String, + pub host: String, + pub port: u16, + pub client_id: Option, + pub username: Option, + pub password: Option, + pub qos: QoS, + pub clean_session: bool, + pub keep_alive_secs: u64, + pub metadata_fields: Vec, + + _client: Option, + eventloop: Option, + deserializer: Box, + last_flush_time: Instant, +} + +impl MqttSourceOperator { + #[allow(clippy::too_many_arguments)] + pub fn new( + topic: String, + host: String, + port: u16, + client_id: Option, + username: Option, + password: Option, + qos: QoS, + clean_session: bool, + keep_alive_secs: u64, + metadata_fields: Vec, + deserializer: Box, + ) -> Self { + Self { + topic, + host, + port, + client_id, + username, + password, + qos, + clean_session, + keep_alive_secs, + metadata_fields, + _client: None, + eventloop: None, + deserializer, + last_flush_time: Instant::now(), + } + } + + fn try_emit_buffered_batch(&mut self, reason: &str) -> Result> { + let should_flush_by_size = self.deserializer.should_flush(); + let should_flush_by_time = self.last_flush_time.elapsed() > MAX_BATCH_LINGER_TIME; + + if !self.deserializer.is_empty() + && (should_flush_by_size || should_flush_by_time) + && let Some(batch) = self.deserializer.flush_buffer()? + { + self.last_flush_time = Instant::now(); + debug!( + num_rows = batch.num_rows(), + num_columns = batch.num_columns(), + flush_by_size = should_flush_by_size, + flush_by_time = should_flush_by_time, + reason, + "mqtt source emitting record batch" + ); + return Ok(Some(SourceEvent::Data(batch))); + } + Ok(None) + } + + async fn connect(&mut self, ctx: &mut TaskContext) -> Result<()> { + if ctx.parallelism > 1 { + warn!( + job_id = %ctx.job_id, + subtask = ctx.subtask_index, + parallelism = ctx.parallelism, + "MQTT source with pipeline parallelism > 1 will duplicate messages across subtasks; use parallelism = 1" + ); + } + + let client_id = self.client_id.clone().unwrap_or_else(|| { + format!("fs-{}-{}-mqtt", ctx.job_id, ctx.subtask_index) + }); + + info!( + host = %self.host, + port = self.port, + topic = %self.topic, + client_id = %client_id, + "Creating MQTT client" + ); + + let mut mqttoptions = MqttOptions::new(client_id, &self.host, self.port); + mqttoptions.set_keep_alive(std::time::Duration::from_secs(self.keep_alive_secs.max(1))); + mqttoptions.set_clean_session(self.clean_session); + if let (Some(user), Some(pass)) = (&self.username, &self.password) { + mqttoptions.set_credentials(user, pass); + } + + let (client, eventloop) = AsyncClient::new(mqttoptions, 100); + client + .subscribe(self.topic.clone(), self.qos) + .await + .with_context(|| format!("Failed to subscribe to MQTT topic '{}'", self.topic))?; + + self._client = Some(client); + self.eventloop = Some(eventloop); + Ok(()) + } +} + +#[async_trait] +impl SourceOperator for MqttSourceOperator { + fn name(&self) -> &str { + &self.topic + } + + async fn on_start(&mut self, ctx: &mut TaskContext) -> Result<()> { + self.connect(ctx).await?; + Ok(()) + } + + async fn fetch_next(&mut self, _ctx: &mut TaskContext) -> Result { + loop { + if let Some(event) = self.try_emit_buffered_batch("linger")? { + return Ok(event); + } + + let poll_result = { + let eventloop = self + .eventloop + .as_mut() + .ok_or_else(|| anyhow!("MQTT event loop not initialized"))?; + tokio::time::timeout(SOURCE_POLL_TIMEOUT, eventloop.poll()).await + }; + + match poll_result { + Ok(Ok(Event::Incoming(Packet::Publish(publish)))) => { + let payload = publish.payload.as_ref(); + let timestamp_ms = wall_clock_ms(); + + if !payload.is_empty() { + debug!( + topic = %publish.topic, + payload_bytes = payload.len(), + qos = ?publish.qos, + "mqtt source consumed message" + ); + let _ = &self.metadata_fields; + self.deserializer.deserialize_slice( + payload, + timestamp_ms, + None, + )?; + } + + if let Some(event) = self.try_emit_buffered_batch("batch full")? { + return Ok(event); + } + } + Ok(Ok(_)) => { + // ConnAck, SubAck, PingResp, etc. + } + Ok(Err(e)) => { + error!("MQTT poll error: {}", e); + return Err(anyhow!("MQTT error: {}", e)); + } + Err(_) => { + if let Some(event) = self.try_emit_buffered_batch("poll timeout")? { + return Ok(event); + } + return Ok(SourceEvent::Idle); + } + } + } + } + + async fn snapshot_state( + &mut self, + barrier: CheckpointBarrier, + ctx: &mut TaskContext, + ) -> Result { + debug!( + subtask = ctx.subtask_index, + epoch = barrier.epoch, + "MQTT source checkpoint (no offset state in v1)" + ); + Ok(SourceCheckpointReport::default()) + } + + async fn on_close(&mut self, _ctx: &mut TaskContext) -> Result<()> { + info!("MQTT source shutting down"); + self._client.take(); + self.eventloop.take(); + Ok(()) + } +} diff --git a/src/streaming_runtime_operators/src/operators/source/robot_bag/mod.rs b/src/streaming_runtime_operators/src/operators/source/robot_bag/mod.rs new file mode 100644 index 00000000..91dbe278 --- /dev/null +++ b/src/streaming_runtime_operators/src/operators/source/robot_bag/mod.rs @@ -0,0 +1,460 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Robot recording file source: MCAP, ROS1 `.bag`, ROS2 `.db3`, PCD, JSONL/NDJSON, CSV. + +mod pcd; +mod pcd_body; +mod ros1_bag; + +use anyhow::{Context as _, Result, anyhow}; +use async_trait::async_trait; +use pcd::{PcdEmitMode, load_pcd_paths, parse_pcd_file, pcd_to_messages}; +use ros1_bag::{is_ros1_bag_file, load_ros1_bag}; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use tracing::{debug, info, warn}; + +use crate::core::api::context::TaskContext; +use crate::core::api::source::{SourceCheckpointReport, SourceEvent, SourceOperator}; +use crate::operators::source::batch_buffer::{MAX_BATCH_LINGER_TIME, SOURCE_POLL_TIMEOUT}; +use crate::operators::source::kafka::BatchDeserializer; +use crate::sql::common::CheckpointBarrier; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RobotBagFormat { + Auto, + Mcap, + Ros1, + Ros2, + Pcd, + Jsonl, + Csv, +} + +pub fn proto_bag_format_to_runtime(fmt: i32) -> RobotBagFormat { + use protocol::function_stream_graph::RobotBagFormat as Proto; + match Proto::try_from(fmt) { + Ok(Proto::RobotBagMcap) => RobotBagFormat::Mcap, + Ok(Proto::RobotBagRos1) => RobotBagFormat::Ros1, + Ok(Proto::RobotBagRos2) => RobotBagFormat::Ros2, + Ok(Proto::RobotBagPcd) => RobotBagFormat::Pcd, + Ok(Proto::RobotBagJsonl) => RobotBagFormat::Jsonl, + Ok(Proto::RobotBagCsv) => RobotBagFormat::Csv, + _ => RobotBagFormat::Auto, + } +} + +fn wall_clock_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +#[derive(Clone)] +struct BagMessage { + payload: Vec, + timestamp_ms: u64, +} + +fn directory_has_pcd_files(dir: &Path) -> bool { + std::fs::read_dir(dir) + .ok() + .into_iter() + .flatten() + .filter_map(|e| e.ok()) + .any(|e| { + e.path() + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("pcd")) + }) +} + +fn detect_format(path: &Path) -> RobotBagFormat { + if path.is_dir() { + if directory_has_pcd_files(path) { + return RobotBagFormat::Pcd; + } + return RobotBagFormat::Ros2; + } + if is_ros1_bag_file(path) { + return RobotBagFormat::Ros1; + } + match path.extension().and_then(|e| e.to_str()).map(|s| s.to_ascii_lowercase()) { + Some(ext) if ext == "mcap" => RobotBagFormat::Mcap, + Some(ext) if ext == "bag" => RobotBagFormat::Ros1, + Some(ext) if ext == "db3" => RobotBagFormat::Ros2, + Some(ext) if ext == "pcd" => RobotBagFormat::Pcd, + Some(ext) if ext == "jsonl" || ext == "ndjson" => RobotBagFormat::Jsonl, + Some(ext) if ext == "csv" => RobotBagFormat::Csv, + _ => RobotBagFormat::Jsonl, + } +} + +fn resolve_format(path: &Path, fmt: RobotBagFormat) -> RobotBagFormat { + if fmt == RobotBagFormat::Auto { + detect_format(path) + } else { + fmt + } +} + +fn find_ros2_db3(path: &Path) -> Result { + if path.is_file() { + return Ok(path.to_path_buf()); + } + if path.is_dir() { + for entry in std::fs::read_dir(path).with_context(|| format!("read dir {}", path.display()))? + { + let p = entry?.path(); + if p.extension().is_some_and(|e| e == "db3") { + return Ok(p); + } + } + } + Err(anyhow!( + "no ROS2 bag .db3 file found under '{}'", + path.display() + )) +} + +fn load_mcap(path: &Path, topic_filter: Option<&str>) -> Result> { + let data = std::fs::read(path).with_context(|| format!("read mcap {}", path.display()))?; + + let mut out = Vec::new(); + for msg in mcap::MessageStream::new(&data).context("mcap message stream")? { + let msg = msg.context("mcap message")?; + let topic = msg.channel.topic.as_str(); + if let Some(filter) = topic_filter + && topic != filter + { + continue; + } + out.push(BagMessage { + payload: msg.data.to_vec(), + timestamp_ms: msg.log_time / 1_000_000, + }); + } + out.sort_by_key(|m| m.timestamp_ms); + info!( + path = %path.display(), + messages = out.len(), + topic_filter = ?topic_filter, + "loaded MCAP robot bag" + ); + Ok(out) +} + +fn load_ros1(path: &Path, topic_filter: Option<&str>) -> Result> { + let raw = load_ros1_bag(path, topic_filter)?; + Ok(raw + .into_iter() + .map(|(payload, timestamp_ms)| BagMessage { + payload, + timestamp_ms, + }) + .collect()) +} + +fn load_ros2_db3(path: &Path, topic_filter: Option<&str>) -> Result> { + let db3 = find_ros2_db3(path)?; + let conn = rusqlite::Connection::open(&db3) + .with_context(|| format!("open ros2 bag sqlite {}", db3.display()))?; + + let mut out = Vec::new(); + if let Some(topic) = topic_filter { + let mut stmt = conn.prepare( + "SELECT m.timestamp, m.data FROM messages m \ + JOIN topics t ON m.topic_id = t.id \ + WHERE t.name = ?1 ORDER BY m.timestamp", + )?; + let rows = stmt.query_map([topic], |row| { + let ts: i64 = row.get(0)?; + let data: Vec = row.get(1)?; + Ok((ts, data)) + })?; + for row in rows { + let (ts, data) = row?; + out.push(BagMessage { + payload: data, + timestamp_ms: (ts / 1_000_000).max(0) as u64, + }); + } + } else { + let mut stmt = + conn.prepare("SELECT timestamp, data FROM messages ORDER BY timestamp")?; + let rows = stmt.query_map([], |row| { + let ts: i64 = row.get(0)?; + let data: Vec = row.get(1)?; + Ok((ts, data)) + })?; + for row in rows { + let (ts, data) = row?; + out.push(BagMessage { + payload: data, + timestamp_ms: (ts / 1_000_000).max(0) as u64, + }); + } + } + + info!( + db3 = %db3.display(), + messages = out.len(), + topic_filter = ?topic_filter, + "loaded ROS2 bag" + ); + Ok(out) +} + +fn load_pcd(path: &Path, emit_mode: PcdEmitMode) -> Result> { + let files = load_pcd_paths(path)?; + let mut out = Vec::new(); + let base_ts = wall_clock_ms(); + + for (file_idx, file_path) in files.iter().enumerate() { + let cloud = parse_pcd_file(file_path)?; + let file_ts = base_ts.saturating_add(file_idx as u64); + let msgs = pcd_to_messages(&cloud, emit_mode, file_ts)?; + info!( + path = %file_path.display(), + points = cloud.header.points, + data_mode = ?cloud.header.data_mode, + emit_mode = ?emit_mode, + messages = msgs.len(), + "loaded PCD point cloud" + ); + for (payload, ts) in msgs { + out.push(BagMessage { + payload, + timestamp_ms: ts, + }); + } + } + Ok(out) +} + +fn load_jsonl(path: &Path) -> Result> { + let file = File::open(path).with_context(|| format!("open jsonl {}", path.display()))?; + let reader = BufReader::new(file); + let mut out = Vec::new(); + for line in reader.lines() { + let line = line?; + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + out.push(BagMessage { + payload: trimmed.as_bytes().to_vec(), + timestamp_ms: wall_clock_ms(), + }); + } + info!(path = %path.display(), messages = out.len(), "loaded JSONL robot log"); + Ok(out) +} + +fn load_csv(path: &Path) -> Result> { + let file = File::open(path).with_context(|| format!("open csv {}", path.display()))?; + let reader = BufReader::new(file); + let mut out = Vec::new(); + let mut first = true; + for line in reader.lines() { + let line = line?; + if first { + first = false; + continue; + } + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + out.push(BagMessage { + payload: trimmed.as_bytes().to_vec(), + timestamp_ms: wall_clock_ms(), + }); + } + info!(path = %path.display(), messages = out.len(), "loaded CSV robot log"); + Ok(out) +} + +fn load_bag_messages( + path: &Path, + format: RobotBagFormat, + topic_filter: Option<&str>, + pcd_emit_mode: PcdEmitMode, +) -> Result> { + let fmt = resolve_format(path, format); + match fmt { + RobotBagFormat::Mcap => load_mcap(path, topic_filter), + RobotBagFormat::Ros1 => load_ros1(path, topic_filter), + RobotBagFormat::Ros2 => load_ros2_db3(path, topic_filter), + RobotBagFormat::Pcd => load_pcd(path, pcd_emit_mode), + RobotBagFormat::Jsonl => load_jsonl(path), + RobotBagFormat::Csv => load_csv(path), + RobotBagFormat::Auto => unreachable!("resolved above"), + } +} + +pub struct RobotBagSourceOperator { + path: PathBuf, + bag_format: RobotBagFormat, + topic_filter: Option, + pcd_emit_mode: PcdEmitMode, + replay_interval: Duration, + loop_replay: bool, + deserializer: Box, + messages: Vec, + cursor: usize, + last_flush_time: Instant, + last_emit_time: Instant, +} + +impl RobotBagSourceOperator { + pub fn new( + path: String, + bag_format: RobotBagFormat, + topic_filter: Option, + pcd_emit_mode: PcdEmitMode, + replay_interval: Duration, + loop_replay: bool, + deserializer: Box, + ) -> Self { + Self { + path: PathBuf::from(path), + bag_format, + topic_filter, + pcd_emit_mode, + replay_interval, + loop_replay, + deserializer, + messages: Vec::new(), + cursor: 0, + last_flush_time: Instant::now(), + last_emit_time: Instant::now(), + } + } + + fn try_emit_buffered_batch(&mut self, reason: &str) -> Result> { + let should_flush_by_size = self.deserializer.should_flush(); + let should_flush_by_time = self.last_flush_time.elapsed() > MAX_BATCH_LINGER_TIME; + + if !self.deserializer.is_empty() + && (should_flush_by_size || should_flush_by_time) + && let Some(batch) = self.deserializer.flush_buffer()? + { + self.last_flush_time = Instant::now(); + debug!(num_rows = batch.num_rows(), reason, "robot-bag emitting batch"); + return Ok(Some(SourceEvent::Data(batch))); + } + Ok(None) + } + + fn reload_messages(&mut self) -> Result<()> { + self.messages = load_bag_messages( + &self.path, + self.bag_format, + self.topic_filter.as_deref(), + self.pcd_emit_mode, + )?; + self.cursor = 0; + Ok(()) + } +} + +#[async_trait] +impl SourceOperator for RobotBagSourceOperator { + fn name(&self) -> &str { + self.path.to_str().unwrap_or("robot-bag") + } + + async fn on_start(&mut self, _ctx: &mut TaskContext) -> Result<()> { + self.reload_messages()?; + if self.messages.is_empty() { + warn!(path = %self.path.display(), "robot bag loaded zero messages"); + } + Ok(()) + } + + async fn fetch_next(&mut self, _ctx: &mut TaskContext) -> Result { + loop { + if let Some(event) = self.try_emit_buffered_batch("linger")? { + return Ok(event); + } + + if self.cursor >= self.messages.len() { + if !self.deserializer.is_empty() + && let Some(batch) = self.deserializer.flush_buffer()? + { + self.last_flush_time = Instant::now(); + return Ok(SourceEvent::Data(batch)); + } + if self.loop_replay { + info!(path = %self.path.display(), "robot bag loop replay"); + self.reload_messages()?; + if self.messages.is_empty() { + return Ok(SourceEvent::Idle); + } + } else { + return Ok(SourceEvent::EndOfStream); + } + } + + if !self.replay_interval.is_zero() + && self.last_emit_time.elapsed() < self.replay_interval + { + tokio::time::sleep(SOURCE_POLL_TIMEOUT).await; + continue; + } + + let msg = self.messages[self.cursor].clone(); + self.cursor += 1; + self.deserializer + .deserialize_slice(&msg.payload, msg.timestamp_ms, None)?; + self.last_emit_time = Instant::now(); + + if let Some(event) = self.try_emit_buffered_batch("batch full")? { + return Ok(event); + } + } + } + + async fn snapshot_state( + &mut self, + barrier: CheckpointBarrier, + ctx: &mut TaskContext, + ) -> Result { + debug!( + subtask = ctx.subtask_index, + epoch = barrier.epoch, + cursor = self.cursor, + "robot-bag checkpoint (file offset v1: cursor only in memory)" + ); + Ok(SourceCheckpointReport::default()) + } + + async fn on_close(&mut self, _ctx: &mut TaskContext) -> Result<()> { + info!(path = %self.path.display(), "robot-bag source closed"); + Ok(()) + } +} + +pub fn parse_pcd_emit_mode(value: Option<&str>) -> Result { + match value.map(|s| s.to_ascii_lowercase()).as_deref() { + None | Some("point") | Some("points") => Ok(PcdEmitMode::Point), + Some("cloud") | Some("file") => Ok(PcdEmitMode::Cloud), + Some(other) => Err(anyhow!( + "invalid pcd.emit.mode '{other}'; expected 'point' or 'cloud'" + )), + } +} diff --git a/src/streaming_runtime_operators/src/operators/source/robot_bag/pcd.rs b/src/streaming_runtime_operators/src/operators/source/robot_bag/pcd.rs new file mode 100644 index 00000000..d8c4ba99 --- /dev/null +++ b/src/streaming_runtime_operators/src/operators/source/robot_bag/pcd.rs @@ -0,0 +1,461 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Strict PCL PCD v0.7 parser (ascii / binary / binary_compressed). + +use anyhow::{Context as _, Result, anyhow, bail}; +use super::pcd_body::{ + append_binary_point_fields, decompress_binary_compressed_body, fill_point_object_from_flat_values, + find_data_body_offset, parse_ascii_rows, PcdDataMode, PcdHeader, +}; +use serde_json::{Map, Value, json}; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PcdEmitMode { + Point, + Cloud, +} + +#[derive(Debug, Clone)] +pub struct PcdPointCloud { + pub header: PcdHeader, + pub source_path: PathBuf, + aos_body: Vec, + ascii_rows: Vec>, +} + +impl PcdPointCloud { + pub fn point_to_json(&self, index: usize) -> Result { + if index >= self.header.points { + bail!("PCD point index {index} out of range (points={})", self.header.points); + } + let mut obj = Map::new(); + obj.insert( + "_source_file".into(), + Value::String( + self.source_path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("") + .to_string(), + ), + ); + obj.insert("_point_index".into(), json!(index)); + + match self.header.data_mode { + PcdDataMode::Ascii => { + fill_point_object_from_flat_values( + &self.header, + &self.ascii_rows[index], + &mut obj, + )?; + } + PcdDataMode::Binary | PcdDataMode::BinaryCompressed => { + append_binary_point_fields(&self.aos_body, &self.header, index, &mut obj)?; + } + } + Ok(Value::Object(obj)) + } + + pub fn cloud_to_json(&self) -> Result { + let mut points = Vec::with_capacity(self.header.points); + for i in 0..self.header.points { + let mut point = self.point_to_json(i)?; + if let Some(map) = point.as_object_mut() { + map.remove("_source_file"); + } + points.push(point); + } + Ok(json!({ + "_source_file": self.source_path.file_name().and_then(|s| s.to_str()).unwrap_or(""), + "width": self.header.width, + "height": self.header.height, + "points": self.header.points, + "fields": self.header.fields, + "viewpoint": self.header.viewpoint, + "data": points, + })) + } +} + +pub fn load_pcd_paths(path: &Path) -> Result> { + if path.is_file() { + return Ok(vec![path.to_path_buf()]); + } + if path.is_dir() { + let mut files: Vec = std::fs::read_dir(path) + .with_context(|| format!("read pcd dir {}", path.display()))? + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| { + p.is_file() + && p.extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("pcd")) + }) + .collect(); + files.sort(); + if files.is_empty() { + bail!("no .pcd files found under '{}'", path.display()); + } + return Ok(files); + } + bail!("pcd path '{}' is neither file nor directory", path.display()) +} + +pub fn parse_pcd_file(path: &Path) -> Result { + let data = std::fs::read(path).with_context(|| format!("read pcd {}", path.display()))?; + let body_offset = find_data_body_offset(&data)?; + let header = parse_pcd_header(&data[..body_offset])?; + let body = &data[body_offset..]; + + let (aos_body, ascii_rows) = match header.data_mode { + PcdDataMode::Ascii => { + let text = std::str::from_utf8(body).context("pcd ascii body must be utf-8")?; + (Vec::new(), parse_ascii_rows(text, &header)?) + } + PcdDataMode::Binary => { + let expected = header.points * header.point_step; + if body.len() < expected { + bail!( + "PCD binary body too short: need {expected} bytes, got {}", + body.len() + ); + } + (body[..expected].to_vec(), Vec::new()) + } + PcdDataMode::BinaryCompressed => { + (decompress_binary_compressed_body(body, &header)?, Vec::new()) + } + }; + + Ok(PcdPointCloud { + header, + source_path: path.to_path_buf(), + aos_body, + ascii_rows, + }) +} + +pub fn pcd_to_messages( + cloud: &PcdPointCloud, + emit_mode: PcdEmitMode, + timestamp_ms: u64, +) -> Result, u64)>> { + match emit_mode { + PcdEmitMode::Point => { + let mut out = Vec::with_capacity(cloud.header.points); + for i in 0..cloud.header.points { + let json = cloud.point_to_json(i)?; + out.push((serde_json::to_vec(&json)?, timestamp_ms)); + } + Ok(out) + } + PcdEmitMode::Cloud => { + let json = cloud.cloud_to_json()?; + Ok(vec![(serde_json::to_vec(&json)?, timestamp_ms)]) + } + } +} + +fn parse_pcd_header(data: &[u8]) -> Result { + let text = std::str::from_utf8(data).context("pcd header must be utf-8")?; + let mut version = None; + let mut fields = None; + let mut sizes = None; + let mut types = None; + let mut counts = None; + let mut width = None; + let mut height = None; + let mut viewpoint = None; + let mut points = None; + let mut data_mode = None; + + for line in text.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let Some((key, value)) = line.split_once(' ') else { + continue; + }; + let key = key.trim().to_ascii_uppercase(); + let value = value.trim(); + match key.as_str() { + "VERSION" => version = Some(value.to_string()), + "FIELDS" => { + fields = Some( + value + .split_whitespace() + .map(|s| s.to_string()) + .collect::>(), + ); + } + "SIZE" => sizes = Some(parse_u32_list(value, "SIZE")?), + "TYPE" => types = Some(parse_type_tokens(value)?), + "COUNT" => counts = Some(parse_u32_list(value, "COUNT")?), + "WIDTH" => width = Some(value.parse::().context("WIDTH")?), + "HEIGHT" => height = Some(value.parse::().context("HEIGHT")?), + "VIEWPOINT" => { + let vals = parse_f64_list(value, "VIEWPOINT")?; + if vals.len() != 7 { + bail!("VIEWPOINT must contain 7 values, got {}", vals.len()); + } + let mut vp = [0.0; 7]; + vp.copy_from_slice(&vals); + viewpoint = Some(vp); + } + "POINTS" => points = Some(value.parse::().context("POINTS")?), + "DATA" => { + data_mode = Some(parse_data_mode(value)?); + break; + } + _ => {} + } + } + + version.ok_or_else(|| anyhow!("PCD header missing VERSION"))?; + let fields = fields.ok_or_else(|| anyhow!("PCD header missing FIELDS"))?; + let sizes = sizes.ok_or_else(|| anyhow!("PCD header missing SIZE"))?; + let types = types.ok_or_else(|| anyhow!("PCD header missing TYPE"))?; + let counts = counts.unwrap_or_else(|| vec![1; fields.len()]); + let width = width.ok_or_else(|| anyhow!("PCD header missing WIDTH"))?; + let height = height.ok_or_else(|| anyhow!("PCD header missing HEIGHT"))?; + let viewpoint = viewpoint.unwrap_or([0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0]); + let points = points.ok_or_else(|| anyhow!("PCD header missing POINTS"))?; + let data_mode = data_mode.ok_or_else(|| anyhow!("PCD header missing DATA"))?; + + if fields.len() != sizes.len() || fields.len() != types.len() || fields.len() != counts.len() + { + bail!( + "PCD FIELDS/SIZE/TYPE/COUNT length mismatch: {}, {}, {}, {}", + fields.len(), + sizes.len(), + types.len(), + counts.len() + ); + } + for (i, ((&size, &count), &ty)) in sizes.iter().zip(&counts).zip(&types).enumerate() { + validate_field_type(size, ty) + .with_context(|| format!("invalid field '{}' at index {}", fields[i], i))?; + if count == 0 { + bail!("PCD COUNT for field '{}' must be >= 1", fields[i]); + } + } + if width == 0 || height == 0 { + bail!("PCD WIDTH/HEIGHT must be > 0"); + } + if width * height != points as u32 { + bail!( + "PCD POINTS ({points}) must equal WIDTH * HEIGHT ({} * {} = {})", + width, + height, + width * height + ); + } + + let mut field_offsets = Vec::with_capacity(fields.len()); + let mut point_step = 0usize; + for (&size, &count) in sizes.iter().zip(&counts) { + field_offsets.push(point_step); + point_step += (size * count) as usize; + } + + Ok(PcdHeader { + fields, + sizes, + types, + counts, + width, + height, + viewpoint, + points, + data_mode, + point_step, + field_offsets, + }) +} + +fn parse_type_tokens(value: &str) -> Result> { + let parts: Vec<&str> = value.split_whitespace().collect(); + if parts.is_empty() { + bail!("PCD TYPE is empty"); + } + if parts.len() > 1 { + Ok(parts.iter().filter_map(|p| p.chars().next()).collect()) + } else { + Ok(parts[0].chars().collect()) + } +} + +fn parse_data_mode(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "ascii" => Ok(PcdDataMode::Ascii), + "binary" => Ok(PcdDataMode::Binary), + "binary_compressed" => Ok(PcdDataMode::BinaryCompressed), + other => bail!( + "unsupported PCD DATA mode '{other}'; expected ascii, binary, or binary_compressed" + ), + } +} + +fn parse_u32_list(value: &str, key: &str) -> Result> { + value + .split_whitespace() + .map(|v| v.parse::().with_context(|| format!("{key} value '{v}'"))) + .collect() +} + +fn parse_f64_list(value: &str, key: &str) -> Result> { + value + .split_whitespace() + .map(|v| v.parse::().with_context(|| format!("{key} value '{v}'"))) + .collect() +} + +fn validate_field_type(size: u32, ty: char) -> Result<()> { + match (ty, size) { + ('I' | 'U', 1 | 2 | 4) | ('F', 4 | 8) => Ok(()), + (other, sz) => bail!("unsupported PCD TYPE '{other}' with SIZE {sz}"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const ASCII_PCD: &str = "\ +# .PCD v0.7 - Point Cloud Data file format +VERSION .7 +FIELDS x y z intensity +SIZE 4 4 4 4 +TYPE F F F F +COUNT 1 1 1 1 +WIDTH 2 +HEIGHT 1 +VIEWPOINT 0 0 0 1 0 0 0 +POINTS 2 +DATA ascii +1.0 2.0 3.0 0.5 +4.0 5.0 6.0 0.8 +"; + + fn write_temp(name: &str, content: impl AsRef<[u8]>) -> PathBuf { + let dir = std::env::temp_dir().join("fs_pcd_test"); + let _ = std::fs::create_dir_all(&dir); + let path = dir.join(name); + std::fs::write(&path, content.as_ref()).expect("write"); + path + } + + #[test] + fn parse_ascii_pcd_header_and_points() { + let path = write_temp("ascii.pcd", ASCII_PCD); + let cloud = parse_pcd_file(&path).expect("parse"); + assert_eq!(cloud.header.points, 2); + assert_eq!(cloud.header.data_mode, PcdDataMode::Ascii); + + let p0 = cloud.point_to_json(0).expect("p0"); + assert_eq!(p0["x"], 1.0); + assert_eq!(p0["y"], 2.0); + assert_eq!(p0["z"], 3.0); + assert_eq!(p0["intensity"], 0.5); + let p1 = cloud.point_to_json(1).expect("p1"); + assert_eq!(p1["x"], 4.0); + + let msgs = pcd_to_messages(&cloud, PcdEmitMode::Point, 0).expect("msgs"); + assert_eq!(msgs.len(), 2); + let _ = std::fs::remove_file(path); + } + + #[test] + fn parse_binary_pcd_xyz() { + let mut body: Vec = Vec::new(); + for (x, y, z) in [(1.0_f32, 2.0_f32, 3.0_f32), (4.0_f32, 5.0_f32, 6.0_f32)] { + body.extend_from_slice(&x.to_le_bytes()); + body.extend_from_slice(&y.to_le_bytes()); + body.extend_from_slice(&z.to_le_bytes()); + } + let header = r"# .PCD v0.7 +VERSION .7 +FIELDS x y z +SIZE 4 4 4 +TYPE F F F +COUNT 1 1 1 +WIDTH 2 +HEIGHT 1 +VIEWPOINT 0 0 0 1 0 0 0 +POINTS 2 +DATA binary +"; + let mut file = header.as_bytes().to_vec(); + file.extend(body); + let path = write_temp("binary.pcd", file); + let cloud = parse_pcd_file(&path).expect("parse"); + let p0 = cloud.point_to_json(0).expect("p0"); + assert!((p0["x"].as_f64().unwrap() - 1.0).abs() < 1e-5); + let _ = std::fs::remove_file(path); + } + + #[test] + fn parse_binary_compressed_pcd() { + let points = 64usize; + let mut aos = Vec::with_capacity(points * 12); + for i in 0..points { + let f = i as f32; + aos.extend_from_slice(&f.to_le_bytes()); + aos.extend_from_slice(&(f + 100.0).to_le_bytes()); + aos.extend_from_slice(&(f + 200.0).to_le_bytes()); + } + + let mut soa = vec![0u8; points * 12]; + for field in 0..3usize { + let field_bytes = 4; + let soa_base = field * points * field_bytes; + for pt in 0..points { + let aos_off = pt * 12 + field * field_bytes; + let dst = soa_base + pt * field_bytes; + soa[dst..dst + field_bytes].copy_from_slice(&aos[aos_off..aos_off + field_bytes]); + } + } + + let compressed = lzf::compress(&soa).expect("compress"); + let mut body = Vec::new(); + body.extend_from_slice(&(compressed.len() as u32).to_le_bytes()); + body.extend_from_slice(&(soa.len() as u32).to_le_bytes()); + body.extend_from_slice(&compressed); + + let header = format!( + "# .PCD v0.7\n\ +VERSION .7\n\ +FIELDS x y z\n\ +SIZE 4 4 4\n\ +TYPE F F F\n\ +COUNT 1 1 1\n\ +WIDTH {points}\n\ +HEIGHT 1\n\ +VIEWPOINT 0 0 0 1 0 0 0\n\ +POINTS {points}\n\ +DATA binary_compressed\n" + ); + let mut file = header.into_bytes(); + file.extend(body); + let path = write_temp("compressed.pcd", file); + let cloud = parse_pcd_file(&path).expect("parse"); + let p1 = cloud.point_to_json(1).expect("p1"); + assert!((p1["x"].as_f64().unwrap() - 1.0).abs() < 1e-5); + assert!((p1["z"].as_f64().unwrap() - 201.0).abs() < 1e-5); + + let cloud_msgs = pcd_to_messages(&cloud, PcdEmitMode::Cloud, 0).expect("cloud"); + assert_eq!(cloud_msgs.len(), 1); + let _ = std::fs::remove_file(path); + } +} diff --git a/src/streaming_runtime_operators/src/operators/source/robot_bag/pcd_body.rs b/src/streaming_runtime_operators/src/operators/source/robot_bag/pcd_body.rs new file mode 100644 index 00000000..17ff2b68 --- /dev/null +++ b/src/streaming_runtime_operators/src/operators/source/robot_bag/pcd_body.rs @@ -0,0 +1,230 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use anyhow::{Context as _, Result, bail}; +use byteorder::{ByteOrder, LittleEndian}; +use serde_json::{Map, Value, json}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PcdDataMode { + Ascii, + Binary, + BinaryCompressed, +} + +#[derive(Debug, Clone)] +pub struct PcdHeader { + pub fields: Vec, + pub sizes: Vec, + pub types: Vec, + pub counts: Vec, + pub width: u32, + pub height: u32, + pub viewpoint: [f64; 7], + pub points: usize, + pub data_mode: PcdDataMode, + pub point_step: usize, + pub field_offsets: Vec, +} + +pub(crate) fn find_data_body_offset(data: &[u8]) -> Result { + let mut offset = 0usize; + while offset < data.len() { + let line_start = offset; + let line_end = data[offset..] + .iter() + .position(|&b| b == b'\n') + .map(|pos| line_start + pos) + .unwrap_or(data.len()); + let line_bytes = &data[line_start..line_end]; + let line = std::str::from_utf8(line_bytes).context("pcd header line must be utf-8")?; + let trimmed = line.trim(); + if !trimmed.is_empty() + && !trimmed.starts_with('#') + && trimmed.to_ascii_uppercase().starts_with("DATA ") + { + return Ok(if line_end < data.len() { + line_end + 1 + } else { + line_end + }); + } + if line_end >= data.len() { + break; + } + offset = line_end + 1; + } + bail!("PCD header missing DATA line") +} + +pub(crate) fn decompress_binary_compressed_body(body: &[u8], header: &PcdHeader) -> Result> { + if body.len() < 8 { + bail!("binary_compressed body too short"); + } + let compressed_size = LittleEndian::read_u32(&body[0..4]) as usize; + let uncompressed_size = LittleEndian::read_u32(&body[4..8]) as usize; + if body.len() < 8 + compressed_size { + bail!( + "binary_compressed truncated: need {} bytes, got {}", + 8 + compressed_size, + body.len() + ); + } + let compressed = &body[8..8 + compressed_size]; + let decompressed = lzf::decompress(compressed, uncompressed_size) + .map_err(|e| anyhow::anyhow!("LZF decompress failed: {e:?}"))?; + if decompressed.len() != uncompressed_size { + bail!( + "LZF decompressed size mismatch: expected {uncompressed_size}, got {}", + decompressed.len() + ); + } + + let expected: usize = header + .sizes + .iter() + .zip(&header.counts) + .map(|(&s, &c)| header.points * (s * c) as usize) + .sum(); + if decompressed.len() != expected { + bail!( + "binary_compressed uncompressed payload size mismatch: expected {expected}, got {}", + decompressed.len() + ); + } + + let mut aos = vec![0u8; header.points * header.point_step]; + for (field_idx, (&size, &count)) in header.sizes.iter().zip(&header.counts).enumerate() { + let field_bytes = (size * count) as usize; + let soa_offset: usize = header + .sizes + .iter() + .zip(&header.counts) + .take(field_idx) + .map(|(&s, &c)| header.points * (s * c) as usize) + .sum(); + let point_offset = header.field_offsets[field_idx]; + for pt in 0..header.points { + let src = soa_offset + pt * field_bytes; + let dst = pt * header.point_step + point_offset; + aos[dst..dst + field_bytes].copy_from_slice(&decompressed[src..src + field_bytes]); + } + } + Ok(aos) +} + +pub(crate) fn parse_ascii_rows(body: &str, header: &PcdHeader) -> Result>> { + let values_per_point: usize = header.counts.iter().map(|c| *c as usize).sum(); + let mut rows = Vec::with_capacity(header.points); + let mut tokens = body.split_whitespace(); + for pt in 0..header.points { + let mut row = Vec::with_capacity(values_per_point); + for _ in 0..values_per_point { + let Some(tok) = tokens.next() else { + bail!("PCD ascii data ended early at point {pt}"); + }; + row.push(tok.parse::().with_context(|| format!("ascii token '{tok}'"))?); + } + rows.push(row); + } + Ok(rows) +} + +pub(crate) fn fill_point_object_from_flat_values( + header: &PcdHeader, + values: &[f64], + obj: &mut Map, +) -> Result<()> { + let mut cursor = 0; + for (field_idx, field_name) in header.fields.iter().enumerate() { + let count = header.counts[field_idx] as usize; + if cursor + count > values.len() { + bail!("PCD ascii row too short for field '{field_name}'"); + } + insert_field_values(obj, field_name, &values[cursor..cursor + count]); + cursor += count; + } + Ok(()) +} + +pub(crate) fn append_binary_point_fields( + body: &[u8], + header: &PcdHeader, + point_index: usize, + obj: &mut Map, +) -> Result<()> { + let start = point_index * header.point_step; + let end = start + header.point_step; + if end > body.len() { + bail!( + "PCD binary data too short for point {point_index}: need {end}, got {}", + body.len() + ); + } + fill_point_object_from_bytes(header, &body[start..end], obj) +} + +fn fill_point_object_from_bytes( + header: &PcdHeader, + point_bytes: &[u8], + obj: &mut Map, +) -> Result<()> { + for (field_idx, field_name) in header.fields.iter().enumerate() { + let offset = header.field_offsets[field_idx]; + let size = header.sizes[field_idx] as usize; + let count = header.counts[field_idx] as usize; + let mut values = Vec::with_capacity(count); + for elem in 0..count { + let start = offset + elem * size; + let end = start + size; + if end > point_bytes.len() { + bail!("PCD binary point truncated at field '{field_name}'"); + } + values.push(decode_field_value( + header.types[field_idx], + size, + &point_bytes[start..end], + )?); + } + insert_field_values(obj, field_name, &values); + } + Ok(()) +} + +pub(crate) fn decode_field_value(ty: char, size: usize, bytes: &[u8]) -> Result { + let v = match (ty, size) { + ('I', 1) => bytes[0] as i8 as f64, + ('I', 2) => LittleEndian::read_i16(bytes) as f64, + ('I', 4) => LittleEndian::read_i32(bytes) as f64, + ('U', 1) => bytes[0] as f64, + ('U', 2) => LittleEndian::read_u16(bytes) as f64, + ('U', 4) => LittleEndian::read_u32(bytes) as f64, + ('F', 4) => LittleEndian::read_f32(bytes) as f64, + ('F', 8) => LittleEndian::read_f64(bytes), + _ => bail!("unsupported PCD field TYPE '{ty}' SIZE {size}"), + }; + Ok(v) +} + +pub(crate) fn insert_field_values(obj: &mut Map, field_name: &str, values: &[f64]) { + match values.len() { + 0 => {} + 1 => { + obj.insert(field_name.to_string(), json!(values[0])); + } + _ => { + for (i, v) in values.iter().enumerate() { + obj.insert(format!("{field_name}_{i}"), json!(v)); + } + } + } +} diff --git a/src/streaming_runtime_operators/src/operators/source/robot_bag/ros1_bag.rs b/src/streaming_runtime_operators/src/operators/source/robot_bag/ros1_bag.rs new file mode 100644 index 00000000..f499aa28 --- /dev/null +++ b/src/streaming_runtime_operators/src/operators/source/robot_bag/ros1_bag.rs @@ -0,0 +1,145 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! ROS1 rosbag (v1/v2) loader using the `rosbag` crate. + +use anyhow::{Context as _, Result, anyhow, bail}; +use rosbag::{ChunkRecord, IndexRecord, MessageRecord, RosBag, record_types::MessageData}; +use std::collections::HashMap; +use std::io::Read; +use std::path::Path; +use tracing::info; + +pub fn is_ros1_bag_file(path: &Path) -> bool { + let Ok(mut file) = std::fs::File::open(path) else { + return false; + }; + let mut buf = [0u8; 13]; + let Ok(n) = file.read(&mut buf) else { + return false; + }; + if n < 13 { + return false; + } + std::str::from_utf8(&buf).ok().is_some_and(|s| { + s.starts_with("#ROSBAG V2.0") || s.starts_with("#ROSBAG V1.02") || s.starts_with("#ROSBAG V1.0") + }) +} + +pub fn load_ros1_bag(path: &Path, topic_filter: Option<&str>) -> Result, u64)>> { + if !path.is_file() { + bail!("ROS1 bag path '{}' must be a .bag file", path.display()); + } + if !is_ros1_bag_file(path) { + bail!( + "file '{}' is not a ROS1 rosbag (expected header '#ROSBAG V2.0' or '#ROSBAG V1.02')", + path.display() + ); + } + + let bag = RosBag::new(path).with_context(|| format!("open ros1 bag {}", path.display()))?; + + let mut conn_topics: HashMap = HashMap::new(); + + for record in bag.index_records() { + match record.context("ros1 bag index record")? { + IndexRecord::Connection(conn) => { + conn_topics.insert( + conn.id, + (conn.topic.to_string(), conn.tp.to_string()), + ); + } + IndexRecord::IndexData(_) | IndexRecord::ChunkInfo(_) => {} + } + } + + let mut messages = Vec::new(); + + for record in bag.chunk_records() { + match record.context("ros1 bag chunk record")? { + ChunkRecord::Chunk(chunk) => { + let mut chunk_messages = Vec::new(); + for msg in chunk.messages() { + match msg.context("ros1 bag message record")? { + MessageRecord::Connection(conn) => { + conn_topics.insert( + conn.id, + (conn.topic.to_string(), conn.tp.to_string()), + ); + } + MessageRecord::MessageData(msg_data) => { + chunk_messages.push(msg_data); + } + } + } + for msg_data in chunk_messages { + push_ros1_message(&mut messages, &conn_topics, msg_data, topic_filter); + } + } + ChunkRecord::IndexData(_) => {} + } + } + + messages.sort_by_key(|m| m.1); + + info!( + path = %path.display(), + messages = messages.len(), + topic_filter = ?topic_filter, + connections = conn_topics.len(), + "loaded ROS1 rosbag" + ); + + if messages.is_empty() && topic_filter.is_some() { + return Err(anyhow!( + "ROS1 bag '{}' has no messages for topic '{}'", + path.display(), + topic_filter.unwrap_or("") + )); + } + + Ok(messages) +} + +fn push_ros1_message( + out: &mut Vec<(Vec, u64)>, + conn_topics: &HashMap, + msg_data: MessageData<'_>, + topic_filter: Option<&str>, +) { + let Some((topic, _)) = conn_topics.get(&msg_data.conn_id) else { + return; + }; + if let Some(filter) = topic_filter + && topic != filter + { + return; + } + out.push((msg_data.data.to_vec(), msg_data.time / 1_000_000)); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reject_non_bag_file() { + let dir = std::env::temp_dir().join("fs_ros1_test"); + let _ = std::fs::create_dir_all(&dir); + let path = dir.join("not_bag.txt"); + std::fs::write(&path, b"hello").expect("write"); + assert!(!is_ros1_bag_file(&path)); + let err = load_ros1_bag(&path, None).unwrap_err(); + assert!(err.to_string().contains("not a ROS1 rosbag")); + let _ = std::fs::remove_file(path); + } +} diff --git a/src/streaming_runtime_operators/src/operators/source/ros/mod.rs b/src/streaming_runtime_operators/src/operators/source/ros/mod.rs new file mode 100644 index 00000000..4a88e073 --- /dev/null +++ b/src/streaming_runtime_operators/src/operators/source/ros/mod.rs @@ -0,0 +1,272 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! ROS source via [rosbridge](https://github.com/RobotWebTools/rosbridge_suite) WebSocket protocol. +//! +//! Connects to `rosbridge_websocket`, subscribes to a ROS topic, extracts the `msg` payload +//! (or a custom JSON field), and deserializes with the configured format. + +use anyhow::{Context as _, Result, anyhow}; +use async_trait::async_trait; +use futures::{SinkExt, StreamExt}; +use serde_json::Value; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; +use tokio::time::timeout; +use tokio_tungstenite::{ + connect_async, + tungstenite::{Message, client::IntoClientRequest}, +}; +use tracing::{debug, error, info, warn}; + +use crate::core::api::context::TaskContext; +use crate::core::api::source::{SourceCheckpointReport, SourceEvent, SourceOperator}; +use crate::operators::source::batch_buffer::{MAX_BATCH_LINGER_TIME, SOURCE_POLL_TIMEOUT}; +use crate::operators::source::kafka::BatchDeserializer; +use crate::sql::common::CheckpointBarrier; + +fn wall_clock_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +fn extract_ros_payload(msg: &Value, message_field: &str) -> Option> { + let payload = msg.get(message_field).unwrap_or(msg); + match payload { + Value::String(s) => Some(s.as_bytes().to_vec()), + Value::Object(_) | Value::Array(_) => serde_json::to_vec(payload).ok(), + Value::Null => None, + other => serde_json::to_vec(other).ok(), + } +} + +pub struct RosSourceOperator { + pub url: String, + pub topic: String, + pub message_field: String, + deserializer: Box, + last_flush_time: Instant, + ws_write: Option< + futures::stream::SplitSink< + tokio_tungstenite::WebSocketStream>, + Message, + >, + >, + ws_read: Option< + futures::stream::SplitStream< + tokio_tungstenite::WebSocketStream>, + >, + >, +} + +impl RosSourceOperator { + pub fn new( + url: String, + topic: String, + message_field: String, + deserializer: Box, + ) -> Self { + Self { + url, + topic, + message_field, + deserializer, + last_flush_time: Instant::now(), + ws_write: None, + ws_read: None, + } + } + + fn try_emit_buffered_batch(&mut self, reason: &str) -> Result> { + let should_flush_by_size = self.deserializer.should_flush(); + let should_flush_by_time = self.last_flush_time.elapsed() > MAX_BATCH_LINGER_TIME; + + if !self.deserializer.is_empty() + && (should_flush_by_size || should_flush_by_time) + && let Some(batch) = self.deserializer.flush_buffer()? + { + self.last_flush_time = Instant::now(); + debug!( + num_rows = batch.num_rows(), + reason, + "ros source emitting record batch" + ); + return Ok(Some(SourceEvent::Data(batch))); + } + Ok(None) + } + + async fn connect_and_subscribe(&mut self) -> Result<()> { + info!( + url = %self.url, + topic = %self.topic, + "Connecting to rosbridge WebSocket" + ); + + let request = self + .url + .as_str() + .into_client_request() + .context("invalid rosbridge WebSocket URL")?; + + let (ws, _) = connect_async(request) + .await + .with_context(|| format!("failed to connect rosbridge at {}", self.url))?; + + let (write, read) = ws.split(); + self.ws_write = Some(write); + self.ws_read = Some(read); + + let subscribe = serde_json::json!({ + "op": "subscribe", + "topic": self.topic, + }); + let write = self + .ws_write + .as_mut() + .ok_or_else(|| anyhow!("rosbridge write half not initialized"))?; + write + .send(Message::Text(subscribe.to_string().into())) + .await + .context("failed to send rosbridge subscribe")?; + + info!(topic = %self.topic, "Subscribed to ROS topic via rosbridge"); + Ok(()) + } + + fn handle_rosbridge_text(&mut self, text: &str) -> Result { + let msg: Value = match serde_json::from_str(text) { + Ok(v) => v, + Err(e) => { + debug!(error = %e, "ignoring non-JSON rosbridge frame"); + return Ok(false); + } + }; + + let op = msg.get("op").and_then(|v| v.as_str()).unwrap_or(""); + if op != "publish" { + return Ok(false); + } + + let topic = msg.get("topic").and_then(|v| v.as_str()).unwrap_or(""); + if topic != self.topic { + return Ok(false); + } + + let Some(payload) = extract_ros_payload(&msg, &self.message_field) else { + return Ok(false); + }; + + debug!( + topic, + payload_bytes = payload.len(), + "ros source received message" + ); + self.deserializer + .deserialize_slice(&payload, wall_clock_ms(), None)?; + Ok(true) + } +} + +#[async_trait] +impl SourceOperator for RosSourceOperator { + fn name(&self) -> &str { + &self.topic + } + + async fn on_start(&mut self, ctx: &mut TaskContext) -> Result<()> { + if ctx.parallelism > 1 { + warn!( + subtask = ctx.subtask_index, + parallelism = ctx.parallelism, + "ROS source with parallelism > 1 duplicates rosbridge subscriptions; use parallelism = 1" + ); + } + self.connect_and_subscribe().await + } + + async fn fetch_next(&mut self, _ctx: &mut TaskContext) -> Result { + loop { + if let Some(event) = self.try_emit_buffered_batch("linger")? { + return Ok(event); + } + + let read_result = { + let read = self + .ws_read + .as_mut() + .ok_or_else(|| anyhow!("rosbridge read half not initialized"))?; + timeout(SOURCE_POLL_TIMEOUT, read.next()).await + }; + + match read_result { + Ok(Some(Ok(Message::Text(text)))) => { + if self.handle_rosbridge_text(text.as_ref())? { + if let Some(event) = self.try_emit_buffered_batch("batch full")? { + return Ok(event); + } + } + } + Ok(Some(Ok(Message::Binary(bin)))) => { + if let Ok(text) = std::str::from_utf8(&bin) { + if self.handle_rosbridge_text(text)? { + if let Some(event) = self.try_emit_buffered_batch("batch full")? { + return Ok(event); + } + } + } + } + Ok(Some(Ok(Message::Ping(_))) | Some(Ok(Message::Pong(_)))) => {} + Ok(Some(Ok(Message::Close(_)))) => { + return Err(anyhow!("rosbridge WebSocket closed")); + } + Ok(Some(Ok(_))) => {} + Ok(Some(Err(e))) => { + error!("rosbridge WebSocket error: {e}"); + return Err(anyhow!("rosbridge error: {e}")); + } + Ok(None) => { + return Err(anyhow!("rosbridge WebSocket stream ended")); + } + Err(_) => { + if let Some(event) = self.try_emit_buffered_batch("poll timeout")? { + return Ok(event); + } + return Ok(SourceEvent::Idle); + } + } + } + } + + async fn snapshot_state( + &mut self, + barrier: CheckpointBarrier, + ctx: &mut TaskContext, + ) -> Result { + debug!( + subtask = ctx.subtask_index, + epoch = barrier.epoch, + "ROS source checkpoint (no offset state in v1)" + ); + Ok(SourceCheckpointReport::default()) + } + + async fn on_close(&mut self, _ctx: &mut TaskContext) -> Result<()> { + info!("ROS source shutting down"); + if let Some(mut write) = self.ws_write.take() { + let _ = write.close().await; + } + self.ws_read.take(); + Ok(()) + } +} diff --git a/src/runtime/streaming/operators/stateless_physical_executor.rs b/src/streaming_runtime_operators/src/operators/stateless_physical_executor.rs similarity index 98% rename from src/runtime/streaming/operators/stateless_physical_executor.rs rename to src/streaming_runtime_operators/src/operators/stateless_physical_executor.rs index eb595d31..11b1e307 100644 --- a/src/runtime/streaming/operators/stateless_physical_executor.rs +++ b/src/streaming_runtime_operators/src/operators/stateless_physical_executor.rs @@ -24,7 +24,7 @@ use datafusion_proto::protobuf::PhysicalPlanNode; use futures::StreamExt; use prost::Message; -use crate::runtime::streaming::factory::Registry; +use crate::factory::Registry; use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; pub struct StatelessPhysicalExecutor { diff --git a/src/runtime/streaming/operators/value_execution.rs b/src/streaming_runtime_operators/src/operators/value_execution.rs similarity index 89% rename from src/runtime/streaming/operators/value_execution.rs rename to src/streaming_runtime_operators/src/operators/value_execution.rs index b93cd78b..dc93eeb6 100644 --- a/src/runtime/streaming/operators/value_execution.rs +++ b/src/streaming_runtime_operators/src/operators/value_execution.rs @@ -15,10 +15,10 @@ use arrow_array::RecordBatch; use async_trait::async_trait; use futures::StreamExt; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::operators::StatelessPhysicalExecutor; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::operators::StatelessPhysicalExecutor; use crate::sql::common::{CheckpointBarrier, Watermark}; pub struct ValueExecutionOperator { diff --git a/src/runtime/streaming/operators/watermark/mod.rs b/src/streaming_runtime_operators/src/operators/watermark/mod.rs similarity index 100% rename from src/runtime/streaming/operators/watermark/mod.rs rename to src/streaming_runtime_operators/src/operators/watermark/mod.rs diff --git a/src/runtime/streaming/operators/watermark/watermark_generator.rs b/src/streaming_runtime_operators/src/operators/watermark/watermark_generator.rs similarity index 76% rename from src/runtime/streaming/operators/watermark/watermark_generator.rs rename to src/streaming_runtime_operators/src/operators/watermark/watermark_generator.rs index 497553eb..3120d524 100644 --- a/src/runtime/streaming/operators/watermark/watermark_generator.rs +++ b/src/streaming_runtime_operators/src/operators/watermark/watermark_generator.rs @@ -13,8 +13,12 @@ use anyhow::{Result, anyhow}; use arrow::compute::kernels::aggregate; use arrow_array::cast::AsArray; -use arrow_array::types::TimestampNanosecondType; -use arrow_array::{RecordBatch, TimestampNanosecondArray}; +use arrow_array::types::{ + TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, + TimestampSecondType, +}; +use arrow_array::{Array, RecordBatch}; +use arrow_schema::{DataType, TimeUnit}; use bincode::{Decode, Encode}; use datafusion::physical_expr::PhysicalExpr; use datafusion_proto::physical_plan::DefaultPhysicalExtensionCodec; @@ -25,10 +29,10 @@ use std::sync::Arc; use std::time::{Duration, SystemTime}; use tracing::debug; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::factory::Registry; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::factory::Registry; use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark, from_nanos, to_millis}; use async_trait::async_trait; use protocol::function_stream_graph::ExpressionWatermarkConfig; @@ -77,10 +81,7 @@ impl WatermarkGeneratorOperator { } fn extract_max_timestamp(&self, batch: &RecordBatch) -> Option { - let ts_column = batch.column(self.timestamp_index); - let arr = ts_column.as_primitive::(); - let max_ts = aggregate::max(arr)?; - Some(from_nanos(max_ts as u128)) + max_timestamp_from_array(batch.column(self.timestamp_index).as_ref()).map(from_nanos) } fn evaluate_watermark(&self, batch: &RecordBatch) -> Result { @@ -89,15 +90,40 @@ impl WatermarkGeneratorOperator { .evaluate(batch)? .into_array(batch.num_rows())?; - let typed_array = watermark_array - .as_any() - .downcast_ref::() - .ok_or_else(|| anyhow!("watermark expression must return TimestampNanosecondArray"))?; + let max_nanos = max_timestamp_from_array(watermark_array.as_ref()).ok_or_else(|| { + anyhow!( + "watermark expression must return a timestamp array, got {:?}", + watermark_array.data_type() + ) + })?; - let max_watermark_nanos = aggregate::max(typed_array) - .ok_or_else(|| anyhow!("failed to extract max watermark from batch"))?; + Ok(from_nanos(max_nanos)) + } +} - Ok(from_nanos(max_watermark_nanos as u128)) +/// Returns event time as nanoseconds since epoch (internal watermark representation). +fn max_timestamp_from_array(array: &dyn Array) -> Option { + match array.data_type() { + DataType::Timestamp(TimeUnit::Nanosecond, _) => { + let arr = array.as_primitive::(); + aggregate::max(arr).map(|v| v as u128) + } + DataType::Timestamp(TimeUnit::Microsecond, _) => { + let arr = array.as_primitive::(); + aggregate::max(arr).map(|v| (v as u128) * 1_000) + } + DataType::Timestamp(TimeUnit::Millisecond, _) => { + let arr = array.as_primitive::(); + aggregate::max(arr).map(|v| (v as u128) * 1_000_000) + } + DataType::Timestamp(TimeUnit::Second, _) => { + let arr = array.as_primitive::(); + aggregate::max(arr).map(|v| (v as u128) * 1_000_000_000) + } + other => { + debug!(?other, "skip max timestamp: not a timestamp array"); + None + } } } diff --git a/src/runtime/streaming/operators/windows/mod.rs b/src/streaming_runtime_operators/src/operators/windows/mod.rs similarity index 100% rename from src/runtime/streaming/operators/windows/mod.rs rename to src/streaming_runtime_operators/src/operators/windows/mod.rs diff --git a/src/runtime/streaming/operators/windows/session_aggregating_window.rs b/src/streaming_runtime_operators/src/operators/windows/session_aggregating_window.rs similarity index 99% rename from src/runtime/streaming/operators/windows/session_aggregating_window.rs rename to src/streaming_runtime_operators/src/operators/windows/session_aggregating_window.rs index 2056cdd9..9b32107c 100644 --- a/src/runtime/streaming/operators/windows/session_aggregating_window.rs +++ b/src/streaming_runtime_operators/src/operators/windows/session_aggregating_window.rs @@ -36,11 +36,11 @@ use std::time::{Duration, SystemTime}; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; use tracing::info; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::factory::Registry; -use crate::runtime::streaming::state::OperatorStateStore; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::core::state::OperatorStateStore; +use crate::factory::Registry; use crate::sql::common::converter::Converter; use crate::sql::common::{ CheckpointBarrier, FsSchema, FsSchemaRef, Watermark, from_nanos, to_nanos, @@ -711,7 +711,6 @@ impl SessionWindowOperator { .context("failed to create session window output batch") } - #[allow(dead_code)] fn earliest_batch_time(&self) -> Option { self.pq_start_times .first_key_value() diff --git a/src/runtime/streaming/operators/windows/sliding_aggregating_window.rs b/src/streaming_runtime_operators/src/operators/windows/sliding_aggregating_window.rs similarity index 98% rename from src/runtime/streaming/operators/windows/sliding_aggregating_window.rs rename to src/streaming_runtime_operators/src/operators/windows/sliding_aggregating_window.rs index f18b3b14..475554a0 100644 --- a/src/runtime/streaming/operators/windows/sliding_aggregating_window.rs +++ b/src/streaming_runtime_operators/src/operators/windows/sliding_aggregating_window.rs @@ -33,11 +33,11 @@ use std::time::{Duration, SystemTime}; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; use tracing::info; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::factory::Registry; -use crate::runtime::streaming::state::OperatorStateStore; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::core::state::OperatorStateStore; +use crate::factory::Registry; use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark, from_nanos, to_nanos}; use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; use async_trait::async_trait; diff --git a/src/runtime/streaming/operators/windows/tumbling_aggregating_window.rs b/src/streaming_runtime_operators/src/operators/windows/tumbling_aggregating_window.rs similarity index 98% rename from src/runtime/streaming/operators/windows/tumbling_aggregating_window.rs rename to src/streaming_runtime_operators/src/operators/windows/tumbling_aggregating_window.rs index 5c805625..ece41718 100644 --- a/src/runtime/streaming/operators/windows/tumbling_aggregating_window.rs +++ b/src/streaming_runtime_operators/src/operators/windows/tumbling_aggregating_window.rs @@ -34,11 +34,11 @@ use std::time::{Duration, SystemTime}; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; use tracing::{info, warn}; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::factory::Registry; -use crate::runtime::streaming::state::OperatorStateStore; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::core::state::OperatorStateStore; +use crate::factory::Registry; use crate::sql::common::time_utils::print_time; use crate::sql::common::{CheckpointBarrier, FsSchema, Watermark, from_nanos, to_nanos}; use crate::sql::physical::{StreamingDecodingContext, StreamingExtensionCodec}; diff --git a/src/runtime/streaming/operators/windows/window_function.rs b/src/streaming_runtime_operators/src/operators/windows/window_function.rs similarity index 97% rename from src/runtime/streaming/operators/windows/window_function.rs rename to src/streaming_runtime_operators/src/operators/windows/window_function.rs index 37815b78..30bf601a 100644 --- a/src/runtime/streaming/operators/windows/window_function.rs +++ b/src/streaming_runtime_operators/src/operators/windows/window_function.rs @@ -26,11 +26,11 @@ use std::time::SystemTime; use tokio::sync::mpsc::{UnboundedReceiver, unbounded_channel}; use tracing::{info, warn}; -use crate::runtime::streaming::StreamOutput; -use crate::runtime::streaming::api::context::TaskContext; -use crate::runtime::streaming::api::operator::{Collector, Operator}; -use crate::runtime::streaming::factory::Registry; -use crate::runtime::streaming::state::OperatorStateStore; +use crate::core::StreamOutput; +use crate::core::api::context::TaskContext; +use crate::core::api::operator::{Collector, Operator}; +use crate::core::state::OperatorStateStore; +use crate::factory::Registry; use crate::sql::common::time_utils::print_time; use crate::sql::common::{ CheckpointBarrier, FsSchema, FsSchemaRef, Watermark, from_nanos, to_nanos, diff --git a/src/runtime/util/mod.rs b/src/streaming_runtime_operators/src/util/mod.rs similarity index 100% rename from src/runtime/util/mod.rs rename to src/streaming_runtime_operators/src/util/mod.rs diff --git a/src/runtime/util/physical_aggregate.rs b/src/streaming_runtime_operators/src/util/physical_aggregate.rs similarity index 100% rename from src/runtime/util/physical_aggregate.rs rename to src/streaming_runtime_operators/src/util/physical_aggregate.rs diff --git a/src/wasm_runtime/Cargo.toml b/src/wasm_runtime/Cargo.toml new file mode 100644 index 00000000..2f385240 --- /dev/null +++ b/src/wasm_runtime/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "function-stream-wasm-runtime" +version.workspace = true +edition.workspace = true + +[lib] +name = "function_stream_wasm_runtime" +path = "src/lib.rs" diff --git a/src/wasm_runtime/src/lib.rs b/src/wasm_runtime/src/lib.rs new file mode 100644 index 00000000..12b3ebee --- /dev/null +++ b/src/wasm_runtime/src/lib.rs @@ -0,0 +1,22 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! WebAssembly execution runtime. +//! +//! Implementation lives under `src/wasm/` in this package. It is currently **compiled as +//! part of the `function-stream` crate** via `#[path]` in `src/lib.rs` / `src/main.rs`, so paths +//! like `crate::sql` (streaming planner dependency) and `crate::memory` keep resolving. +//! +//! Operator state storage (`state_backend/`) also lives in this package and is compiled via +//! `#[path]` from the root crate as `crate::state_backend`. + +pub const CRATE_NAME: &str = "function-stream-wasm-runtime"; diff --git a/src/storage/state_backend/error.rs b/src/wasm_runtime/src/state_backend/error.rs similarity index 100% rename from src/storage/state_backend/error.rs rename to src/wasm_runtime/src/state_backend/error.rs diff --git a/src/storage/state_backend/factory.rs b/src/wasm_runtime/src/state_backend/factory.rs similarity index 82% rename from src/storage/state_backend/factory.rs rename to src/wasm_runtime/src/state_backend/factory.rs index 5dc38632..2bc0b55c 100644 --- a/src/storage/state_backend/factory.rs +++ b/src/wasm_runtime/src/state_backend/factory.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage::state_backend::error::BackendError; -use crate::storage::state_backend::store::StateStore; +use crate::state_backend::error::BackendError; +use crate::state_backend::store::StateStore; use std::path::Path; use std::sync::Arc; @@ -47,11 +47,11 @@ pub fn get_factory_for_task>( task_name: String, created_at: u64, base_dir: Option

, - rocksdb_config: Option, + rocksdb_config: Option, ) -> Result, BackendError> { match factory_type { FactoryType::Memory => { - Ok(crate::storage::state_backend::memory::MemoryStateStoreFactory::default_factory()) + Ok(crate::state_backend::memory::MemoryStateStoreFactory::default_factory()) } FactoryType::RocksDB => { let base_dir = base_dir.ok_or_else(|| { @@ -63,9 +63,8 @@ pub fn get_factory_for_task>( .join(format!("{}-{}", task_name, created_at)); let config = rocksdb_config.unwrap_or_default(); - let factory = crate::storage::state_backend::rocksdb::RocksDBStateStoreFactory::new( - db_path, config, - )?; + let factory = + crate::state_backend::rocksdb::RocksDBStateStoreFactory::new(db_path, config)?; Ok(Arc::new(factory)) } diff --git a/src/storage/state_backend/key_builder.rs b/src/wasm_runtime/src/state_backend/key_builder.rs similarity index 100% rename from src/storage/state_backend/key_builder.rs rename to src/wasm_runtime/src/state_backend/key_builder.rs diff --git a/src/storage/state_backend/memory/factory.rs b/src/wasm_runtime/src/state_backend/memory/factory.rs similarity index 86% rename from src/storage/state_backend/memory/factory.rs rename to src/wasm_runtime/src/state_backend/memory/factory.rs index b62bd444..0526ecbd 100644 --- a/src/storage/state_backend/memory/factory.rs +++ b/src/wasm_runtime/src/state_backend/memory/factory.rs @@ -11,8 +11,8 @@ // limitations under the License. use super::store::MemoryStateStore; -use crate::storage::state_backend::error::BackendError; -use crate::storage::state_backend::factory::StateStoreFactory; +use crate::state_backend::error::BackendError; +use crate::state_backend::factory::StateStoreFactory; use std::sync::{Arc, Mutex}; pub struct MemoryStateStoreFactory {} @@ -43,7 +43,7 @@ impl StateStoreFactory for MemoryStateStoreFactory { fn new_state_store( &self, _column_family: Option, - ) -> Result, BackendError> { + ) -> Result, BackendError> { Ok(Box::new(MemoryStateStore::new())) } } diff --git a/src/storage/state_backend/memory/mod.rs b/src/wasm_runtime/src/state_backend/memory/mod.rs similarity index 100% rename from src/storage/state_backend/memory/mod.rs rename to src/wasm_runtime/src/state_backend/memory/mod.rs diff --git a/src/storage/state_backend/memory/store.rs b/src/wasm_runtime/src/state_backend/memory/store.rs similarity index 95% rename from src/storage/state_backend/memory/store.rs rename to src/wasm_runtime/src/state_backend/memory/store.rs index e65e839d..5d07ad99 100644 --- a/src/storage/state_backend/memory/store.rs +++ b/src/wasm_runtime/src/state_backend/memory/store.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage::state_backend::error::BackendError; -use crate::storage::state_backend::store::{StateIterator, StateStore}; +use crate::state_backend::error::BackendError; +use crate::state_backend::store::{StateIterator, StateStore}; use std::collections::HashMap; use std::sync::{Arc, Mutex}; @@ -94,9 +94,8 @@ impl StateStore for MemoryStateStore { user_key: Vec, value: Vec, ) -> Result<(), BackendError> { - let key_bytes = crate::storage::state_backend::key_builder::build_key( - &key_group, &key, &namespace, &user_key, - ); + let key_bytes = + crate::state_backend::key_builder::build_key(&key_group, &key, &namespace, &user_key); let existing = self.get_state(key_bytes.clone())?; diff --git a/src/storage/state_backend/mod.rs b/src/wasm_runtime/src/state_backend/mod.rs similarity index 100% rename from src/storage/state_backend/mod.rs rename to src/wasm_runtime/src/state_backend/mod.rs diff --git a/src/storage/state_backend/rocksdb/factory.rs b/src/wasm_runtime/src/state_backend/rocksdb/factory.rs similarity index 94% rename from src/storage/state_backend/rocksdb/factory.rs rename to src/wasm_runtime/src/state_backend/rocksdb/factory.rs index 11554e13..74759b91 100644 --- a/src/storage/state_backend/rocksdb/factory.rs +++ b/src/wasm_runtime/src/state_backend/rocksdb/factory.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage::state_backend::error::BackendError; -use crate::storage::state_backend::factory::StateStoreFactory; +use crate::state_backend::error::BackendError; +use crate::state_backend::factory::StateStoreFactory; use rocksdb::{ColumnFamilyDescriptor, DB, Options}; use std::path::Path; use std::sync::{Arc, Mutex}; @@ -56,7 +56,7 @@ impl StateStoreFactory for RocksDBStateStoreFactory { fn new_state_store( &self, column_family: Option, - ) -> Result, BackendError> { + ) -> Result, BackendError> { self.new_state_store(column_family) } } @@ -137,7 +137,7 @@ impl RocksDBStateStoreFactory { pub fn new_state_store( &self, column_family: Option, - ) -> Result, BackendError> { + ) -> Result, BackendError> { if let Some(ref cf_name) = column_family && cf_name != "default" && self.db.cf_handle(cf_name).is_none() @@ -158,7 +158,7 @@ impl RocksDBStateStoreFactory { } } - crate::storage::state_backend::rocksdb::store::RocksDBStateStore::new_with_factory( + crate::state_backend::rocksdb::store::RocksDBStateStore::new_with_factory( self.db.clone(), column_family, ) diff --git a/src/storage/state_backend/rocksdb/mod.rs b/src/wasm_runtime/src/state_backend/rocksdb/mod.rs similarity index 100% rename from src/storage/state_backend/rocksdb/mod.rs rename to src/wasm_runtime/src/state_backend/rocksdb/mod.rs diff --git a/src/storage/state_backend/rocksdb/store.rs b/src/wasm_runtime/src/state_backend/rocksdb/store.rs similarity index 97% rename from src/storage/state_backend/rocksdb/store.rs rename to src/wasm_runtime/src/state_backend/rocksdb/store.rs index e8a2ad13..b7d98e4c 100644 --- a/src/storage/state_backend/rocksdb/store.rs +++ b/src/wasm_runtime/src/state_backend/rocksdb/store.rs @@ -10,9 +10,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage::state_backend::error::BackendError; -use crate::storage::state_backend::key_builder::{build_key, increment_key, is_all_0xff}; -use crate::storage::state_backend::store::{StateIterator, StateStore}; +use crate::state_backend::error::BackendError; +use crate::state_backend::key_builder::{build_key, increment_key, is_all_0xff}; +use crate::state_backend::store::{StateIterator, StateStore}; use rocksdb::{ BlockBasedOptions, Cache, ColumnFamilyDescriptor, DB, DBCompressionType, Direction, IteratorMode, Options, ReadOptions, WriteBatch, WriteOptions, diff --git a/src/storage/state_backend/server.rs b/src/wasm_runtime/src/state_backend/server.rs similarity index 92% rename from src/storage/state_backend/server.rs rename to src/wasm_runtime/src/state_backend/server.rs index efeceaa4..1fb24c38 100644 --- a/src/storage/state_backend/server.rs +++ b/src/wasm_runtime/src/state_backend/server.rs @@ -12,11 +12,9 @@ use crate::config::storage::{StateStorageConfig, StateStorageType}; use crate::config::{get_state_dir, get_state_dir_for_base}; -use crate::storage::state_backend::error::BackendError; -use crate::storage::state_backend::factory::{ - FactoryType, StateStoreFactory, get_factory_for_task, -}; -use crate::storage::state_backend::rocksdb::RocksDBConfig; +use crate::state_backend::error::BackendError; +use crate::state_backend::factory::{FactoryType, StateStoreFactory, get_factory_for_task}; +use crate::state_backend::rocksdb::RocksDBConfig; use std::fs; use std::path::PathBuf; use std::sync::Arc; diff --git a/src/storage/state_backend/store.rs b/src/wasm_runtime/src/state_backend/store.rs similarity index 88% rename from src/storage/state_backend/store.rs rename to src/wasm_runtime/src/state_backend/store.rs index aaa02988..cb897f6b 100644 --- a/src/storage/state_backend/store.rs +++ b/src/wasm_runtime/src/state_backend/store.rs @@ -10,7 +10,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage::state_backend::error::BackendError; +use crate::state_backend::error::BackendError; pub type StateIteratorItem = Result, Vec)>, BackendError>; @@ -107,9 +107,8 @@ pub trait StateStore: Send + Sync { user_key: Vec, value: Vec, ) -> Result<(), BackendError> { - let key_bytes = crate::storage::state_backend::key_builder::build_key( - &key_group, &key, &namespace, &user_key, - ); + let key_bytes = + crate::state_backend::key_builder::build_key(&key_group, &key, &namespace, &user_key); self.put_state(key_bytes, value) } @@ -132,9 +131,8 @@ pub trait StateStore: Send + Sync { namespace: Vec, user_key: Vec, ) -> Result>, BackendError> { - let key_bytes = crate::storage::state_backend::key_builder::build_key( - &key_group, &key, &namespace, &user_key, - ); + let key_bytes = + crate::state_backend::key_builder::build_key(&key_group, &key, &namespace, &user_key); self.get_state(key_bytes) } @@ -156,9 +154,8 @@ pub trait StateStore: Send + Sync { namespace: Vec, user_key: Vec, ) -> Result<(), BackendError> { - let key_bytes = crate::storage::state_backend::key_builder::build_key( - &key_group, &key, &namespace, &user_key, - ); + let key_bytes = + crate::state_backend::key_builder::build_key(&key_group, &key, &namespace, &user_key); self.delete_state(key_bytes) } @@ -199,12 +196,8 @@ pub trait StateStore: Send + Sync { key: Vec, namespace: Vec, ) -> Result { - let prefix_bytes = crate::storage::state_backend::key_builder::build_key( - &key_group, - &key, - &namespace, - &[], - ); + let prefix_bytes = + crate::state_backend::key_builder::build_key(&key_group, &key, &namespace, &[]); self.delete_prefix_bytes(prefix_bytes) } @@ -228,13 +221,13 @@ pub trait StateStore: Send + Sync { start_inclusive: Vec, end_exclusive: Vec, ) -> Result>, BackendError> { - let start_key = crate::storage::state_backend::key_builder::build_key( + let start_key = crate::state_backend::key_builder::build_key( &key_group, &key, &namespace, &start_inclusive, ); - let end_key = crate::storage::state_backend::key_builder::build_key( + let end_key = crate::state_backend::key_builder::build_key( &key_group, &key, &namespace, @@ -279,12 +272,8 @@ pub trait StateStore: Send + Sync { key: Vec, namespace: Vec, ) -> Result, BackendError> { - let prefix = crate::storage::state_backend::key_builder::build_key( - &key_group, - &key, - &namespace, - &[], - ); + let prefix = + crate::state_backend::key_builder::build_key(&key_group, &key, &namespace, &[]); self.scan(prefix) } } diff --git a/src/runtime/wasm/buffer_and_event/buffer_or_event.rs b/src/wasm_runtime/src/wasm/buffer_and_event/buffer_or_event.rs similarity index 100% rename from src/runtime/wasm/buffer_and_event/buffer_or_event.rs rename to src/wasm_runtime/src/wasm/buffer_and_event/buffer_or_event.rs diff --git a/src/runtime/wasm/buffer_and_event/mod.rs b/src/wasm_runtime/src/wasm/buffer_and_event/mod.rs similarity index 100% rename from src/runtime/wasm/buffer_and_event/mod.rs rename to src/wasm_runtime/src/wasm/buffer_and_event/mod.rs diff --git a/src/runtime/wasm/buffer_and_event/stream_element/mod.rs b/src/wasm_runtime/src/wasm/buffer_and_event/stream_element/mod.rs similarity index 100% rename from src/runtime/wasm/buffer_and_event/stream_element/mod.rs rename to src/wasm_runtime/src/wasm/buffer_and_event/stream_element/mod.rs diff --git a/src/runtime/wasm/buffer_and_event/stream_element/stream_element.rs b/src/wasm_runtime/src/wasm/buffer_and_event/stream_element/stream_element.rs similarity index 100% rename from src/runtime/wasm/buffer_and_event/stream_element/stream_element.rs rename to src/wasm_runtime/src/wasm/buffer_and_event/stream_element/stream_element.rs diff --git a/src/runtime/wasm/input/input_protocol.rs b/src/wasm_runtime/src/wasm/input/input_protocol.rs similarity index 95% rename from src/runtime/wasm/input/input_protocol.rs rename to src/wasm_runtime/src/wasm/input/input_protocol.rs index 50294201..782a7f4f 100644 --- a/src/runtime/wasm/input/input_protocol.rs +++ b/src/wasm_runtime/src/wasm/input/input_protocol.rs @@ -10,7 +10,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::wasm::buffer_and_event::BufferOrEvent; +use crate::wasm::buffer_and_event::BufferOrEvent; use std::time::Duration; pub trait InputProtocol: Send + Sync + 'static { diff --git a/src/runtime/wasm/input/input_provider.rs b/src/wasm_runtime/src/wasm/input/input_provider.rs similarity index 94% rename from src/runtime/wasm/input/input_provider.rs rename to src/wasm_runtime/src/wasm/input/input_provider.rs index 8eee649d..d7ed64b9 100644 --- a/src/runtime/wasm/input/input_provider.rs +++ b/src/wasm_runtime/src/wasm/input/input_provider.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::input::Input; -use crate::runtime::wasm::task::InputConfig; +use crate::input::Input; +use crate::wasm::task::InputConfig; pub struct InputProvider; @@ -66,8 +66,8 @@ impl InputProvider { extra, runtime: _, } => { - use crate::runtime::input::InputRunner; - use crate::runtime::input::protocol::kafka::{KafkaConfig, KafkaProtocol}; + use crate::input::InputRunner; + use crate::input::protocol::kafka::{KafkaConfig, KafkaProtocol}; let servers: Vec = bootstrap_servers .split(',') diff --git a/src/runtime/wasm/input/input_runner.rs b/src/wasm_runtime/src/wasm/input/input_runner.rs similarity index 96% rename from src/runtime/wasm/input/input_runner.rs rename to src/wasm_runtime/src/wasm/input/input_runner.rs index ece85e3d..24b96dae 100644 --- a/src/runtime/wasm/input/input_runner.rs +++ b/src/wasm_runtime/src/wasm/input/input_runner.rs @@ -10,13 +10,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::common::TaskCompletionFlag; -use crate::runtime::input::input_protocol::InputProtocol; -use crate::runtime::input::{Input, InputState}; -use crate::runtime::processor::function_error::FunctionErrorReport; -use crate::runtime::wasm::buffer_and_event::BufferOrEvent; -use crate::runtime::wasm::task::ControlMailBox; -use crate::runtime::wasm::task::InputRuntimeConfig; +use crate::common::TaskCompletionFlag; +use crate::input::input_protocol::InputProtocol; +use crate::input::{Input, InputState}; +use crate::processor::function_error::FunctionErrorReport; +use crate::wasm::buffer_and_event::BufferOrEvent; +use crate::wasm::task::ControlMailBox; +use crate::wasm::task::InputRuntimeConfig; use crossbeam_channel::{Receiver, Sender, bounded, unbounded}; use std::sync::{Arc, Mutex}; use std::thread; @@ -250,7 +250,7 @@ impl InputRunner

{ impl Input for InputRunner

{ fn init_with_context( &mut self, - init_context: &crate::runtime::wasm::taskexecutor::InitContext, + init_context: &crate::wasm::taskexecutor::InitContext, ) -> Result<(), Box> { if !matches!(*self.state.lock().unwrap(), InputState::Uninitialized) { return Ok(()); @@ -298,7 +298,7 @@ impl Input for InputRunner

{ }) .map_err(|e| Box::new(std::io::Error::other(e)) as Box)?; - use crate::runtime::processor::wasm::thread_pool::{ThreadGroup, ThreadGroupType}; + use crate::processor::wasm::thread_pool::{ThreadGroup, ThreadGroupType}; let mut group = ThreadGroup::new( ThreadGroupType::Input(self.group_id), format!("Input-g{}", self.group_id), diff --git a/src/runtime/wasm/input/interface.rs b/src/wasm_runtime/src/wasm/input/interface.rs similarity index 89% rename from src/runtime/wasm/input/interface.rs rename to src/wasm_runtime/src/wasm/input/interface.rs index 06da4923..cbfc61cc 100644 --- a/src/runtime/wasm/input/interface.rs +++ b/src/wasm_runtime/src/wasm/input/interface.rs @@ -10,10 +10,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::wasm::buffer_and_event::BufferOrEvent; -use crate::runtime::wasm::taskexecutor::InitContext; +use crate::wasm::buffer_and_event::BufferOrEvent; +use crate::wasm::taskexecutor::InitContext; -pub use crate::runtime::common::ComponentState as InputState; +pub use crate::common::ComponentState as InputState; pub trait Input: Send + Sync { fn init_with_context( diff --git a/src/runtime/wasm/input/mod.rs b/src/wasm_runtime/src/wasm/input/mod.rs similarity index 100% rename from src/runtime/wasm/input/mod.rs rename to src/wasm_runtime/src/wasm/input/mod.rs diff --git a/src/runtime/wasm/input/protocol/kafka/config.rs b/src/wasm_runtime/src/wasm/input/protocol/kafka/config.rs similarity index 100% rename from src/runtime/wasm/input/protocol/kafka/config.rs rename to src/wasm_runtime/src/wasm/input/protocol/kafka/config.rs diff --git a/src/runtime/wasm/input/protocol/kafka/kafka_protocol.rs b/src/wasm_runtime/src/wasm/input/protocol/kafka/kafka_protocol.rs similarity index 97% rename from src/runtime/wasm/input/protocol/kafka/kafka_protocol.rs rename to src/wasm_runtime/src/wasm/input/protocol/kafka/kafka_protocol.rs index 1fb487a6..928c9050 100644 --- a/src/runtime/wasm/input/protocol/kafka/kafka_protocol.rs +++ b/src/wasm_runtime/src/wasm/input/protocol/kafka/kafka_protocol.rs @@ -11,8 +11,8 @@ // limitations under the License. use super::config::KafkaConfig; -use crate::runtime::input::input_protocol::InputProtocol; -use crate::runtime::wasm::buffer_and_event::BufferOrEvent; +use crate::input::input_protocol::InputProtocol; +use crate::wasm::buffer_and_event::BufferOrEvent; use rdkafka::Message; use rdkafka::TopicPartitionList; use rdkafka::config::ClientConfig; diff --git a/src/runtime/wasm/input/protocol/kafka/mod.rs b/src/wasm_runtime/src/wasm/input/protocol/kafka/mod.rs similarity index 100% rename from src/runtime/wasm/input/protocol/kafka/mod.rs rename to src/wasm_runtime/src/wasm/input/protocol/kafka/mod.rs diff --git a/src/runtime/streaming/operators/source/mod.rs b/src/wasm_runtime/src/wasm/input/protocol/mod.rs similarity index 100% rename from src/runtime/streaming/operators/source/mod.rs rename to src/wasm_runtime/src/wasm/input/protocol/mod.rs diff --git a/src/runtime/wasm/mod.rs b/src/wasm_runtime/src/wasm/mod.rs similarity index 100% rename from src/runtime/wasm/mod.rs rename to src/wasm_runtime/src/wasm/mod.rs diff --git a/src/runtime/wasm/output/interface.rs b/src/wasm_runtime/src/wasm/output/interface.rs similarity index 93% rename from src/runtime/wasm/output/interface.rs rename to src/wasm_runtime/src/wasm/output/interface.rs index 21c3055d..e38217a1 100644 --- a/src/runtime/wasm/output/interface.rs +++ b/src/wasm_runtime/src/wasm/output/interface.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::wasm::buffer_and_event::BufferOrEvent; -use crate::runtime::wasm::taskexecutor::InitContext; +use crate::wasm::buffer_and_event::BufferOrEvent; +use crate::wasm::taskexecutor::InitContext; pub trait Output: Send + Sync { fn init_with_context( diff --git a/src/runtime/wasm/output/mod.rs b/src/wasm_runtime/src/wasm/output/mod.rs similarity index 100% rename from src/runtime/wasm/output/mod.rs rename to src/wasm_runtime/src/wasm/output/mod.rs diff --git a/src/runtime/wasm/output/output_protocol.rs b/src/wasm_runtime/src/wasm/output/output_protocol.rs similarity index 95% rename from src/runtime/wasm/output/output_protocol.rs rename to src/wasm_runtime/src/wasm/output/output_protocol.rs index 6140d3eb..8efc5f7d 100644 --- a/src/runtime/wasm/output/output_protocol.rs +++ b/src/wasm_runtime/src/wasm/output/output_protocol.rs @@ -10,7 +10,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::wasm::buffer_and_event::BufferOrEvent; +use crate::wasm::buffer_and_event::BufferOrEvent; pub trait OutputProtocol: Send + Sync + 'static { fn name(&self) -> String; diff --git a/src/runtime/wasm/output/output_provider.rs b/src/wasm_runtime/src/wasm/output/output_provider.rs similarity index 91% rename from src/runtime/wasm/output/output_provider.rs rename to src/wasm_runtime/src/wasm/output/output_provider.rs index 25ca8431..33b470d1 100644 --- a/src/runtime/wasm/output/output_provider.rs +++ b/src/wasm_runtime/src/wasm/output/output_provider.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::output::Output; -use crate::runtime::wasm::task::OutputConfig; +use crate::output::Output; +use crate::wasm::task::OutputConfig; pub struct OutputProvider; @@ -59,10 +59,8 @@ impl OutputProvider { extra, runtime: _, } => { - use crate::runtime::output::output_runner::OutputRunner; - use crate::runtime::output::protocol::kafka::{ - KafkaOutputProtocol, KafkaProducerConfig, - }; + use crate::output::output_runner::OutputRunner; + use crate::output::protocol::kafka::{KafkaOutputProtocol, KafkaProducerConfig}; let servers: Vec = bootstrap_servers .split(',') diff --git a/src/runtime/wasm/output/output_runner.rs b/src/wasm_runtime/src/wasm/output/output_runner.rs similarity index 96% rename from src/runtime/wasm/output/output_runner.rs rename to src/wasm_runtime/src/wasm/output/output_runner.rs index ca6d780c..b732693a 100644 --- a/src/runtime/wasm/output/output_runner.rs +++ b/src/wasm_runtime/src/wasm/output/output_runner.rs @@ -10,13 +10,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::common::{ComponentState, TaskCompletionFlag}; -use crate::runtime::output::Output; -use crate::runtime::output::output_protocol::OutputProtocol; -use crate::runtime::processor::function_error::FunctionErrorReport; -use crate::runtime::wasm::buffer_and_event::BufferOrEvent; -use crate::runtime::wasm::task::ControlMailBox; -use crate::runtime::wasm::task::OutputRuntimeConfig; +use crate::common::{ComponentState, TaskCompletionFlag}; +use crate::output::Output; +use crate::output::output_protocol::OutputProtocol; +use crate::processor::function_error::FunctionErrorReport; +use crate::wasm::buffer_and_event::BufferOrEvent; +use crate::wasm::task::ControlMailBox; +use crate::wasm::task::OutputRuntimeConfig; use crossbeam_channel::{Receiver, Sender, bounded, unbounded}; use std::sync::{Arc, Mutex}; use std::thread; @@ -288,7 +288,7 @@ impl OutputRunner

{ impl Output for OutputRunner

{ fn init_with_context( &mut self, - ctx: &crate::runtime::wasm::taskexecutor::InitContext, + ctx: &crate::wasm::taskexecutor::InitContext, ) -> Result<(), Box> { if !matches!(*self.state.lock().unwrap(), ComponentState::Uninitialized) { return Ok(()); @@ -333,7 +333,7 @@ impl Output for OutputRunner

{ }) .map_err(|e| Box::new(std::io::Error::other(e)) as Box)?; - use crate::runtime::processor::wasm::thread_pool::{ThreadGroup, ThreadGroupType}; + use crate::processor::wasm::thread_pool::{ThreadGroup, ThreadGroupType}; let mut group = ThreadGroup::new( ThreadGroupType::Output(self.output_id), format!("Output-{}", self.output_id), diff --git a/src/runtime/wasm/output/protocol/kafka/kafka_protocol.rs b/src/wasm_runtime/src/wasm/output/protocol/kafka/kafka_protocol.rs similarity index 96% rename from src/runtime/wasm/output/protocol/kafka/kafka_protocol.rs rename to src/wasm_runtime/src/wasm/output/protocol/kafka/kafka_protocol.rs index d9e6db4d..787804a2 100644 --- a/src/runtime/wasm/output/protocol/kafka/kafka_protocol.rs +++ b/src/wasm_runtime/src/wasm/output/protocol/kafka/kafka_protocol.rs @@ -11,8 +11,8 @@ // limitations under the License. use super::producer_config::KafkaProducerConfig; -use crate::runtime::output::output_protocol::OutputProtocol; -use crate::runtime::wasm::buffer_and_event::BufferOrEvent; +use crate::output::output_protocol::OutputProtocol; +use crate::wasm::buffer_and_event::BufferOrEvent; use rdkafka::producer::{BaseRecord, DefaultProducerContext, Producer, ThreadedProducer}; use std::sync::Mutex; use std::time::Duration; diff --git a/src/runtime/wasm/output/protocol/kafka/mod.rs b/src/wasm_runtime/src/wasm/output/protocol/kafka/mod.rs similarity index 100% rename from src/runtime/wasm/output/protocol/kafka/mod.rs rename to src/wasm_runtime/src/wasm/output/protocol/kafka/mod.rs diff --git a/src/runtime/wasm/output/protocol/kafka/producer_config.rs b/src/wasm_runtime/src/wasm/output/protocol/kafka/producer_config.rs similarity index 100% rename from src/runtime/wasm/output/protocol/kafka/producer_config.rs rename to src/wasm_runtime/src/wasm/output/protocol/kafka/producer_config.rs diff --git a/src/runtime/wasm/output/protocol/mod.rs b/src/wasm_runtime/src/wasm/output/protocol/mod.rs similarity index 100% rename from src/runtime/wasm/output/protocol/mod.rs rename to src/wasm_runtime/src/wasm/output/protocol/mod.rs diff --git a/src/runtime/wasm/processor/function_error.rs b/src/wasm_runtime/src/wasm/processor/function_error.rs similarity index 100% rename from src/runtime/wasm/processor/function_error.rs rename to src/wasm_runtime/src/wasm/processor/function_error.rs diff --git a/src/runtime/wasm/processor/mod.rs b/src/wasm_runtime/src/wasm/processor/mod.rs similarity index 100% rename from src/runtime/wasm/processor/mod.rs rename to src/wasm_runtime/src/wasm/processor/mod.rs diff --git a/src/runtime/wasm/processor/python/mod.rs b/src/wasm_runtime/src/wasm/processor/python/mod.rs similarity index 100% rename from src/runtime/wasm/processor/python/mod.rs rename to src/wasm_runtime/src/wasm/processor/python/mod.rs diff --git a/src/runtime/wasm/processor/python/python_host.rs b/src/wasm_runtime/src/wasm/processor/python/python_host.rs similarity index 100% rename from src/runtime/wasm/processor/python/python_host.rs rename to src/wasm_runtime/src/wasm/processor/python/python_host.rs diff --git a/src/runtime/wasm/processor/python/python_service.rs b/src/wasm_runtime/src/wasm/processor/python/python_service.rs similarity index 100% rename from src/runtime/wasm/processor/python/python_service.rs rename to src/wasm_runtime/src/wasm/processor/python/python_service.rs diff --git a/src/runtime/wasm/processor/wasm/input_strategy.rs b/src/wasm_runtime/src/wasm/processor/wasm/input_strategy.rs similarity index 100% rename from src/runtime/wasm/processor/wasm/input_strategy.rs rename to src/wasm_runtime/src/wasm/processor/wasm/input_strategy.rs diff --git a/src/runtime/wasm/processor/wasm/mod.rs b/src/wasm_runtime/src/wasm/processor/wasm/mod.rs similarity index 100% rename from src/runtime/wasm/processor/wasm/mod.rs rename to src/wasm_runtime/src/wasm/processor/wasm/mod.rs diff --git a/src/runtime/wasm/processor/wasm/thread_pool.rs b/src/wasm_runtime/src/wasm/processor/wasm/thread_pool.rs similarity index 99% rename from src/runtime/wasm/processor/wasm/thread_pool.rs rename to src/wasm_runtime/src/wasm/processor/wasm/thread_pool.rs index c78eef79..d625d86a 100644 --- a/src/runtime/wasm/processor/wasm/thread_pool.rs +++ b/src/wasm_runtime/src/wasm/processor/wasm/thread_pool.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::common::ComponentState; -use crate::runtime::processor::wasm::wasm_task::WasmTask; +use crate::common::ComponentState; +use crate::processor::wasm::wasm_task::WasmTask; use std::collections::HashMap; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; diff --git a/src/runtime/wasm/processor/wasm/wasm_cache.rs b/src/wasm_runtime/src/wasm/processor/wasm/wasm_cache.rs similarity index 100% rename from src/runtime/wasm/processor/wasm/wasm_cache.rs rename to src/wasm_runtime/src/wasm/processor/wasm/wasm_cache.rs diff --git a/src/runtime/wasm/processor/wasm/wasm_host.rs b/src/wasm_runtime/src/wasm/processor/wasm/wasm_host.rs similarity index 94% rename from src/runtime/wasm/processor/wasm/wasm_host.rs rename to src/wasm_runtime/src/wasm/processor/wasm/wasm_host.rs index 2bf7d4f0..348f3063 100644 --- a/src/runtime/wasm/processor/wasm/wasm_host.rs +++ b/src/wasm_runtime/src/wasm/processor/wasm/wasm_host.rs @@ -10,10 +10,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::output::Output; -use crate::runtime::processor::wasm::wasm_cache; -use crate::runtime::wasm::buffer_and_event::BufferOrEvent; -use crate::storage::state_backend::{StateStore, StateStoreFactory}; +use crate::output::Output; +use crate::processor::wasm::wasm_cache; +use crate::state_backend::{StateStore, StateStoreFactory}; +use crate::wasm::buffer_and_event::BufferOrEvent; use std::sync::{Arc, OnceLock}; use wasmtime::component::{Component, HasData, Linker, Resource, bindgen}; use wasmtime::{Config, Engine, Store}; @@ -67,7 +67,7 @@ fn get_global_engine(_wasm_size: usize) -> anyhow::Result> { bindgen!({ world: "processor", - path: "wit", + path: "../../wit", with: { "functionstream:core/kv.store": FunctionStreamStoreHandle, "functionstream:core/kv.iterator": FunctionStreamIteratorHandle, @@ -86,7 +86,7 @@ impl Drop for FunctionStreamStoreHandle { } pub struct FunctionStreamIteratorHandle { - pub state_iterator: Box, + pub state_iterator: Box, } pub struct HostState { @@ -205,7 +205,7 @@ impl HostStore for HostState { .get(&self_) .map_err(|e| Error::Other(format!("Failed to get store resource: {}", e)))?; - let real_key = crate::storage::state_backend::key_builder::build_key( + let real_key = crate::state_backend::key_builder::build_key( &key.key_group, &key.key, &key.namespace, @@ -228,7 +228,7 @@ impl HostStore for HostState { .get(&self_) .map_err(|e| Error::Other(format!("Failed to get store resource: {}", e)))?; - let real_key = crate::storage::state_backend::key_builder::build_key( + let real_key = crate::state_backend::key_builder::build_key( &key.key_group, &key.key, &key.namespace, @@ -251,7 +251,7 @@ impl HostStore for HostState { .get(&self_) .map_err(|e| Error::Other(format!("Failed to get store resource: {}", e)))?; - let real_key = crate::storage::state_backend::key_builder::build_key( + let real_key = crate::state_backend::key_builder::build_key( &key.key_group, &key.key, &key.namespace, @@ -284,7 +284,7 @@ impl HostStore for HostState { .get(&self_) .map_err(|e| Error::Other(format!("Failed to get store resource: {}", e)))?; - let prefix_key = crate::storage::state_backend::key_builder::build_key( + let prefix_key = crate::state_backend::key_builder::build_key( &key.key_group, &key.key, &key.namespace, @@ -323,13 +323,13 @@ impl HostStore for HostState { .get(&self_) .map_err(|e| Error::Other(format!("Failed to get store resource: {}", e)))?; - let start_key = crate::storage::state_backend::key_builder::build_key( + let start_key = crate::state_backend::key_builder::build_key( &key_group, &key, &namespace, &start_inclusive, ); - let end_key = crate::storage::state_backend::key_builder::build_key( + let end_key = crate::state_backend::key_builder::build_key( &key_group, &key, &namespace, @@ -449,7 +449,7 @@ pub fn create_wasm_host_with_component( engine: &Engine, component: &Component, outputs: Vec>, - init_context: &crate::runtime::wasm::taskexecutor::InitContext, + init_context: &crate::wasm::taskexecutor::InitContext, task_name: String, create_time: u64, ) -> anyhow::Result<(Processor, Store)> { @@ -495,7 +495,7 @@ pub fn create_wasm_host_with_component( pub fn create_wasm_host( wasm_bytes: &[u8], outputs: Vec>, - init_context: &crate::runtime::wasm::taskexecutor::InitContext, + init_context: &crate::wasm::taskexecutor::InitContext, task_name: String, create_time: u64, ) -> anyhow::Result<(Processor, Store)> { diff --git a/src/runtime/wasm/processor/wasm/wasm_processor.rs b/src/wasm_runtime/src/wasm/processor/wasm/wasm_processor.rs similarity index 99% rename from src/runtime/wasm/processor/wasm/wasm_processor.rs rename to src/wasm_runtime/src/wasm/processor/wasm/wasm_processor.rs index 52234bfe..741280bb 100644 --- a/src/runtime/wasm/processor/wasm/wasm_processor.rs +++ b/src/wasm_runtime/src/wasm/processor/wasm/wasm_processor.rs @@ -17,7 +17,7 @@ use super::wasm_host::{HostState, Processor}; use super::wasm_processor_trait::WasmProcessor; -use crate::runtime::output::Output; +use crate::output::Output; use std::cell::RefCell; use std::error::Error; use std::fmt; @@ -134,7 +134,7 @@ impl WasmProcessorImpl { impl WasmProcessor for WasmProcessorImpl { fn init_with_context( &mut self, - _init_context: &crate::runtime::wasm::taskexecutor::InitContext, + _init_context: &crate::wasm::taskexecutor::InitContext, ) -> Result<(), Box> { if self.initialized { log::warn!("WasmProcessor '{}' already initialized", self.name); @@ -405,7 +405,7 @@ impl WasmProcessor for WasmProcessorImpl { fn init_wasm_host( &mut self, outputs: Vec>, - init_context: &crate::runtime::wasm::taskexecutor::InitContext, + init_context: &crate::wasm::taskexecutor::InitContext, task_name: String, create_time: u64, ) -> Result<(), Box> { diff --git a/src/runtime/wasm/processor/wasm/wasm_processor_trait.rs b/src/wasm_runtime/src/wasm/processor/wasm/wasm_processor_trait.rs similarity index 97% rename from src/runtime/wasm/processor/wasm/wasm_processor_trait.rs rename to src/wasm_runtime/src/wasm/processor/wasm/wasm_processor_trait.rs index fb2c17fb..c699fe8b 100644 --- a/src/runtime/wasm/processor/wasm/wasm_processor_trait.rs +++ b/src/wasm_runtime/src/wasm/processor/wasm/wasm_processor_trait.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::output::Output; -use crate::runtime::wasm::taskexecutor::InitContext; +use crate::output::Output; +use crate::wasm::taskexecutor::InitContext; pub trait WasmProcessor: Send + Sync { fn process( diff --git a/src/runtime/wasm/processor/wasm/wasm_task.rs b/src/wasm_runtime/src/wasm/processor/wasm/wasm_task.rs similarity index 97% rename from src/runtime/wasm/processor/wasm/wasm_task.rs rename to src/wasm_runtime/src/wasm/processor/wasm/wasm_task.rs index 4330aaaf..ea0f5dc1 100644 --- a/src/runtime/wasm/processor/wasm/wasm_task.rs +++ b/src/wasm_runtime/src/wasm/processor/wasm/wasm_task.rs @@ -13,14 +13,14 @@ use super::input_strategy::{InputStrategy, RoundRobinStrategy, from_selector_name}; use super::thread_pool::ThreadGroup; use super::wasm_processor_trait::WasmProcessor; -use crate::runtime::common::{ComponentState, TaskCompletionFlag}; -use crate::runtime::input::Input; -use crate::runtime::output::Output; -use crate::runtime::processor::function_error::FunctionErrorReport; -use crate::runtime::wasm::buffer_and_event::BufferOrEvent; -use crate::runtime::wasm::task::ProcessorRuntimeConfig; -use crate::runtime::wasm::task::{ControlMailBox, TaskControlSignal, TaskLifecycle}; -use crate::storage::task::FunctionInfo; +use crate::common::{ComponentState, TaskCompletionFlag}; +use crate::input::Input; +use crate::output::Output; +use crate::processor::function_error::FunctionErrorReport; +use crate::task::FunctionInfo; +use crate::wasm::buffer_and_event::BufferOrEvent; +use crate::wasm::task::ProcessorRuntimeConfig; +use crate::wasm::task::{ControlMailBox, TaskControlSignal, TaskLifecycle}; use crossbeam_channel::{Receiver, after, select, unbounded}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc; @@ -120,7 +120,7 @@ impl WasmTask { pub fn init_with_context( &mut self, - init_context: &crate::runtime::wasm::taskexecutor::InitContext, + init_context: &crate::wasm::taskexecutor::InitContext, ) -> Result<(), Box> { let mut inputs = self.inputs.take().ok_or_else(|| { Box::new(std::io::Error::other("inputs already moved to thread")) @@ -235,7 +235,7 @@ impl WasmTask { ))) as Box })?; - use crate::runtime::processor::wasm::thread_pool::{ThreadGroup, ThreadGroupType}; + use crate::processor::wasm::thread_pool::{ThreadGroup, ThreadGroupType}; let mut main_runloop_group = ThreadGroup::new( ThreadGroupType::MainRunloop, format!("MainRunloop-{}", self.task_name), @@ -262,7 +262,7 @@ impl WasmTask { shared_state: Arc>, failure_cause: Arc>>, execution_state: Arc>, - _init_context: crate::runtime::wasm::taskexecutor::InitContext, + _init_context: crate::wasm::taskexecutor::InitContext, ) { let mut state = TaskState::Initialized; let mut last_idx: usize = 0; @@ -729,7 +729,7 @@ impl WasmTask { impl TaskLifecycle for WasmTask { fn init_with_context( &mut self, - init_context: &crate::runtime::wasm::taskexecutor::InitContext, + init_context: &crate::wasm::taskexecutor::InitContext, ) -> Result<(), Box> { ::init_with_context(self, init_context) } diff --git a/src/runtime/wasm/task/builder/mod.rs b/src/wasm_runtime/src/wasm/task/builder/mod.rs similarity index 100% rename from src/runtime/wasm/task/builder/mod.rs rename to src/wasm_runtime/src/wasm/task/builder/mod.rs diff --git a/src/runtime/wasm/task/builder/processor/mod.rs b/src/wasm_runtime/src/wasm/task/builder/processor/mod.rs similarity index 92% rename from src/runtime/wasm/task/builder/processor/mod.rs rename to src/wasm_runtime/src/wasm/task/builder/processor/mod.rs index c1306924..1ff39a7d 100644 --- a/src/runtime/wasm/task/builder/processor/mod.rs +++ b/src/wasm_runtime/src/wasm/task/builder/processor/mod.rs @@ -14,13 +14,13 @@ // // Specifically handles building logic for Processor type configuration -use crate::runtime::input::{Input, InputProvider}; -use crate::runtime::output::{Output, OutputProvider}; -use crate::runtime::processor::wasm::wasm_processor::WasmProcessorImpl; -use crate::runtime::processor::wasm::wasm_processor_trait::WasmProcessor; -use crate::runtime::processor::wasm::wasm_task::WasmTask; -use crate::runtime::wasm::task::yaml_keys::{TYPE, type_values}; -use crate::runtime::wasm::task::{InputConfig, OutputConfig, ProcessorConfig, WasmTaskConfig}; +use crate::input::{Input, InputProvider}; +use crate::output::{Output, OutputProvider}; +use crate::processor::wasm::wasm_processor::WasmProcessorImpl; +use crate::processor::wasm::wasm_processor_trait::WasmProcessor; +use crate::processor::wasm::wasm_task::WasmTask; +use crate::wasm::task::yaml_keys::{TYPE, type_values}; +use crate::wasm::task::{InputConfig, OutputConfig, ProcessorConfig, WasmTaskConfig}; use serde_yaml::Value; use std::sync::Arc; diff --git a/src/runtime/wasm/task/builder/python/mod.rs b/src/wasm_runtime/src/wasm/task/builder/python/mod.rs similarity index 89% rename from src/runtime/wasm/task/builder/python/mod.rs rename to src/wasm_runtime/src/wasm/task/builder/python/mod.rs index 1b31d2e5..8bc9aa2d 100644 --- a/src/runtime/wasm/task/builder/python/mod.rs +++ b/src/wasm_runtime/src/wasm/task/builder/python/mod.rs @@ -14,14 +14,14 @@ // // Specifically handles building logic for python runtime configuration -use crate::runtime::input::{Input, InputProvider}; -use crate::runtime::output::{Output, OutputProvider}; -use crate::runtime::processor::python::get_python_engine_and_component; -use crate::runtime::processor::wasm::wasm_processor::WasmProcessorImpl; -use crate::runtime::processor::wasm::wasm_processor_trait::WasmProcessor; -use crate::runtime::processor::wasm::wasm_task::WasmTask; -use crate::runtime::wasm::task::yaml_keys::{TYPE, type_values}; -use crate::runtime::wasm::task::{InputConfig, OutputConfig, ProcessorConfig, WasmTaskConfig}; +use crate::input::{Input, InputProvider}; +use crate::output::{Output, OutputProvider}; +use crate::processor::python::get_python_engine_and_component; +use crate::processor::wasm::wasm_processor::WasmProcessorImpl; +use crate::processor::wasm::wasm_processor_trait::WasmProcessor; +use crate::processor::wasm::wasm_task::WasmTask; +use crate::wasm::task::yaml_keys::{TYPE, type_values}; +use crate::wasm::task::{InputConfig, OutputConfig, ProcessorConfig, WasmTaskConfig}; use serde_yaml::Value; use std::sync::Arc; @@ -33,8 +33,7 @@ impl PythonBuilder { yaml_value: &Value, modules: &[(String, Vec)], create_time: u64, - ) -> Result, Box> - { + ) -> Result, Box> { let config_type = yaml_value .get(TYPE) .and_then(|v| v.as_str()) diff --git a/src/runtime/wasm/task/builder/sink/mod.rs b/src/wasm_runtime/src/wasm/task/builder/sink/mod.rs similarity index 95% rename from src/runtime/wasm/task/builder/sink/mod.rs rename to src/wasm_runtime/src/wasm/task/builder/sink/mod.rs index 65e8bc95..844b0aef 100644 --- a/src/runtime/wasm/task/builder/sink/mod.rs +++ b/src/wasm_runtime/src/wasm/task/builder/sink/mod.rs @@ -14,8 +14,8 @@ // // Specifically handles building logic for Sink type configuration (future support) -use crate::runtime::processor::wasm::wasm_task::WasmTask; -use crate::runtime::wasm::task::yaml_keys::{TYPE, type_values}; +use crate::processor::wasm::wasm_task::WasmTask; +use crate::wasm::task::yaml_keys::{TYPE, type_values}; use serde_yaml::Value; use std::sync::Arc; diff --git a/src/runtime/wasm/task/builder/source/mod.rs b/src/wasm_runtime/src/wasm/task/builder/source/mod.rs similarity index 95% rename from src/runtime/wasm/task/builder/source/mod.rs rename to src/wasm_runtime/src/wasm/task/builder/source/mod.rs index fc81bea9..938a5046 100644 --- a/src/runtime/wasm/task/builder/source/mod.rs +++ b/src/wasm_runtime/src/wasm/task/builder/source/mod.rs @@ -14,8 +14,8 @@ // // Specifically handles building logic for Source type configuration (future support) -use crate::runtime::processor::wasm::wasm_task::WasmTask; -use crate::runtime::wasm::task::yaml_keys::{TYPE, type_values}; +use crate::processor::wasm::wasm_task::WasmTask; +use crate::wasm::task::yaml_keys::{TYPE, type_values}; use serde_yaml::Value; use std::sync::Arc; diff --git a/src/runtime/wasm/task/builder/task_builder.rs b/src/wasm_runtime/src/wasm/task/builder/task_builder.rs similarity index 93% rename from src/runtime/wasm/task/builder/task_builder.rs rename to src/wasm_runtime/src/wasm/task/builder/task_builder.rs index 2246d6d8..9ca7236d 100644 --- a/src/runtime/wasm/task/builder/task_builder.rs +++ b/src/wasm_runtime/src/wasm/task/builder/task_builder.rs @@ -15,13 +15,13 @@ //! Provides unified factory methods to create TaskLifecycle instances from YAML config. //! Dispatches to specific builders (Processor, Source, Sink, Python) based on task type. -use crate::runtime::wasm::task::TaskLifecycle; -use crate::runtime::wasm::task::builder::processor::ProcessorBuilder; +use crate::wasm::task::TaskLifecycle; +use crate::wasm::task::builder::processor::ProcessorBuilder; #[cfg(feature = "python")] -use crate::runtime::wasm::task::builder::python::PythonBuilder; -use crate::runtime::wasm::task::builder::sink::SinkBuilder; -use crate::runtime::wasm::task::builder::source::SourceBuilder; -use crate::runtime::wasm::task::yaml_keys::{NAME, TYPE, type_values}; +use crate::wasm::task::builder::python::PythonBuilder; +use crate::wasm::task::builder::sink::SinkBuilder; +use crate::wasm::task::builder::source::SourceBuilder; +use crate::wasm::task::yaml_keys::{NAME, TYPE, type_values}; use serde_yaml::Value; use std::sync::Arc; @@ -169,7 +169,7 @@ impl TaskBuilder { /// Build and unwrap WASM task from Arc fn build_wasm_task( result: Result< - Arc, + Arc, Box, >, task_name: &str, diff --git a/src/runtime/wasm/task/control_mailbox.rs b/src/wasm_runtime/src/wasm/task/control_mailbox.rs similarity index 97% rename from src/runtime/wasm/task/control_mailbox.rs rename to src/wasm_runtime/src/wasm/task/control_mailbox.rs index 8aaf2de3..07397ac6 100644 --- a/src/runtime/wasm/task/control_mailbox.rs +++ b/src/wasm_runtime/src/wasm/task/control_mailbox.rs @@ -10,8 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::runtime::common::TaskCompletionFlag; -use crate::runtime::processor::function_error::FunctionErrorReport; +use crate::common::TaskCompletionFlag; +use crate::processor::function_error::FunctionErrorReport; use crossbeam_channel::Sender; #[derive(Clone)] diff --git a/src/runtime/wasm/task/lifecycle.rs b/src/wasm_runtime/src/wasm/task/lifecycle.rs similarity index 96% rename from src/runtime/wasm/task/lifecycle.rs rename to src/wasm_runtime/src/wasm/task/lifecycle.rs index ea00f7c2..cbe7955d 100644 --- a/src/runtime/wasm/task/lifecycle.rs +++ b/src/wasm_runtime/src/wasm/task/lifecycle.rs @@ -14,10 +14,10 @@ // // Defines the complete lifecycle management interface for Task, including initialization, start, stop, checkpoint and close -use crate::runtime::common::ComponentState; -use crate::runtime::wasm::task::control_mailbox::ControlMailBox; -use crate::runtime::wasm::taskexecutor::InitContext; -use crate::storage::task::FunctionInfo; +use crate::common::ComponentState; +use crate::task::FunctionInfo; +use crate::wasm::task::control_mailbox::ControlMailBox; +use crate::wasm::taskexecutor::InitContext; use std::sync::Arc; /// Task lifecycle management interface diff --git a/src/runtime/wasm/task/mod.rs b/src/wasm_runtime/src/wasm/task/mod.rs similarity index 100% rename from src/runtime/wasm/task/mod.rs rename to src/wasm_runtime/src/wasm/task/mod.rs diff --git a/src/runtime/wasm/task/processor_config.rs b/src/wasm_runtime/src/wasm/task/processor_config.rs similarity index 99% rename from src/runtime/wasm/task/processor_config.rs rename to src/wasm_runtime/src/wasm/task/processor_config.rs index a3069adc..8e642ed6 100644 --- a/src/runtime/wasm/task/processor_config.rs +++ b/src/wasm_runtime/src/wasm/task/processor_config.rs @@ -608,7 +608,7 @@ impl WasmTaskConfig { task_name: String, value: &Value, ) -> Result> { - use crate::runtime::wasm::task::yaml_keys::{INPUT_GROUPS, INPUTS, NAME, OUTPUTS}; + use crate::wasm::task::yaml_keys::{INPUT_GROUPS, INPUTS, NAME, OUTPUTS}; // 1. Get name from config (if exists), otherwise use the passed task_name let config_name = value diff --git a/src/runtime/wasm/task/yaml_keys.rs b/src/wasm_runtime/src/wasm/task/yaml_keys.rs similarity index 100% rename from src/runtime/wasm/task/yaml_keys.rs rename to src/wasm_runtime/src/wasm/task/yaml_keys.rs diff --git a/src/runtime/wasm/taskexecutor/init_context.rs b/src/wasm_runtime/src/wasm/taskexecutor/init_context.rs similarity index 90% rename from src/runtime/wasm/taskexecutor/init_context.rs rename to src/wasm_runtime/src/wasm/taskexecutor/init_context.rs index fca44a32..edd8837d 100644 --- a/src/runtime/wasm/taskexecutor/init_context.rs +++ b/src/wasm_runtime/src/wasm/taskexecutor/init_context.rs @@ -14,10 +14,10 @@ // // Provides various resources needed for task initialization, including state storage, task storage, thread pool, etc. -use crate::runtime::processor::wasm::thread_pool::{TaskThreadPool, ThreadGroup}; -use crate::runtime::wasm::task::ControlMailBox; -use crate::storage::state_backend::StateStorageServer; -use crate::storage::task::TaskStorage; +use crate::processor::wasm::thread_pool::{TaskThreadPool, ThreadGroup}; +use crate::state_backend::StateStorageServer; +use crate::task::TaskStorage; +use crate::wasm::task::ControlMailBox; use std::sync::{Arc, Mutex}; #[derive(Clone)] diff --git a/src/runtime/wasm/taskexecutor/mod.rs b/src/wasm_runtime/src/wasm/taskexecutor/mod.rs similarity index 100% rename from src/runtime/wasm/taskexecutor/mod.rs rename to src/wasm_runtime/src/wasm/taskexecutor/mod.rs diff --git a/src/runtime/wasm/taskexecutor/task_manager.rs b/src/wasm_runtime/src/wasm/taskexecutor/task_manager.rs similarity index 96% rename from src/runtime/wasm/taskexecutor/task_manager.rs rename to src/wasm_runtime/src/wasm/taskexecutor/task_manager.rs index 897e0a3d..d88c3df4 100644 --- a/src/runtime/wasm/taskexecutor/task_manager.rs +++ b/src/wasm_runtime/src/wasm/taskexecutor/task_manager.rs @@ -10,15 +10,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::common::ComponentState; use crate::config::GlobalConfig; -use crate::runtime::common::ComponentState; -use crate::runtime::processor::wasm::thread_pool::{GlobalTaskThreadPool, TaskThreadPool}; -use crate::runtime::wasm::task::{TaskBuilder, TaskLifecycle}; -use crate::runtime::wasm::taskexecutor::init_context::InitContext; -use crate::storage::state_backend::StateStorageServer; -use crate::storage::task::{ - FunctionInfo, StoredTaskInfo, TaskModuleBytes, TaskStorage, TaskStorageFactory, -}; +use crate::processor::wasm::thread_pool::{GlobalTaskThreadPool, TaskThreadPool}; +use crate::state_backend::StateStorageServer; +use crate::task::{FunctionInfo, StoredTaskInfo, TaskModuleBytes, TaskStorage, TaskStorageFactory}; +use crate::wasm::task::{TaskBuilder, TaskLifecycle}; +use crate::wasm::taskexecutor::init_context::InitContext; use anyhow::{Context, Result, anyhow}; use parking_lot::RwLock;