diff --git a/Cargo.lock b/Cargo.lock index 0bd7f0e1fda..79c6ed14cfc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -250,7 +250,7 @@ dependencies = [ "rustls-native-certs 0.7.3", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls", "tokio-util", "tracing", "webpki-roots 0.26.11", @@ -957,19 +957,14 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.19" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" dependencies = [ - "bzip2 0.5.2", - "flate2", - "futures-core", - "memchr", + "compression-codecs", + "compression-core", "pin-project-lite", "tokio", - "xz2", - "zstd 0.13.3", - "zstd-safe 7.2.4", ] [[package]] @@ -1050,7 +1045,7 @@ dependencies = [ "ring", "rustls-native-certs 0.8.1", "rustls-pki-types", - "rustls-webpki 0.103.12", + "rustls-webpki", "serde", "serde_json", "serde_nanos", @@ -1058,7 +1053,7 @@ dependencies = [ "thiserror 1.0.69", "time", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls", "tokio-stream", "tokio-util", "tokio-websockets", @@ -1221,7 +1216,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rand 0.9.4", - "rustls 0.23.37", + "rustls", "serde", "serde_json", "serde_urlencoded", @@ -1356,9 +1351,9 @@ dependencies = [ [[package]] name = "aws-sdk-dynamodb" -version = "1.104.0" +version = "1.110.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04c47115cc8d46dcc94a9a81e7a3384cea859283c1a737729691d4221f11584" +checksum = "0c597424385739456cd04535982831c15bd58f97ca28c5bcb232c0d730d5cb39" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1612,23 +1607,17 @@ dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "h2 0.3.27", "h2 0.4.13", - "http 0.2.12", "http 1.3.1", - "http-body 0.4.6", - "hyper 0.14.32", "hyper 1.6.0", - "hyper-rustls 0.24.2", - "hyper-rustls 0.27.6", + "hyper-rustls", "hyper-util", "pin-project-lite", - "rustls 0.21.12", - "rustls 0.23.37", + "rustls", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls", "tower", "tracing", ] @@ -2043,6 +2032,16 @@ dependencies = [ "piper", ] +[[package]] +name = "bnum" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f781dba93de3a5ef6dc5b17c9958b208f6f3f021623b360fb605ea51ce443f10" +dependencies = [ + "serde", + "serde-big-array", +] + [[package]] name = "bon" version = "3.9.0" @@ -2213,15 +2212,6 @@ dependencies = [ "bytes", ] -[[package]] -name = "bzip2" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" -dependencies = [ - "bzip2-sys", -] - [[package]] name = "bzip2" version = "0.6.1" @@ -2231,16 +2221,6 @@ dependencies = [ "libbz2-rs-sys", ] -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" -dependencies = [ - "cc", - "pkg-config", -] - [[package]] name = "cached" version = "0.43.0" @@ -2575,6 +2555,27 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "120133d4db2ec47efe2e26502ee984747630c67f51974fca0b6c1340cf2368d3" +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "bzip2", + "compression-core", + "flate2", + "liblzma", + "memchr", + "zstd 0.13.3", + "zstd-safe 7.2.4", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -3137,15 +3138,15 @@ checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" [[package]] name = "datafusion" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ba7cb113e9c0bedf9e9765926031e132fa05a1b09ba6e93a6d1a4d7044457b8" +checksum = "7541353e77dc7262b71ca27be07d8393661737e3a73b5d1b1c6f7d814c64fa2a" dependencies = [ "arrow", "arrow-schema", "async-trait", "bytes", - "bzip2 0.6.1", + "bzip2", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -3175,27 +3176,26 @@ dependencies = [ "flate2", "futures", "itertools 0.14.0", + "liblzma", "log", - "object_store 0.12.5", + "object_store", "parking_lot 0.12.4", "parquet", "rand 0.9.4", "regex", - "rstest 0.26.1", "sqlparser", "tempfile", "tokio", "url", "uuid", - "xz2", "zstd 0.13.3", ] [[package]] name = "datafusion-catalog" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66a3a799f914a59b1ea343906a0486f17061f39509af74e874a866428951130d" +checksum = "9997731f90fa5398ef831ad0e69600f92c861b79c0d38bd1a29b6f0e3a0ce4c8" dependencies = [ "arrow", "async-trait", @@ -3211,16 +3211,16 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store 0.12.5", + "object_store", "parking_lot 0.12.4", "tokio", ] [[package]] name = "datafusion-catalog-listing" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db1b113c80d7a0febcd901476a57aef378e717c54517a163ed51417d87621b0" +checksum = "2b30a3dd50dec860c9559275c8d97d9de602e611237a6ecfbda0b3b63b872352" dependencies = [ "arrow", "async-trait", @@ -3236,26 +3236,25 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store 0.12.5", - "tokio", + "object_store", ] [[package]] name = "datafusion-common" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c10f7659e96127d25e8366be7c8be4109595d6a2c3eac70421f380a7006a1b0" +checksum = "d551054acec0398ca604512310b77ce05c46f66e54b54d48200a686e385cca4e" dependencies = [ "ahash 0.8.12", "arrow", "arrow-ipc", "chrono", "half", - "hashbrown 0.14.5", + "hashbrown 0.16.1", "indexmap 2.13.0", "libc", "log", - "object_store 0.12.5", + "object_store", "parquet", "paste", "recursive", @@ -3266,9 +3265,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b92065bbc6532c6651e2f7dd30b55cba0c7a14f860c7e1d15f165c41a1868d95" +checksum = "567d40e285f5b79f8737b576605721cd6c1133b5d2b00bdbd5d9838d90d0812f" dependencies = [ "futures", "log", @@ -3277,15 +3276,15 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde13794244bc7581cd82f6fff217068ed79cdc344cafe4ab2c3a1c3510b38d6" +checksum = "27d2668f51b3b30befae2207472569e37807fdedd1d14da58acc6f8ca6257eae" dependencies = [ "arrow", "async-compression", "async-trait", "bytes", - "bzip2 0.6.1", + "bzip2", "chrono", "datafusion-common", "datafusion-common-runtime", @@ -3300,21 +3299,21 @@ dependencies = [ "futures", "glob", "itertools 0.14.0", + "liblzma", "log", - "object_store 0.12.5", + "object_store", "rand 0.9.4", "tokio", "tokio-util", "url", - "xz2", "zstd 0.13.3", ] [[package]] name = "datafusion-datasource-arrow" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "804fa9b4ecf3157982021770617200ef7c1b2979d57bec9044748314775a9aea" +checksum = "e02e1b3e3a8ec55f1f62de4252b0407c8567363d056078769a197e24fc834a0f" dependencies = [ "arrow", "arrow-ipc", @@ -3330,15 +3329,15 @@ dependencies = [ "datafusion-session", "futures", "itertools 0.14.0", - "object_store 0.12.5", + "object_store", "tokio", ] [[package]] name = "datafusion-datasource-csv" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a1641a40b259bab38131c5e6f48fac0717bedb7dc93690e604142a849e0568" +checksum = "b559d7bf87d4f900f847baba8509634f838d9718695389e903604cdcccdb01f3" dependencies = [ "arrow", "async-trait", @@ -3352,16 +3351,16 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "object_store 0.12.5", + "object_store", "regex", "tokio", ] [[package]] name = "datafusion-datasource-json" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adeacdb00c1d37271176f8fb6a1d8ce096baba16ea7a4b2671840c5c9c64fe85" +checksum = "250e2d7591ba8b638f063854650faa40bca4e8bd4059b2ece8836f6388d02db4" dependencies = [ "arrow", "async-trait", @@ -3375,15 +3374,15 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "object_store 0.12.5", + "object_store", "tokio", ] [[package]] name = "datafusion-datasource-parquet" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d0b60ffd66f28bfb026565d62b0a6cbc416da09814766a3797bba7d85a3cd9" +checksum = "0b043149f2c3557ca94abc58de40f68a8d412ff53365c06126ed234f8596399d" dependencies = [ "arrow", "async-trait", @@ -3403,7 +3402,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store 0.12.5", + "object_store", "parking_lot 0.12.4", "parquet", "tokio", @@ -3411,24 +3410,25 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b99e13947667b36ad713549237362afb054b2d8f8cc447751e23ec61202db07" +checksum = "b9496cb0db222dbb9a3735760ceca7fc56f35e1d5502c38d0caa77a81e9c1f6a" [[package]] name = "datafusion-execution" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63695643190679037bc946ad46a263b62016931547bf119859c511f7ff2f5178" +checksum = "dc45d23c516ed8d3637751e44e09e21b45b3f58b473c802dddd1f1ad4fe435ff" dependencies = [ "arrow", "async-trait", + "chrono", "dashmap 6.1.0", "datafusion-common", "datafusion-expr", "futures", "log", - "object_store 0.12.5", + "object_store", "parking_lot 0.12.4", "rand 0.9.4", "tempfile", @@ -3437,9 +3437,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a4787cbf5feb1ab351f789063398f67654a6df75c4d37d7f637dc96f951a91" +checksum = "63dd30526d2db4fda6440806a41e4676334a94bc0596cc9cc2a0efed20ef2c44" dependencies = [ "arrow", "async-trait", @@ -3460,9 +3460,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce2fb1b8c15c9ac45b0863c30b268c69dc9ee7a1ee13ecf5d067738338173dc" +checksum = "1b486b5f6255d40976b88bb83813b0d035a8333e0ec39864824e78068cf42fa6" dependencies = [ "arrow", "datafusion-common", @@ -3473,9 +3473,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "794a9db7f7b96b3346fc007ff25e994f09b8f0511b4cf7dff651fadfe3ebb28f" +checksum = "07356c94118d881130dd0ffbff127540407d969c8978736e324edcd6c41cd48f" dependencies = [ "arrow", "arrow-buffer", @@ -3483,6 +3483,7 @@ dependencies = [ "blake2", "blake3", "chrono", + "chrono-tz", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -3503,9 +3504,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c25210520a9dcf9c2b2cbbce31ebd4131ef5af7fc60ee92b266dc7d159cb305" +checksum = "b644f9cf696df9233ce6958b9807666d78563b56f923267474dd6c07795f1f8f" dependencies = [ "ahash 0.8.12", "arrow", @@ -3524,9 +3525,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f4a66f3b87300bb70f4124b55434d2ae3fe80455f3574701d0348da040b55d" +checksum = "c1de2deaaabe8923ce9ea9f29c47bbb4ee14f67ea2fe1ab5398d9bbebcf86e56" dependencies = [ "ahash 0.8.12", "arrow", @@ -3537,9 +3538,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae5c06eed03918dc7fe7a9f082a284050f0e9ecf95d72f57712d1496da03b8c4" +checksum = "552f8d92e4331ee91d23c02d12bb6acf32cbfd5215117e01c0fb63cd4b15af1a" dependencies = [ "arrow", "arrow-ord", @@ -3560,9 +3561,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4fed1d71738fbe22e2712d71396db04c25de4111f1ec252b8f4c6d3b25d7f5" +checksum = "970fd0cdd3df8802b9a9975ff600998289ba9d46682a4f7285cba4820c9ada78" dependencies = [ "arrow", "async-trait", @@ -3576,9 +3577,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d92206aa5ae21892f1552b4d61758a862a70956e6fd7a95cb85db1de74bc6d1" +checksum = "40b4c21a7c8a986a1866c0a87ab756d0bbf7b5f41f306009fa2d9af79c52ed31" dependencies = [ "arrow", "datafusion-common", @@ -3594,9 +3595,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53ae9bcc39800820d53a22d758b3b8726ff84a5a3e24cecef04ef4e5fdf1c7cc" +checksum = "b1210ad73b8b3211aeaf4a42bef9bd7a2b7fce3ec119a478831f18c6ff7f7b93" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -3604,9 +3605,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1063ad4c9e094b3f798acee16d9a47bd7372d9699be2de21b05c3bd3f34ab848" +checksum = "aaa566a963013a38681ad82a727a654bc7feb19632426aea8c3412d415d200c5" dependencies = [ "datafusion-doc", "quote", @@ -3615,9 +3616,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35f9ec5d08b87fd1893a30c2929f2559c2f9806ca072d8fefca5009dc0f06a" +checksum = "ff9aa82b240252a88dee118372f9b9757c545ab9e53c0736bebab2e7da0ef1f2" dependencies = [ "arrow", "chrono", @@ -3635,9 +3636,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c30cc8012e9eedcb48bbe112c6eff4ae5ed19cf3003cb0f505662e88b7014c5d" +checksum = "7d48022b8af9988c1d852644f9e8b5584c490659769a550c5e8d39457a1da0a5" dependencies = [ "ahash 0.8.12", "arrow", @@ -3647,19 +3648,21 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", - "hashbrown 0.14.5", + "hashbrown 0.16.1", "indexmap 2.13.0", "itertools 0.14.0", "parking_lot 0.12.4", "paste", "petgraph 0.8.3", + "recursive", + "tokio", ] [[package]] name = "datafusion-physical-expr-adapter" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9ff2dbd476221b1f67337699eff432781c4e6e1713d2aefdaa517dfbf79768" +checksum = "ae7a8abc0b4fe624000972a9b145b30b7f1b680bffaa950ea53f78d9b21c27c3" dependencies = [ "arrow", "datafusion-common", @@ -3672,23 +3675,26 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90da43e1ec550b172f34c87ec68161986ced70fd05c8d2a2add66eef9c276f03" +checksum = "147253ca3e6b9d59c162de64c02800973018660e13340dd1886dd038d17ac429" dependencies = [ "ahash 0.8.12", "arrow", + "chrono", "datafusion-common", "datafusion-expr-common", - "hashbrown 0.14.5", + "hashbrown 0.16.1", + "indexmap 2.13.0", "itertools 0.14.0", + "parking_lot 0.12.4", ] [[package]] name = "datafusion-physical-optimizer" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce9804f799acd7daef3be7aaffe77c0033768ed8fdbf5fb82fc4c5f2e6bc14e6" +checksum = "689156bb2282107b6239db8d7ef44b4dab10a9b33d3491a0c74acac5e4fedd72" dependencies = [ "arrow", "datafusion-common", @@ -3705,27 +3711,27 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0acf0ad6b6924c6b1aa7d213b181e012e2d3ec0a64ff5b10ee6282ab0f8532ac" +checksum = "68253dc0ee5330aa558b2549c9b0da5af9fc17d753ae73022939014ad616fc28" dependencies = [ "ahash 0.8.12", "arrow", "arrow-ord", "arrow-schema", "async-trait", - "chrono", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", + "datafusion-functions", "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", "futures", "half", - "hashbrown 0.14.5", + "hashbrown 0.16.1", "indexmap 2.13.0", "itertools 0.14.0", "log", @@ -3736,9 +3742,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d368093a98a17d1449b1083ac22ed16b7128e4c67789991869480d8c4a40ecb9" +checksum = "3f5ab57d0b5a368258fff1d828f1619a10541fa5c4ec4930a383deb3a23204c8" dependencies = [ "arrow", "chrono", @@ -3757,15 +3763,15 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-proto-common", - "object_store 0.12.5", + "object_store", "prost 0.14.3", ] [[package]] name = "datafusion-proto-common" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b6aef3d5e5c1d2bc3114c4876730cb76a9bdc5a8df31ef1b6db48f0c1671895" +checksum = "bd21d2c804802ca4b1719191dfe8e3d0860686649de6375ddc9237f85beb82b3" dependencies = [ "arrow", "datafusion-common", @@ -3774,9 +3780,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac2c2498a1f134a9e11a9f5ed202a2a7d7e9774bd9249295593053ea3be999db" +checksum = "0fcad240a54d0b1d3e8f668398900260a53122d522b2102ab57218590decacd6" dependencies = [ "arrow", "datafusion-common", @@ -3791,9 +3797,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f96eebd17555386f459037c65ab73aae8df09f464524c709d6a3134ad4f4776" +checksum = "f58e83a68bb67007a8fcbf005c44cefe441270c7ee7f6dee10c0e0109b556f6d" dependencies = [ "async-trait", "datafusion-common", @@ -3805,9 +3811,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fc195fe60634b2c6ccfd131b487de46dc30eccae8a3c35a13f136e7f440414f" +checksum = "be53e9eb55db0fbb8980bb6d87f2435b0524acf4c718ed54a57cabbb299b2ab3" dependencies = [ "arrow", "bigdecimal", @@ -3963,7 +3969,7 @@ dependencies = [ "deltalake-catalog-unity", "dyn-clone", "enum-map", - "erased-serde", + "erased-serde 0.3.31", "etl", "etl-config", "etl-postgres", @@ -4020,7 +4026,7 @@ dependencies = [ "rkyv", "rmp-serde", "rmpv", - "rustls 0.23.37", + "rustls", "schema_registry_converter", "semver", "sentry", @@ -4075,7 +4081,7 @@ dependencies = [ "rdkafka", "regex", "rkyv", - "rstest 0.15.0", + "rstest", "serde", "serde_with", "time", @@ -4162,7 +4168,7 @@ dependencies = [ "futures", "indexmap 2.13.0", "itertools 0.14.0", - "object_store 0.12.5", + "object_store", "parquet", "reqwest 0.12.24", "roaring", @@ -4191,8 +4197,8 @@ dependencies = [ [[package]] name = "deltalake" -version = "0.30.2" -source = "git+https://github.com/gz/delta-rs.git?rev=c37cf6e788d653db4149b205542559cf25674002#c37cf6e788d653db4149b205542559cf25674002" +version = "0.31.1" +source = "git+https://github.com/feldera/delta-rs.git?rev=93d94dc66976788a378313af7eff3312ad0d294c#93d94dc66976788a378313af7eff3312ad0d294c" dependencies = [ "ctor", "delta_kernel", @@ -4205,8 +4211,8 @@ dependencies = [ [[package]] name = "deltalake-aws" -version = "0.13.1" -source = "git+https://github.com/gz/delta-rs.git?rev=c37cf6e788d653db4149b205542559cf25674002#c37cf6e788d653db4149b205542559cf25674002" +version = "0.14.0" +source = "git+https://github.com/feldera/delta-rs.git?rev=93d94dc66976788a378313af7eff3312ad0d294c#93d94dc66976788a378313af7eff3312ad0d294c" dependencies = [ "async-trait", "aws-config", @@ -4219,7 +4225,7 @@ dependencies = [ "chrono", "deltalake-core", "futures", - "object_store 0.12.5", + "object_store", "regex", "thiserror 2.0.17", "tokio", @@ -4231,12 +4237,12 @@ dependencies = [ [[package]] name = "deltalake-azure" -version = "0.13.0" -source = "git+https://github.com/gz/delta-rs.git?rev=c37cf6e788d653db4149b205542559cf25674002#c37cf6e788d653db4149b205542559cf25674002" +version = "0.14.0" +source = "git+https://github.com/feldera/delta-rs.git?rev=93d94dc66976788a378313af7eff3312ad0d294c#93d94dc66976788a378313af7eff3312ad0d294c" dependencies = [ "bytes", "deltalake-core", - "object_store 0.12.5", + "object_store", "thiserror 2.0.17", "tokio", "url", @@ -4244,8 +4250,8 @@ dependencies = [ [[package]] name = "deltalake-catalog-unity" -version = "0.14.1" -source = "git+https://github.com/gz/delta-rs.git?rev=c37cf6e788d653db4149b205542559cf25674002#c37cf6e788d653db4149b205542559cf25674002" +version = "0.15.0" +source = "git+https://github.com/feldera/delta-rs.git?rev=93d94dc66976788a378313af7eff3312ad0d294c#93d94dc66976788a378313af7eff3312ad0d294c" dependencies = [ "async-trait", "chrono", @@ -4271,8 +4277,8 @@ dependencies = [ [[package]] name = "deltalake-core" -version = "0.30.2" -source = "git+https://github.com/gz/delta-rs.git?rev=c37cf6e788d653db4149b205542559cf25674002#c37cf6e788d653db4149b205542559cf25674002" +version = "0.31.1" +source = "git+https://github.com/feldera/delta-rs.git?rev=93d94dc66976788a378313af7eff3312ad0d294c#93d94dc66976788a378313af7eff3312ad0d294c" dependencies = [ "arrow", "arrow-arith", @@ -4292,6 +4298,7 @@ dependencies = [ "dashmap 6.1.0", "datafusion", "datafusion-datasource", + "datafusion-physical-expr-adapter", "datafusion-proto", "delta_kernel", "deltalake-derive", @@ -4302,7 +4309,7 @@ dependencies = [ "indexmap 2.13.0", "itertools 0.14.0", "num_cpus", - "object_store 0.12.5", + "object_store", "parking_lot 0.12.4", "parquet", "percent-encoding", @@ -4324,8 +4331,8 @@ dependencies = [ [[package]] name = "deltalake-derive" -version = "0.30.0" -source = "git+https://github.com/gz/delta-rs.git?rev=c37cf6e788d653db4149b205542559cf25674002#c37cf6e788d653db4149b205542559cf25674002" +version = "0.31.0" +source = "git+https://github.com/feldera/delta-rs.git?rev=93d94dc66976788a378313af7eff3312ad0d294c#93d94dc66976788a378313af7eff3312ad0d294c" dependencies = [ "convert_case 0.9.0", "itertools 0.14.0", @@ -4336,14 +4343,14 @@ dependencies = [ [[package]] name = "deltalake-gcp" -version = "0.14.0" -source = "git+https://github.com/gz/delta-rs.git?rev=c37cf6e788d653db4149b205542559cf25674002#c37cf6e788d653db4149b205542559cf25674002" +version = "0.15.0" +source = "git+https://github.com/feldera/delta-rs.git?rev=93d94dc66976788a378313af7eff3312ad0d294c#93d94dc66976788a378313af7eff3312ad0d294c" dependencies = [ "async-trait", "bytes", "deltalake-core", "futures", - "object_store 0.12.5", + "object_store", "thiserror 2.0.17", "tokio", "tracing", @@ -4841,6 +4848,17 @@ dependencies = [ "serde", ] +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + [[package]] name = "errno" version = "0.3.12" @@ -4886,14 +4904,14 @@ dependencies = [ "pg_escape", "pin-project-lite", "postgres-replication", - "rustls 0.23.37", + "rustls", "serde", "serde_json", "sqlx", "sysinfo 0.38.4", "tokio", "tokio-postgres 0.7.11", - "tokio-rustls 0.26.2", + "tokio-rustls", "tokio-stream", "tracing", "uuid", @@ -5010,6 +5028,18 @@ dependencies = [ "siphasher", ] +[[package]] +name = "fastnum" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4089ab2dfd45d8ddc92febb5ca80644389d5ebb954f40231274a3f18341762e2" +dependencies = [ + "bnum", + "num-integer", + "num-traits", + "serde", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -5092,7 +5122,7 @@ dependencies = [ "datafusion", "dbsp", "dyn-clone", - "erased-serde", + "erased-serde 0.3.31", "feldera-sqllib", "feldera-types", "num-derive", @@ -5199,9 +5229,11 @@ dependencies = [ "iceberg-catalog-glue", "iceberg-catalog-rest", "iceberg-datafusion", + "iceberg-storage-opendal", "log", "serde_json", "tokio", + "url", ] [[package]] @@ -5351,7 +5383,7 @@ dependencies = [ "itertools 0.14.0", "libc", "nix 0.27.1", - "object_store 0.11.2", + "object_store", "once_cell", "rkyv", "serde", @@ -5374,7 +5406,7 @@ dependencies = [ "clap", "csv", "enum-map", - "erased-serde", + "erased-serde 0.3.31", "feldera-ir", "libc", "log", @@ -6284,21 +6316,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper-rustls" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" -dependencies = [ - "futures-util", - "http 0.2.12", - "hyper 0.14.32", - "log", - "rustls 0.21.12", - "tokio", - "tokio-rustls 0.24.1", -] - [[package]] name = "hyper-rustls" version = "0.27.6" @@ -6308,11 +6325,11 @@ dependencies = [ "http 1.3.1", "hyper 1.6.0", "hyper-util", - "rustls 0.23.37", + "rustls", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls", "tower-service", "webpki-roots 1.0.0", ] @@ -6398,9 +6415,9 @@ dependencies = [ [[package]] name = "iceberg" -version = "0.8.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e65918e701cf610ab0cea57f7f31db5bf4f973230c2c160244067bce01f7c5fa" +checksum = "4d9c3fc1f55c84ff64645c0d2ee35159f5574d33f729fc0860783bc247c8b8c5" dependencies = [ "anyhow", "apache-avro 0.21.0", @@ -6422,22 +6439,19 @@ dependencies = [ "chrono", "derive_builder", "expect-test", + "fastnum", "flate2", "fnv", "futures", "itertools 0.13.0", "moka", "murmur3", - "num-bigint", "once_cell", - "opendal", "ordered-float 4.6.0", "parquet", - "rand 0.8.6", - "reqsign", + "rand 0.9.4", "reqwest 0.12.24", "roaring", - "rust_decimal", "serde", "serde_bytes", "serde_derive", @@ -6447,6 +6461,7 @@ dependencies = [ "strum", "tokio", "typed-builder 0.20.1", + "typetag", "url", "uuid", "zstd 0.13.3", @@ -6454,15 +6469,16 @@ dependencies = [ [[package]] name = "iceberg-catalog-glue" -version = "0.8.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf46eabd76b154e569627ce7642933df7c2450eb770111241f4b40735a2644a" +checksum = "5c510ae5f666202151bd24728a2cb569af04694d04e67788ba5ec00770314301" dependencies = [ "anyhow", "async-trait", "aws-config", "aws-sdk-glue", "iceberg", + "iceberg-storage-opendal", "serde_json", "tokio", "tracing", @@ -6470,9 +6486,9 @@ dependencies = [ [[package]] name = "iceberg-catalog-rest" -version = "0.8.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6d5e120317ab88a3af332c17166aad101f2aee9bfb098d63d4525bdd5cc2da7" +checksum = "a49dfef578060c3a2a3f619522dcb25382a7ce85336dcb500495d4b8d72ae714" dependencies = [ "async-trait", "chrono", @@ -6491,12 +6507,13 @@ dependencies = [ [[package]] name = "iceberg-datafusion" -version = "0.8.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3fe6bf6f07cc8ace8ab19a07e7c51c38cac3e225f5157d5d72b8fda30edbb1" +checksum = "15f668954750597310308ba6ce3a6d3a6874196e64a8a3197e3d1c5017977ba6" dependencies = [ "anyhow", "async-trait", + "dashmap 6.1.0", "datafusion", "futures", "iceberg", @@ -6505,6 +6522,25 @@ dependencies = [ "uuid", ] +[[package]] +name = "iceberg-storage-opendal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30acae4698949eea6cfbd3e64ddf1c3ec39e6b7191acb0b0da2dcd1a25dfa842" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "cfg-if", + "iceberg", + "opendal", + "reqsign", + "reqwest 0.12.24", + "serde", + "typetag", + "url", +] + [[package]] name = "icu_collections" version = "2.0.0" @@ -7058,6 +7094,26 @@ dependencies = [ "windows-targets 0.53.5", ] +[[package]] +name = "liblzma" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6033b77c21d1f56deeae8014eb9fbe7bdf1765185a6c508b5ca82eeaed7f899" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a60851d15cd8c5346eca4ab8babff585be2ae4bc8097c067291d3ffe2add3b6" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "libm" version = "0.2.15" @@ -7215,17 +7271,6 @@ dependencies = [ "sha2", ] -[[package]] -name = "lzma-sys" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "mach2" version = "0.4.3" @@ -7773,38 +7818,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "object_store" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cfccb68961a56facde1163f9319e0d15743352344e7808a11795fb99698dcaf" -dependencies = [ - "async-trait", - "base64 0.22.1", - "bytes", - "chrono", - "futures", - "httparse", - "humantime", - "hyper 1.6.0", - "itertools 0.13.0", - "md-5", - "parking_lot 0.12.4", - "percent-encoding", - "quick-xml 0.37.5", - "rand 0.8.6", - "reqwest 0.12.24", - "ring", - "rustls-pemfile", - "serde", - "serde_json", - "snafu", - "tokio", - "tracing", - "url", - "walkdir", -] - [[package]] name = "object_store" version = "0.12.5" @@ -8197,7 +8210,7 @@ dependencies = [ "num-bigint", "num-integer", "num-traits", - "object_store 0.12.5", + "object_store", "paste", "seq-macro", "serde_json", @@ -8460,7 +8473,7 @@ dependencies = [ "regex", "reqwest 0.12.24", "rmp-serde", - "rustls 0.23.37", + "rustls", "semver", "sentry", "serde", @@ -9333,7 +9346,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls 0.23.37", + "rustls", "socket2 0.5.10", "thiserror 2.0.17", "tokio", @@ -9353,7 +9366,7 @@ dependencies = [ "rand 0.9.4", "ring", "rustc-hash 2.1.1", - "rustls 0.23.37", + "rustls", "rustls-pki-types", "slab", "thiserror 2.0.17", @@ -9788,12 +9801,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "relative-path" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" - [[package]] name = "rend" version = "0.4.2" @@ -9889,7 +9896,7 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "hyper 1.6.0", - "hyper-rustls 0.27.6", + "hyper-rustls", "hyper-tls", "hyper-util", "js-sys", @@ -9899,7 +9906,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.37", + "rustls", "rustls-native-certs 0.8.1", "rustls-pki-types", "serde", @@ -9908,7 +9915,7 @@ dependencies = [ "sync_wrapper 1.0.2", "tokio", "tokio-native-tls", - "tokio-rustls 0.26.2", + "tokio-rustls", "tokio-util", "tower", "tower-http", @@ -10164,21 +10171,10 @@ checksum = "e9c9dc66cc29792b663ffb5269be669f1613664e69ad56441fdb895c2347b930" dependencies = [ "futures", "futures-timer", - "rstest_macros 0.14.0", + "rstest_macros", "rustc_version", ] -[[package]] -name = "rstest" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" -dependencies = [ - "futures-timer", - "futures-util", - "rstest_macros 0.26.1", -] - [[package]] name = "rstest_macros" version = "0.14.0" @@ -10192,24 +10188,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "rstest_macros" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" -dependencies = [ - "cfg-if", - "glob", - "proc-macro-crate", - "proc-macro2", - "quote", - "regex", - "relative-path", - "rustc_version", - "syn 2.0.117", - "unicode-ident", -] - [[package]] name = "rust-embed" version = "8.7.2" @@ -10265,22 +10243,6 @@ dependencies = [ "trim-in-place", ] -[[package]] -name = "rust_decimal" -version = "1.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35affe401787a9bd846712274d97654355d21b2a2c092a3139aabe31e9022282" -dependencies = [ - "arrayvec 0.7.6", - "borsh", - "bytes", - "num-traits", - "rand 0.8.6", - "rkyv", - "serde", - "serde_json", -] - [[package]] name = "rustc-demangle" version = "0.1.24" @@ -10334,18 +10296,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "rustls" -version = "0.21.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" -dependencies = [ - "log", - "ring", - "rustls-webpki 0.101.7", - "sct", -] - [[package]] name = "rustls" version = "0.23.37" @@ -10357,7 +10307,7 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.12", + "rustls-webpki", "subtle", "zeroize", ] @@ -10406,16 +10356,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustls-webpki" -version = "0.101.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" -dependencies = [ - "ring", - "untrusted 0.9.0", -] - [[package]] name = "rustls-webpki" version = "0.103.12" @@ -10595,16 +10535,6 @@ dependencies = [ "sha2", ] -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted 0.9.0", -] - [[package]] name = "sdd" version = "3.0.8" @@ -10696,7 +10626,7 @@ dependencies = [ "httpdate", "native-tls", "reqwest 0.12.24", - "rustls 0.23.37", + "rustls", "sentry-actix", "sentry-anyhow", "sentry-backtrace", @@ -10838,6 +10768,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-big-array" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" +dependencies = [ + "serde", +] + [[package]] name = "serde_arrow" version = "0.13.7" @@ -11258,27 +11197,6 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" -[[package]] -name = "snafu" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320b01e011bf8d5d7a4a4a4be966d9160968935849c83b918827f6a435e7f627" -dependencies = [ - "snafu-derive", -] - -[[package]] -name = "snafu-derive" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1961e2ef424c1424204d3a5d6975f934f56b6d50ff5732382d84ebf460e147f7" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "snap" version = "1.1.1" @@ -11411,7 +11329,7 @@ dependencies = [ "native-tls", "once_cell", "percent-encoding", - "rustls 0.23.37", + "rustls", "serde", "serde_json", "sha2", @@ -12268,23 +12186,13 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-rustls" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" -dependencies = [ - "rustls 0.21.12", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.23.37", + "rustls", "tokio", ] @@ -12347,7 +12255,7 @@ dependencies = [ "ring", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls", "tokio-util", "webpki-roots 0.26.11", ] @@ -12413,7 +12321,7 @@ dependencies = [ "pin-project", "sync_wrapper 1.0.2", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls", "tokio-stream", "tower", "tower-layer", @@ -12667,12 +12575,42 @@ dependencies = [ "dashmap 5.5.3", ] +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + [[package]] name = "typenum" version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +[[package]] +name = "typetag" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be2212c8a9b9bcfca32024de14998494cf9a5dfa59ea1b829de98bac374b86bf" +dependencies = [ + "erased-serde 0.4.10", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27a7a9b72ba121f6f1f6c3632b85604cac41aedb5ddc70accbebb6cac83de846" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "typify" version = "0.1.0" @@ -12827,7 +12765,7 @@ dependencies = [ "log", "native-tls", "percent-encoding", - "rustls 0.23.37", + "rustls", "rustls-pemfile", "rustls-pki-types", "ureq-proto", @@ -14122,15 +14060,6 @@ version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" -[[package]] -name = "xz2" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] - [[package]] name = "yaml-rust2" version = "0.8.1" @@ -14296,7 +14225,7 @@ checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" dependencies = [ "aes", "arbitrary", - "bzip2 0.6.1", + "bzip2", "constant_time_eq", "crc32fast", "deflate64", diff --git a/Cargo.toml b/Cargo.toml index 2b66e802fcf..5b7cbc12686 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,19 +102,25 @@ crossbeam-utils = "0.8.6" csv = "1.2.2" csv-core = "0.1.10" dashmap = "6.1.0" -datafusion = "51.0" +datafusion = "52.2" dbsp = { path = "crates/dbsp", version = "0.296.0" } dbsp_nexmark = { path = "crates/nexmark" } deadpool-postgres = "0.14.1" -#deltalake = "0.30.2" -#deltalake-catalog-unity = "0.14.1" -#deltalake-catalog-unity = { git = "https://github.com/ryzhyk/delta-rs.git", rev = "b24e5aaec322db0f084449e5c57f97ff4526bbdc" } -#deltalake = { git = "https://github.com/ryzhyk/delta-rs.git", rev = "b24e5aaec322db0f084449e5c57f97ff4526bbdc" } -deltalake-catalog-unity = { git = "https://github.com/gz/delta-rs.git", rev = "c37cf6e788d653db4149b205542559cf25674002" } -deltalake = { git = "https://github.com/gz/delta-rs.git", rev = "c37cf6e788d653db4149b205542559cf25674002" } +# Feldera fork of delta-rs: upstream tag `rust-v0.31.1` + 3 patches, on branch +# `v0.31.1-feldera`. When bumping the pinned revision, preserve these patches: +# - 71d185e5 "Initialize unity from environment." — let the unity catalog +# pick up credentials/host from process env, matching how we configure +# other connectors. +# - d2c12edd "Disable unsupported feature check." — relax the writer-side +# protocol-feature gate so we can write tables produced by newer clients. +# - 93d94dc6 "delta-rs: round-robin pre-split unpartitioned scans into +# target_partitions FileGroups" — improves snapshot read parallelism for +# unpartitioned tables; pairs with the DataFusion knobs we tune in adapters. +# Diff vs. upstream: +# https://github.com/delta-io/delta-rs/compare/rust-v0.31.1...feldera:delta-rs:v0.31.1-feldera +deltalake = { git = "https://github.com/feldera/delta-rs.git", rev = "93d94dc66976788a378313af7eff3312ad0d294c" } +deltalake-catalog-unity = { git = "https://github.com/feldera/delta-rs.git", rev = "93d94dc66976788a378313af7eff3312ad0d294c" } -# deltalake = { git = "https://github.com/ryzhyk/delta-rs.git", rev = "06f42db5344f4ad8560af341352572488a4e6e06" } -# deltalake-catalog-unity = { git = "https://github.com/ryzhyk/delta-rs.git", rev = "06f42db5344f4ad8560af341352572488a4e6e06" } delta_kernel = "0.19.0" derive_more = { version = "1.0.0" } directories = "6.0" @@ -153,10 +159,11 @@ hashbrown = "0.14.2" hdrhist = "0.5" hex = "0.4.3" home = "=0.5.9" -iceberg = { version = "0.8.0", features = ["storage-all"]} -iceberg-catalog-glue = "0.8.0" -iceberg-catalog-rest = "0.8.0" -iceberg-datafusion = "0.8.0" +iceberg = { version = "0.9.1" } +iceberg-catalog-glue = "0.9.1" +iceberg-catalog-rest = "0.9.1" +iceberg-datafusion = "0.9.1" +iceberg-storage-opendal = { version = "0.9.1", features = ["opendal-s3", "opendal-gcs", "opendal-fs", "opendal-memory"] } impl-trait-for-tuples = "0.2" indexmap = "2.7.1" indicatif = "0.17.0-rc.11" @@ -185,7 +192,7 @@ num-bigint = "0.4.6" num-derive = "0.4.2" num-format = "0.4.0" num-traits = "0.2.19" -object_store = "0.11.2" +object_store = "0.12.1" once_cell = "1.20.2" openssl = "0.10.79" ordered-float = { version = "4.2.0", features = ["serde"] } @@ -304,3 +311,4 @@ debug = true [profile.release] debug = "line-tables-only" + diff --git a/crates/adapters/src/integrated/delta_table/input.rs b/crates/adapters/src/integrated/delta_table/input.rs index 31eecfdd6ed..34bc057de4c 100644 --- a/crates/adapters/src/integrated/delta_table/input.rs +++ b/crates/adapters/src/integrated/delta_table/input.rs @@ -7,7 +7,6 @@ use crate::{ControllerError, InputConsumer, InputReader, PipelineState}; use anyhow::{Error as AnyError, Result as AnyResult, anyhow, bail}; use arrow::array::BooleanArray; use chrono::{DateTime, Utc}; -use datafusion::catalog::TableProvider; use datafusion::common::arrow::array::RecordBatch; use datafusion::datasource::file_format::parquet::ParquetFormat; use datafusion::datasource::listing::{ @@ -54,7 +53,6 @@ use tokio::sync::watch::{Receiver, Sender, channel}; use tokio::sync::{Semaphore, mpsc}; use tokio::time::sleep; use tracing::{debug, info, trace, warn}; -use url::Url; /// Polling interval when following a delta table. const POLL_INTERVAL: Duration = Duration::from_millis(1000); @@ -798,10 +796,12 @@ impl DeltaTableInputEndpointInner { .collect::>() }; - let table_schema = table.schema(); + let table_schema = table + .snapshot() + .expect("Delta table snapshot must be loaded before computing used columns") + .schema(); let delta_columns = table_schema .fields() - .iter() .map(|f| f.name().to_string()) .collect::>(); @@ -1517,16 +1517,23 @@ impl DeltaTableInputEndpointInner { DeltaResumeInfo::follow_mode(version, false); } - // Register object store with datafusion, so it will recognize individual parquet - // file URIs when processing transaction log. The `object_store_url` function - // generates a unique URL, which only makes sense to datafusion. We must append - // the same string to the relative file path we read from the log below to make - // sure datafusion links it to this object store. - let object_store_url = delta_table.log_store().object_store_url(); - let url: &Url = object_store_url.as_ref(); - self.datafusion - .runtime_env() - .register_object_store(url, delta_table.log_store().object_store(None)); + // Register object store & url pairs with datafusion. + // + // - Synthetic `delta-rs://...` URL + table-prefixed store: used when + // we build parquet URIs from `Add.path` entries in the transaction + // log (see `process_actions` and the listing-table path). + // - Table root URL + bucket-root store: used by the snapshot + // `TableProvider` since delta-rs 0.31.x, which plans scans against + // the root URL via datafusion's standard parquet `FileSource`. + // Without this second registration, `select ... from snapshot` fails + // with "No suitable object store found for ". + let log_store = delta_table.log_store(); + let runtime_env = self.datafusion.runtime_env(); + runtime_env.register_object_store( + log_store.object_store_url().as_ref(), + log_store.object_store(None), + ); + runtime_env.register_object_store(log_store.root_url(), log_store.root_object_store(None)); // if let Some(schema) = delta_table.schema() { // info!("Delta table schema: {schema:?}"); @@ -1554,8 +1561,15 @@ impl DeltaTableInputEndpointInner { &self.endpoint_name, ); + let provider = table.table_provider().await.map_err(|e| { + ControllerError::input_transport_error( + &self.endpoint_name, + true, + anyhow!("failed to build Delta table provider for snapshot: {e}"), + ) + })?; self.datafusion - .register_table("snapshot", table.clone()) + .register_table("snapshot", provider) .map_err(|e| { ControllerError::input_transport_error( &self.endpoint_name, @@ -2339,7 +2353,14 @@ impl DeltaTableInputEndpointInner { files: Vec, description: &str, ) -> AnyResult { - let schema = table.schema(); + // `DeltaTable::snapshot()` returns the table state; calling `.snapshot()` + // on that state hands back the eager snapshot, which exposes the Arrow + // schema. + let schema = table + .snapshot() + .map_err(|e| anyhow!("error accessing Delta table snapshot: {e}"))? + .snapshot() + .arrow_schema(); let mut urls = Vec::with_capacity(files.len()); for file in files.iter() { diff --git a/crates/adapters/src/integrated/delta_table/test.rs b/crates/adapters/src/integrated/delta_table/test.rs index 68907335c5e..27da66ff486 100644 --- a/crates/adapters/src/integrated/delta_table/test.rs +++ b/crates/adapters/src/integrated/delta_table/test.rs @@ -121,8 +121,9 @@ async fn wait_for_output_records( .await .unwrap(); + let provider = table.table_provider().await.unwrap(); let data = datafusion - .read_table(table.clone()) + .read_table(provider) .unwrap() .collect() .await @@ -1590,6 +1591,7 @@ async fn delta_table_cdc_rewrite_test() { }; use deltalake::datafusion::prelude::{col, lit}; use deltalake::kernel::{DataType as KernelDataType, PrimitiveType, StructField}; + use std::num::NonZeroU64; init_logging(); @@ -1728,7 +1730,7 @@ async fn delta_table_cdc_rewrite_test() { let v_before = delta.version().unwrap(); let (optimized, _) = delta .optimize() - .with_target_size(128 * 1024 * 1024) + .with_target_size(NonZeroU64::new(128 * 1024 * 1024).unwrap()) .await .unwrap(); assert!( diff --git a/crates/iceberg/Cargo.toml b/crates/iceberg/Cargo.toml index 691498b8b2a..59f35f31f74 100644 --- a/crates/iceberg/Cargo.toml +++ b/crates/iceberg/Cargo.toml @@ -22,6 +22,8 @@ iceberg = { workspace = true } iceberg-datafusion = { workspace = true } iceberg-catalog-glue = { workspace = true } iceberg-catalog-rest = { workspace = true } +iceberg-storage-opendal = { workspace = true } +url = { workspace = true } chrono = { workspace = true } serde_json = { workspace = true } futures-util = { workspace = true } diff --git a/crates/iceberg/src/input.rs b/crates/iceberg/src/input.rs index ea0fea0cb1a..488a78ddd6b 100644 --- a/crates/iceberg/src/input.rs +++ b/crates/iceberg/src/input.rs @@ -24,7 +24,12 @@ use feldera_types::{ }; use futures_util::StreamExt; use iceberg::CatalogBuilder; -use iceberg::{io::FileIO, spec::TableMetadata, table::Table as IcebergTable, Catalog, TableIdent}; +use iceberg::{ + io::{FileIOBuilder, StorageFactory}, + spec::TableMetadata, + table::Table as IcebergTable, + Catalog, TableIdent, +}; use iceberg_catalog_glue::{ GlueCatalogBuilder, AWS_ACCESS_KEY_ID, AWS_PROFILE_NAME, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, GLUE_CATALOG_PROP_CATALOG_ID, GLUE_CATALOG_PROP_URI, @@ -34,6 +39,7 @@ use iceberg_catalog_rest::{ RestCatalogBuilder, REST_CATALOG_PROP_URI, REST_CATALOG_PROP_WAREHOUSE, }; use iceberg_datafusion::IcebergStaticTableProvider; +use iceberg_storage_opendal::OpenDalStorageFactory; use log::{debug, info, trace}; use std::{sync::Arc, thread}; use tokio::{ @@ -44,6 +50,31 @@ use tokio::{ }, }; +/// Pick an iceberg `StorageFactory` based on a location URL's scheme. +/// +/// Iceberg 0.9 removed scheme-aware FileIO construction from the core crate; +/// callers must now inject a `StorageFactory` explicitly. We map the URL scheme +/// to one of the OpenDAL-backed factories shipped by `iceberg-storage-opendal`. +fn storage_factory_for_url(location: &str) -> AnyResult> { + // `url::Url::parse` returns a scheme that is already lowercased per RFC 3986, + // so no extra normalization is needed. A parse failure (e.g. a bare local + // path with no scheme) falls through to the local-filesystem factory. + let scheme = url::Url::parse(location) + .map(|u| u.scheme().to_string()) + .unwrap_or_else(|_| "file".to_string()); + let factory: Arc = match scheme.as_str() { + "file" => Arc::new(OpenDalStorageFactory::Fs), + "memory" => Arc::new(OpenDalStorageFactory::Memory), + "s3" | "s3a" => Arc::new(OpenDalStorageFactory::S3 { + configured_scheme: scheme, + customized_credential_load: None, + }), + "gs" | "gcs" => Arc::new(OpenDalStorageFactory::Gcs), + other => bail!("unsupported storage scheme '{other}' in location '{location}'"), + }; + Ok(factory) +} + enum SnapshotDescr { /// Open the latest snapshot (default) Latest, @@ -466,21 +497,15 @@ impl IcebergInputEndpointInner { // Safe due to checks in 'validate_catalog_config'. let metadata_location = self.config.metadata_location.as_ref().unwrap(); - let file_io = FileIO::from_path(metadata_location) - .map_err(|e| { - ControllerError::invalid_transport_configuration( - &self.endpoint_name, - &format!("invalid 'metadata_location' value: {e}"), - ) - })? + let factory = storage_factory_for_url(metadata_location).map_err(|e| { + ControllerError::invalid_transport_configuration( + &self.endpoint_name, + &format!("invalid 'metadata_location' value: {e}"), + ) + })?; + let file_io = FileIOBuilder::new(factory) .with_props(&self.config.fileio_config) - .build() - .map_err(|e| { - ControllerError::invalid_transport_configuration( - &self.endpoint_name, - &format!("invalid storage configuration: {e}"), - ) - })?; + .build(); let metadata_file = file_io.new_input(metadata_location).map_err(|e| { ControllerError::invalid_transport_configuration( @@ -666,7 +691,25 @@ impl IcebergInputEndpointInner { } }; + // iceberg 0.9.x's REST catalog requires an explicit StorageFactory. + // Pick one from the configured warehouse URL when available; fall + // back to the same S3 default that iceberg-catalog-glue uses. + let factory: Arc = + match self.config.rest_catalog_config.warehouse.as_deref() { + Some(warehouse) => storage_factory_for_url(warehouse).map_err(|e| { + ControllerError::invalid_transport_configuration( + &self.endpoint_name, + &format!("invalid 'warehouse' value: {e}"), + ) + })?, + None => Arc::new(OpenDalStorageFactory::S3 { + configured_scheme: "s3".to_string(), + customized_credential_load: None, + }), + }; + let catalog = RestCatalogBuilder::default() + .with_storage_factory(factory) .load("rest".to_string(), props) .await .map_err(|e| { @@ -896,3 +939,58 @@ async fn wait_running(receiver: &mut Receiver) { .wait_for(|state| state == &PipelineState::Running) .await; } + +#[cfg(test)] +mod tests { + use super::*; + + fn factory_kind(f: &Arc) -> &'static str { + // `OpenDalStorageFactory` is `#[non_exhaustive]` and the trait object + // hides the variant, so use the `Debug` impl to identify it. + let dbg = format!("{f:?}"); + if dbg.starts_with("Fs") { + "Fs" + } else if dbg.starts_with("Memory") { + "Memory" + } else if dbg.starts_with("S3") { + "S3" + } else if dbg.starts_with("Gcs") { + "Gcs" + } else { + panic!("unexpected factory: {dbg}"); + } + } + + #[test] + fn storage_factory_for_url_picks_expected_backend() { + for (location, expected) in [ + ("file:///tmp/warehouse/metadata.json", "Fs"), + ("memory://warehouse/metadata.json", "Memory"), + ("s3://bucket/path/metadata.json", "S3"), + ("s3a://bucket/path/metadata.json", "S3"), + ("gs://bucket/path/metadata.json", "Gcs"), + ("gcs://bucket/path/metadata.json", "Gcs"), + ] { + let f = storage_factory_for_url(location) + .unwrap_or_else(|e| panic!("expected ok for {location}, got: {e}")); + assert_eq!(factory_kind(&f), expected, "wrong factory for {location}"); + } + } + + #[test] + fn storage_factory_for_url_falls_back_to_fs_when_url_is_unparseable() { + // No scheme — `url::Url::parse` fails, so we land on the local-fs default. + let f = storage_factory_for_url("/tmp/warehouse/metadata.json").unwrap(); + assert_eq!(factory_kind(&f), "Fs"); + } + + #[test] + fn storage_factory_for_url_rejects_unsupported_scheme() { + let err = storage_factory_for_url("abfss://container/path").unwrap_err(); + let msg = err.to_string(); + assert!( + msg.contains("unsupported storage scheme") && msg.contains("abfss"), + "unexpected error message: {msg}" + ); + } +}