diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c1bbb47e6..83ad1298a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,7 +8,7 @@ on: env: # NOTE: the version is also defined in roapi_http_release.yml, columnq_cli_release.yml and Dockerfile - RUST_TC_NIGHTLY_VER: "2023-09-15" + RUST_TC_NIGHTLY_VER: "2024-03-01" jobs: build: @@ -17,6 +17,7 @@ jobs: RUSTFLAGS: "-C target-cpu=skylake" steps: - uses: actions/checkout@v2 + - uses: rui314/setup-mold@v1 - uses: actions/cache@v2 with: path: | @@ -40,21 +41,21 @@ jobs: - name: Format run: cargo fmt --check - name: Build - run: cargo build + run: mold -run cargo build - name: Run tests - run: | - cargo test + run: mold -run cargo test - name: Trim cache run: | which cargo-cache || cargo install cargo-cache cargo cache trim -l 1G - simd_test: + database_test: runs-on: ubuntu-latest env: RUSTFLAGS: "-C target-cpu=skylake" steps: - uses: actions/checkout@v2 + - uses: rui314/setup-mold@v1 - uses: actions/cache@v2 with: path: | @@ -63,67 +64,34 @@ jobs: ~/.cargo/registry/cache/ ~/.cargo/git/db/ target/ - key: simd-3-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }} + key: database-1-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }} restore-keys: | - simd-3-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }} - simd-3-${{ runner.os }}-cargo- + database-1-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }} + database-1-${{ runner.os }}-cargo- - name: Install nightly rust uses: actions-rs/toolchain@v1 with: profile: default - # toolchain: nightly toolchain: nightly-${{ env.RUST_TC_NIGHTLY_VER }} override: true + - name: Check + run: cargo clippy --features database + - name: Build + run: mold -run cargo build --features database - name: Run tests - run: | - cargo test --features simd + run: mold -run cargo test --features database - name: Trim cache run: | which cargo-cache || cargo install cargo-cache cargo cache trim -l 1G - database_test: - runs-on: ubuntu-latest - env: - RUSTFLAGS: "-C target-cpu=skylake" - steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v2 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: database-1-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }} - restore-keys: | - database-1-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }} - database-1-${{ runner.os }}-cargo- - - name: Install nightly rust - uses: actions-rs/toolchain@v1 - with: - profile: default - toolchain: nightly-${{ env.RUST_TC_NIGHTLY_VER }} - override: true - - name: Check - run: cargo clippy --features database - - name: Build - run: cargo build --features database - - name: Run tests - run: | - cargo test --features database - - name: Trim cache - run: | - which cargo-cache || cargo install cargo-cache - cargo cache trim -l 1G - object_store_memory_test: runs-on: ubuntu-latest env: RUSTFLAGS: "-C target-cpu=skylake" steps: - uses: actions/checkout@v2 + - uses: rui314/setup-mold@v1 - uses: actions/cache@v2 with: path: | @@ -171,8 +139,8 @@ jobs: uses: BerniWittmann/background-server-action@v1.0.4 with: command: /home/runner/work/roapi/roapi/test_end_to_end/query_blogs.sh - build: cargo build - start: cargo run --bin roapi -- -c test_end_to_end/test_object_store_memory.yml + build: mold -run cargo build + start: mold -run cargo run --bin roapi -- -c test_end_to_end/test_object_store_memory.yml wait-on: "http://127.0.0.1:8000/api/schema" # By default, wait-on will retry for 60 seconds. You can pass a custom timeout in seconds using wait-on-timeout. # 10 minutes = 600 seconds @@ -184,6 +152,7 @@ jobs: RUSTFLAGS: "-C target-cpu=skylake" steps: - uses: actions/checkout@v2 + - uses: rui314/setup-mold@v1 - uses: actions/cache@v2 with: path: | @@ -231,8 +200,8 @@ jobs: uses: BerniWittmann/background-server-action@v1.0.4 with: command: /home/runner/work/roapi/roapi/test_end_to_end/query_blogs.sh - build: cargo build - start: cargo run --bin roapi -- -c test_end_to_end/test_object_store_direct.yml + build: mold -run cargo build + start: mold -run cargo run --bin roapi -- -c test_end_to_end/test_object_store_direct.yml wait-on: "http://127.0.0.1:8000/api/schema" # By default, wait-on will retry for 60 seconds. You can pass a custom timeout in seconds using wait-on-timeout. # 10 minutes = 600 seconds @@ -300,13 +269,17 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Build Docker image - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 with: context: . push: false load: true tags: roapi:latest + cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/roapi:buildcache - name: Test run: | docker run --rm roapi:latest --help + docker run --rm roapi:latest --version diff --git a/.github/workflows/columnq_cli_release.yml b/.github/workflows/columnq_cli_release.yml index 42673ae1b..035d5bea3 100644 --- a/.github/workflows/columnq_cli_release.yml +++ b/.github/workflows/columnq_cli_release.yml @@ -12,7 +12,7 @@ on: env: # NOTE: the version is also defined in build.yml and Dockerfile - RUST_TC_NIGHTLY_VER: "2023-09-15" + RUST_TC_NIGHTLY_VER: "2024-03-01" jobs: # skip tag version validation on non-release branch run @@ -51,7 +51,7 @@ jobs: run: pip3 install 'maturin<0.12' - name: Build wheels - x86_64 run: | - maturin build -m columnq-cli/Cargo.toml -b bin --target x86_64-apple-darwin --release --out dist --cargo-extra-args="--features=simd,database-sqlite" + maturin build -m columnq-cli/Cargo.toml -b bin --target x86_64-apple-darwin --release --out dist --cargo-extra-args="--features=database-sqlite" pip install columnq-cli --no-index --find-links dist --force-reinstall - name: Build wheels - universal2 env: @@ -60,7 +60,7 @@ jobs: run: | # set SDKROOT for C dependencies export SDKROOT=$(xcrun --sdk macosx --show-sdk-path) - maturin build -m columnq-cli/Cargo.toml -b bin --release --universal2 --out dist --no-sdist --cargo-extra-args="--features=simd,database-sqlite" + maturin build -m columnq-cli/Cargo.toml -b bin --release --universal2 --out dist --no-sdist --cargo-extra-args="--features=database-sqlite" pip install columnq-cli --no-index --find-links dist --force-reinstall columnq --help - name: Upload wheels @@ -89,7 +89,7 @@ jobs: platform: - python-architecture: "x64" target: "x86_64-pc-windows-msvc" - features: "simd,database-sqlite" + features: "database-sqlite" steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 @@ -138,14 +138,14 @@ jobs: - manylinux: "2010" target: "x86_64-unknown-linux-musl" image_tag: "x86_64-musl" - features: "simd,rustls,database-sqlite" + features: "rustls,database-sqlite" rustflags: "-C target-cpu=skylake" name_suffix: "" upload: "true" # - manylinux: "2010" # target: "x86_64-unknown-linux-musl" # image_tag: "x86_64-musl" - # features: "simd,native-tls,database-sqlite" + # features: "native-tls,database-sqlite" # rustflags: "-C target-cpu=skylake" # name_suffix: "-openssl" # upload: "false" diff --git a/.github/workflows/roapi_release.yml b/.github/workflows/roapi_release.yml index 474c5f374..030f4c968 100644 --- a/.github/workflows/roapi_release.yml +++ b/.github/workflows/roapi_release.yml @@ -12,7 +12,7 @@ on: env: # NOTE: the version is also defined in build.yml and Dockerfile - RUST_TC_NIGHTLY_VER: "2023-09-15" + RUST_TC_NIGHTLY_VER: "2024-03-01" jobs: validate-release-tag: @@ -53,7 +53,7 @@ jobs: run: pip3 install 'maturin<0.12' - name: Build wheels - x86_64 run: | - maturin build -m roapi/Cargo.toml -b bin --target x86_64-apple-darwin --release --out dist --cargo-extra-args="--features=simd,database-sqlite" + maturin build -m roapi/Cargo.toml -b bin --target x86_64-apple-darwin --release --out dist --cargo-extra-args="--features=database-sqlite" pip install roapi --no-index --find-links dist --force-reinstall - name: Build wheels - universal2 env: @@ -62,7 +62,7 @@ jobs: run: | # set SDKROOT for C dependencies export SDKROOT=$(xcrun --sdk macosx --show-sdk-path) - maturin build -m roapi/Cargo.toml -b bin --release --universal2 --out dist --no-sdist --cargo-extra-args="--features=simd,database-sqlite" + maturin build -m roapi/Cargo.toml -b bin --release --universal2 --out dist --no-sdist --cargo-extra-args="--features=database-sqlite" pip install roapi --no-index --find-links dist --force-reinstall - name: Upload wheels uses: actions/upload-artifact@v2 @@ -88,7 +88,7 @@ jobs: strategy: matrix: platform: [ - { python-architecture: "x64", target: "x86_64-pc-windows-msvc", features: "simd,database-sqlite" }, + { python-architecture: "x64", target: "x86_64-pc-windows-msvc", features: "database-sqlite" }, # { python-architecture: "x86", target: "i686-pc-windows-msvc", features: "" }, ] steps: @@ -140,14 +140,14 @@ jobs: - manylinux: "2010" target: "x86_64-unknown-linux-musl" image_tag: "x86_64-musl" - features: "simd,rustls,database-sqlite" + features: "rustls,database-sqlite" name_suffix: "" rustflags: "-C target-cpu=skylake" upload: "true" # - manylinux: "2010" # target: "x86_64-unknown-linux-musl" # image_tag: "x86_64-musl" - # features: "simd,native-tls" + # features: "native-tls" # rustflags: "-C target-cpu=skylake" # name_suffix: "-openssl" # upload: "false" @@ -279,3 +279,5 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/roapi:buildcache + cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/roapi:buildcache,mode=max diff --git a/Cargo.lock b/Cargo.lock index 321a76ee6..4069fbc0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,9 +19,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.7.6" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ "getrandom", "once_cell", @@ -30,22 +30,23 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", "const-random", "getrandom", "once_cell", "version_check", + "zerocopy", ] [[package]] name = "aho-corasick" -version = "1.0.5" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] @@ -86,11 +87,59 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" + +[[package]] +name = "anstyle-parse" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" [[package]] name = "arrayref" @@ -106,11 +155,10 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "46.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04a8801ebb147ad240b2d978d3ab9f73c9ccd4557ba6a03e7800496770ed10e0" +checksum = "aa285343fba4d829d49985bdc541e3789cf6000ed0e84be7c039438df4a4e78c" dependencies = [ - "ahash 0.8.3", "arrow-arith", "arrow-array", "arrow-buffer", @@ -128,9 +176,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "46.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "895263144bd4a69751cbe6a34a53f26626e19770b313a9fa792c415cd0e78f11" +checksum = "753abd0a5290c1bcade7c6623a556f7d1659c5f4148b140b5b63ce7bd1a45705" dependencies = [ "arrow-array", "arrow-buffer", @@ -143,27 +191,26 @@ dependencies = [ [[package]] name = "arrow-array" -version = "46.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "226fdc6c3a4ae154a74c24091d36a90b514f0ed7112f5b8322c1d8f354d8e20d" +checksum = "d390feeb7f21b78ec997a4081a025baef1e2e0d6069e181939b61864c9779609" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.11", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", "chrono-tz", "half", - "hashbrown 0.14.0", + "hashbrown 0.14.3", "num", - "packed_simd", ] [[package]] name = "arrow-buffer" -version = "46.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc4843af4dd679c2f35b69c572874da8fde33be53eb549a5fb128e7a4b763510" +checksum = "69615b061701bcdffbc62756bc7e85c827d5290b472b580c972ebbbf690f5aa4" dependencies = [ "bytes", "half", @@ -172,15 +219,16 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "46.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e8b9990733a9b635f656efda3c9b8308c7a19695c9ec2c7046dd154f9b144b" +checksum = "e448e5dd2f4113bf5b74a1f26531708f5edcacc77335b7066f9398f4bcf4cdef" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", + "base64 0.21.7", "chrono", "comfy-table", "half", @@ -190,9 +238,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "46.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "646fbb4e11dd0afb8083e883f53117713b8caadb4413b3c9e63e3f535da3683c" +checksum = "46af72211f0712612f5b18325530b9ad1bfbdc87290d5fbfd32a7da128983781" dependencies = [ "arrow-array", "arrow-buffer", @@ -209,9 +257,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "46.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da900f31ff01a0a84da0572209be72b2b6f980f3ea58803635de47913191c188" +checksum = "67d644b91a162f3ad3135ce1184d0a31c28b816a581e08f29e8e9277a574c64e" dependencies = [ "arrow-buffer", "arrow-schema", @@ -221,9 +269,9 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "46.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e40372d37a860a742f248d4d7c137950cd793f1b46f2b99a5116c55efbe2699f" +checksum = "1d7f215461ad6346f2e4cc853e377d4e076d533e1ed78d327debe83023e3601f" dependencies = [ "arrow-arith", "arrow-array", @@ -236,7 +284,7 @@ dependencies = [ "arrow-schema", "arrow-select", "arrow-string", - "base64 0.21.4", + "base64 0.21.7", "bytes", "futures", "once_cell", @@ -248,9 +296,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "46.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2707a8d7ee2d345d045283ece3ae43416175873483e5d96319c929da542a0b1f" +checksum = "03dea5e79b48de6c2e04f03f62b0afea7105be7b77d134f6c5414868feefb80d" dependencies = [ "arrow-array", "arrow-buffer", @@ -258,13 +306,14 @@ dependencies = [ "arrow-data", "arrow-schema", "flatbuffers", + "lz4_flex", ] [[package]] name = "arrow-json" -version = "46.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d1b91a63c356d14eedc778b76d66a88f35ac8498426bb0799a769a49a74a8b4" +checksum = "8950719280397a47d37ac01492e3506a8a724b3fb81001900b866637a829ee0f" dependencies = [ "arrow-array", "arrow-buffer", @@ -273,7 +322,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.0.0", + "indexmap 2.2.5", "lexical-core", "num", "serde", @@ -282,9 +331,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "46.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "584325c91293abbca7aaaabf8da9fe303245d641f5f4a18a6058dc68009c7ebf" +checksum = "1ed9630979034077982d8e74a942b7ac228f33dd93a93b615b4d02ad60c260be" dependencies = [ "arrow-array", "arrow-buffer", @@ -297,35 +346,36 @@ dependencies = [ [[package]] name = "arrow-row" -version = "46.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e32afc1329f7b372463b21c6ca502b07cf237e1ed420d87706c1770bb0ebd38" +checksum = "007035e17ae09c4e8993e4cb8b5b96edf0afb927cd38e2dff27189b274d83dcf" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.11", "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "half", - "hashbrown 0.14.0", + "hashbrown 0.14.3", ] [[package]] name = "arrow-schema" -version = "46.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b104f5daa730f00fde22adc03a12aa5a2ae9ccbbf99cbd53d284119ddc90e03d" +checksum = "0ff3e9c01f7cd169379d269f926892d0e622a704960350d09d331be3ec9e0029" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.5.0", "serde", ] [[package]] name = "arrow-select" -version = "46.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73b3ca55356d1eae07cf48808d8c462cea674393ae6ad1e0b120f40b422eb2b4" +checksum = "1ce20973c1912de6514348e064829e50947e35977bb9d7fb637dc99ea9ffd78c" dependencies = [ + "ahash 0.8.11", "arrow-array", "arrow-buffer", "arrow-data", @@ -335,9 +385,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "46.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af1433ce02590cae68da0a18ed3a3ed868ffac2c6f24c533ddd2067f7ee04b4a" +checksum = "00f3b37f2aeece31a2636d1b037dabb69ef590e03bdc7eb68519b51ec86932a7" dependencies = [ "arrow-array", "arrow-buffer", @@ -357,20 +407,22 @@ checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" [[package]] name = "async-channel" -version = "1.9.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +checksum = "f28243a43d821d11341ab73c80bed182dc015c514b951616cf79bd4af39af0c3" dependencies = [ "concurrent-queue", - "event-listener", + "event-listener 5.2.0", + "event-listener-strategy 0.5.0", "futures-core", + "pin-project-lite", ] [[package]] name = "async-compression" -version = "0.4.3" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb42b2197bf15ccb092b62c74515dbd8b86d0effd934795f6687c93b6e679a2c" +checksum = "a116f46a969224200a0a97f29cfd4c50e7534e4b4826bd23ea2c3c533039c82c" dependencies = [ "bzip2", "flate2", @@ -390,45 +442,92 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" dependencies = [ - "async-lock", + "async-lock 2.8.0", "autocfg", "cfg-if", "concurrent-queue", - "futures-lite", + "futures-lite 1.13.0", "log", "parking", - "polling", - "rustix 0.37.23", + "polling 2.8.0", + "rustix 0.37.27", "slab", - "socket2 0.4.9", + "socket2 0.4.10", "waker-fn", ] +[[package]] +name = "async-io" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcccb0f599cfa2f8ace422d3555572f47424da5648a4382a9dd0310ff8210884" +dependencies = [ + "async-lock 3.3.0", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite 2.3.0", + "parking", + "polling 3.5.0", + "rustix 0.38.32", + "slab", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "async-lock" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" dependencies = [ - "event-listener", + "event-listener 2.5.3", +] + +[[package]] +name = "async-lock" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" +dependencies = [ + "event-listener 4.0.3", + "event-listener-strategy 0.4.0", + "pin-project-lite", ] [[package]] name = "async-process" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a9d28b1d97e08915212e2e45310d47854eafa69600756fc735fb788f75199c9" +checksum = "ea6438ba0a08d81529c69b36700fa2f95837bfe3e776ab39cde9c14d9149da88" dependencies = [ - "async-io", - "async-lock", - "autocfg", + "async-io 1.13.0", + "async-lock 2.8.0", + "async-signal", "blocking", "cfg-if", - "event-listener", - "futures-lite", - "rustix 0.37.23", - "signal-hook", - "windows-sys", + "event-listener 3.1.0", + "futures-lite 1.13.0", + "rustix 0.38.32", + "windows-sys 0.48.0", +] + +[[package]] +name = "async-signal" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e47d90f65a225c4527103a8d747001fc56e375203592b25ad103e1ca13124c5" +dependencies = [ + "async-io 2.3.2", + "async-lock 2.8.0", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix 0.38.32", + "signal-hook-registry", + "slab", + "windows-sys 0.48.0", ] [[package]] @@ -450,42 +549,31 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.53", ] [[package]] name = "async-task" -version = "4.4.0" +version = "4.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc7ab41815b3c653ccd2978ec3255c81349336702dfdf62ee6f7069b12a3aae" +checksum = "fbb36e985947064623dbd357f727af08ffd077f93d696782f3c56365fa2e2799" [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "461abc97219de0eaaf81fe3ef974a540158f3d079c2ab200f891f1a2ef201e85" dependencies = [ "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.53", ] [[package]] name = "atomic-waker" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1181e1e0d1fce796a03db1ae795d67167da795f9cf4a39c37589e85ef57f26d3" - -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" @@ -542,11 +630,25 @@ dependencies = [ "tower-service", ] +[[package]] +name = "backoff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "futures-core", + "getrandom", + "instant", + "pin-project-lite", + "rand", + "tokio", +] + [[package]] name = "backtrace" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +checksum = "95d8e92cac0961e91dbd517496b00f7e9b92363dbe6d42c3198268323798860c" dependencies = [ "addr2line", "cc", @@ -565,9 +667,15 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.4" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" [[package]] name = "bigdecimal" @@ -607,9 +715,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" [[package]] name = "bitvec" @@ -634,16 +742,15 @@ dependencies = [ [[package]] name = "blake3" -version = "1.4.1" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "199c42ab6972d92c9f8995f086273d25c42fc0f7b2a1fcefba465c1352d25ba5" +checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "digest 0.10.7", ] [[package]] @@ -666,69 +773,49 @@ dependencies = [ [[package]] name = "blocking" -version = "1.3.1" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77231a1c8f801696fc0123ec6150ce92cffb8e164a02afb9c8ddee0e9b65ad65" +checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" dependencies = [ "async-channel", - "async-lock", + "async-lock 3.3.0", "async-task", - "atomic-waker", - "fastrand 1.9.0", - "futures-lite", - "log", + "fastrand 2.0.1", + "futures-io", + "futures-lite 2.3.0", + "piper", + "tracing", ] [[package]] name = "borsh" -version = "0.10.3" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4114279215a005bc675e386011e594e1d9b800918cea18fcadadcce864a2046b" +checksum = "f58b559fd6448c6e2fd0adb5720cd98a2506594cafa4737ff98c396f3e82f667" dependencies = [ "borsh-derive", - "hashbrown 0.13.2", + "cfg_aliases", ] [[package]] name = "borsh-derive" -version = "0.10.3" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0754613691538d51f329cce9af41d7b7ca150bc973056f1156611489475f54f7" +checksum = "7aadb5b6ccbd078890f6d7003694e33816e6b784358f18e15e7e6d9f065a57cd" dependencies = [ - "borsh-derive-internal", - "borsh-schema-derive-internal", + "once_cell", "proc-macro-crate", - "proc-macro2", - "syn 1.0.109", -] - -[[package]] -name = "borsh-derive-internal" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afb438156919598d2c7bad7e1c0adf3d26ed3840dbc010db1a882a65583ca2fb" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "borsh-schema-derive-internal" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634205cc43f74a1b9046ef87c4540ebda95696ec0f315024860cad7c5b0f5ccd" -dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.53", + "syn_derive", ] [[package]] name = "brotli" -version = "3.3.4" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -737,9 +824,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.3.4" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744" +checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -753,15 +840,15 @@ checksum = "40e38929add23cdf8a366df9b0e088953150724bcbe5fc330b0d8eb3b328eec8" [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" [[package]] name = "bytecheck" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6372023ac861f6e6dc89c8344a8f398fb42aaba2b5dbc649ca0c0e9dbcb627" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" dependencies = [ "bytecheck_derive", "ptr_meta", @@ -770,20 +857,26 @@ dependencies = [ [[package]] name = "bytecheck_derive" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7ec4c6f261935ad534c0c22dbef2201b45918860eb1c574b972bd213a76af61" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" dependencies = [ "proc-macro2", "quote", "syn 1.0.109", ] +[[package]] +name = "bytemuck" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d6d68c57235a3a081186990eca2867354726650f42f7516ca50c28d6281fd15" + [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" @@ -814,24 +907,26 @@ dependencies = [ [[package]] name = "calamine" -version = "0.19.1" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6381d1037ee9b8a6c8eb97936add0331a1aabd148d5b6f35f1cda6e5dec44f40" +checksum = "47a4d6ea525ea187df1e3a1c4b23469b1cbe60c5bafc1c0ef14b2b8738a8303d" dependencies = [ "byteorder", + "chrono", "codepage", "encoding_rs", "log", - "quick-xml 0.25.0", + "once_cell", + "quick-xml", "serde", "zip", ] [[package]] name = "cc" -version = "1.0.83" +version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" dependencies = [ "jobserver", "libc", @@ -852,11 +947,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" dependencies = [ "android-tzdata", "iana-time-zone", @@ -864,14 +965,14 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets", + "windows-targets 0.52.4", ] [[package]] name = "chrono-tz" -version = "0.8.3" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1369bc6b9e9a7dfdae2055f6ec151fe9c554a9d23d357c0237cee2e25eaabb7" +checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e" dependencies = [ "chrono", "chrono-tz-build", @@ -880,9 +981,9 @@ dependencies = [ [[package]] name = "chrono-tz-build" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2f5ebdc942f57ed96d560a6d1a459bae5851102a25d5bf89dc04ae453e31ecf" +checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f" dependencies = [ "parse-zoneinfo", "phf", @@ -891,9 +992,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" +checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" dependencies = [ "glob", "libc", @@ -902,27 +1003,30 @@ dependencies = [ [[package]] name = "clap" -version = "3.2.25" +version = "4.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +checksum = "949626d00e063efc93b6dca932419ceb5432f99769911c0b995f7e884c778813" dependencies = [ - "atty", - "bitflags 1.3.2", + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" +dependencies = [ + "anstream", + "anstyle", "clap_lex", - "indexmap 1.9.3", "strsim", - "termcolor", - "textwrap", ] [[package]] name = "clap_lex" -version = "0.2.4" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" -dependencies = [ - "os_str_bytes", -] +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" [[package]] name = "clipboard-win" @@ -953,9 +1057,15 @@ dependencies = [ "encoding_rs", ] +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + [[package]] name = "columnq" -version = "0.7.0" +version = "0.8.1" dependencies = [ "arrow-schema", "bytes", @@ -966,7 +1076,7 @@ dependencies = [ "dotenvy", "futures", "graphql-parser", - "hyper-rustls 0.23.2", + "hyper-rustls 0.24.2", "hyper-tls", "lazy_static", "log", @@ -980,13 +1090,13 @@ dependencies = [ "serde_derive", "serde_json", "serde_yaml", - "snafu", - "sqlparser", + "snafu 0.8.2", + "sqlparser 0.41.0", "tempfile", "thiserror", "tokio", "tokio-postgres", - "toml 0.7.8", + "toml", "uriparse", "url", "yup-oauth2", @@ -1024,20 +1134,20 @@ dependencies = [ [[package]] name = "comfy-table" -version = "7.0.1" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab77dbd8adecaf3f0db40581631b995f312a8a5ae3aa9993188bb8f23d83a5b" +checksum = "7c64043d6c7b7a4c58e39e7efccfdea7b93d885a795d0c054a69dbbf4dd52686" dependencies = [ - "strum 0.24.1", - "strum_macros 0.24.3", + "strum", + "strum_macros", "unicode-width", ] [[package]] name = "concurrent-queue" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" +checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" dependencies = [ "crossbeam-utils", ] @@ -1045,7 +1155,7 @@ dependencies = [ [[package]] name = "connectorx" version = "0.3.3-alpha.1" -source = "git+https://github.com/roapi/connector-x.git?rev=1e642af502d3076476ab4bc8c7e40807059a42fe#1e642af502d3076476ab4bc8c7e40807059a42fe" +source = "git+https://github.com/roapi/connector-x.git?rev=7cfe911133afd56a5fa69b24d11f8b5f7283796c#7cfe911133afd56a5fa69b24d11f8b5f7283796c" dependencies = [ "anyhow", "arrow", @@ -1073,7 +1183,7 @@ dependencies = [ "rust_decimal", "rust_decimal_macros", "serde_json", - "sqlparser", + "sqlparser 0.37.0", "thiserror", "url", "urlencoding", @@ -1082,23 +1192,21 @@ dependencies = [ [[package]] name = "const-random" -version = "0.1.15" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368a7a772ead6ce7e1de82bfb04c485f3db8ec744f72925af5735e29a22cc18e" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" dependencies = [ "const-random-macro", - "proc-macro-hack", ] [[package]] name = "const-random-macro" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d7d6ab3c3a2282db210df5f02c4dab6e0a7057af0fb7ebd4070f30fe05c0ddb" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ "getrandom", "once_cell", - "proc-macro-hack", "tiny-keccak", ] @@ -1110,14 +1218,14 @@ checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" [[package]] name = "convergence" -version = "0.13.0" -source = "git+https://github.com/roapi/convergence?rev=95fe472e429d02f18016a0d03c884a78fa62861e#95fe472e429d02f18016a0d03c884a78fa62861e" +version = "0.15.0" +source = "git+https://github.com/returnString/convergence.git?rev=3d61a5b9bbf9848b075f5b3ae791c90e5380e5e3#3d61a5b9bbf9848b075f5b3ae791c90e5380e5e3" dependencies = [ "async-trait", "bytes", "chrono", "futures", - "sqlparser", + "sqlparser 0.43.1", "thiserror", "tokio", "tokio-util", @@ -1125,22 +1233,21 @@ dependencies = [ [[package]] name = "convergence-arrow" -version = "0.13.0" -source = "git+https://github.com/roapi/convergence?rev=95fe472e429d02f18016a0d03c884a78fa62861e#95fe472e429d02f18016a0d03c884a78fa62861e" +version = "0.15.0" +source = "git+https://github.com/returnString/convergence.git?rev=3d61a5b9bbf9848b075f5b3ae791c90e5380e5e3#3d61a5b9bbf9848b075f5b3ae791c90e5380e5e3" dependencies = [ "async-trait", "chrono", "convergence", "datafusion", - "sqlparser", "tokio", ] [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ "core-foundation-sys", "libc", @@ -1148,35 +1255,34 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" dependencies = [ - "cfg-if", "crossbeam-channel", "crossbeam-deque", "crossbeam-epoch", @@ -1186,56 +1292,46 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.8" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ - "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.15" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset 0.9.0", - "scopeguard", ] [[package]] name = "crossbeam-queue" -version = "0.3.8" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crunchy" @@ -1265,9 +1361,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.2.2" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626ae34994d3d8d668f4269922248239db4ae42d538b14c398b74a52208e8086" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" dependencies = [ "csv-core", "itoa", @@ -1277,9 +1373,9 @@ dependencies = [ [[package]] name = "csv-core" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" dependencies = [ "memchr", ] @@ -1300,7 +1396,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.0", + "hashbrown 0.14.3", "lock_api", "once_cell", "parking_lot_core", @@ -1308,13 +1404,14 @@ dependencies = [ [[package]] name = "datafusion" -version = "31.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a4e4fc25698a14c90b34dda647ba10a5a966dc04b036d22e77fb1048663375d" +checksum = "4328f5467f76d890fe3f924362dbc3a838c6a733f762b32d87f9e0b7bef5fb49" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.11", "arrow", "arrow-array", + "arrow-ipc", "arrow-schema", "async-compression", "async-trait", @@ -1327,68 +1424,65 @@ dependencies = [ "datafusion-expr", "datafusion-optimizer", "datafusion-physical-expr", + "datafusion-physical-plan", "datafusion-sql", "flate2", "futures", "glob", "half", - "hashbrown 0.14.0", - "indexmap 2.0.0", - "itertools 0.11.0", + "hashbrown 0.14.3", + "indexmap 2.2.5", + "itertools 0.12.1", "log", "num_cpus", "object_store", "parking_lot", "parquet", - "percent-encoding", "pin-project-lite", "rand", - "sqlparser", + "sqlparser 0.41.0", "tempfile", "tokio", "tokio-util", "url", - "uuid 1.4.1", + "uuid 1.8.0", "xz2", "zstd", ] [[package]] name = "datafusion-common" -version = "31.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c23ad0229ea4a85bf76b236d8e75edf539881fdb02ce4e2394f9a76de6055206" +checksum = "d29a7752143b446db4a2cccd9a6517293c6b97e8c39e520ca43ccd07135a4f7e" dependencies = [ + "ahash 0.8.11", "arrow", "arrow-array", - "async-compression", - "bytes", - "bzip2", + "arrow-buffer", + "arrow-schema", "chrono", - "flate2", - "futures", + "half", + "libc", "num_cpus", "object_store", "parquet", - "sqlparser", - "tokio", - "tokio-util", - "xz2", - "zstd", + "sqlparser 0.41.0", ] [[package]] name = "datafusion-execution" -version = "31.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b37d2fc1a213baf34e0a57c85b8e6648f1a95152798fd6738163ee96c19203f" +checksum = "2d447650af16e138c31237f53ddaef6dd4f92f0e2d3f2f35d190e16c214ca496" dependencies = [ "arrow", + "chrono", "dashmap", "datafusion-common", "datafusion-expr", "futures", - "hashbrown 0.14.0", + "hashbrown 0.14.3", "log", "object_store", "parking_lot", @@ -1399,23 +1493,25 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "31.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6ea9844395f537730a145e5d87f61fecd37c2bc9d54e1dc89b35590d867345d" +checksum = "d8d19598e48a498850fb79f97a9719b1f95e7deb64a7a06f93f313e8fa1d524b" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.11", "arrow", + "arrow-array", "datafusion-common", - "sqlparser", - "strum 0.25.0", - "strum_macros 0.25.2", + "paste", + "sqlparser 0.41.0", + "strum", + "strum_macros", ] [[package]] name = "datafusion-optimizer" -version = "31.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8a30e0f79c5d59ba14d3d70f2500e87e0ff70236ad5e47f9444428f054fd2be" +checksum = "8b7feb0391f1fc75575acb95b74bfd276903dc37a5409fcebe160bc7ddff2010" dependencies = [ "arrow", "async-trait", @@ -1423,51 +1519,82 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "hashbrown 0.14.0", - "itertools 0.11.0", + "hashbrown 0.14.3", + "itertools 0.12.1", "log", "regex-syntax", ] [[package]] name = "datafusion-physical-expr" -version = "31.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "766c567082c9bbdcb784feec8fe40c7049cedaeb3a18d54f563f75fe0dc1932c" +checksum = "e911bca609c89a54e8f014777449d8290327414d3e10c57a3e3c2122e38878d0" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.11", "arrow", "arrow-array", "arrow-buffer", + "arrow-ord", "arrow-schema", - "base64 0.21.4", + "base64 0.21.7", "blake2", "blake3", "chrono", "datafusion-common", "datafusion-expr", "half", - "hashbrown 0.14.0", + "hashbrown 0.14.3", "hex", - "indexmap 2.0.0", - "itertools 0.11.0", - "libc", + "indexmap 2.2.5", + "itertools 0.12.1", "log", - "md-5 0.10.5", + "md-5 0.10.6", "paste", "petgraph", "rand", "regex", - "sha2 0.10.7", + "sha2 0.10.8", "unicode-segmentation", - "uuid 1.4.1", + "uuid 1.8.0", +] + +[[package]] +name = "datafusion-physical-plan" +version = "35.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e96b546b8a02e9c2ab35ac6420d511f12a4701950c1eb2e568c122b4fefb0be3" +dependencies = [ + "ahash 0.8.11", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "futures", + "half", + "hashbrown 0.14.3", + "indexmap 2.2.5", + "itertools 0.12.1", + "log", + "once_cell", + "parking_lot", + "pin-project-lite", + "rand", + "tokio", + "uuid 1.8.0", ] [[package]] name = "datafusion-proto" -version = "31.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "440553913695dd8db587b43f54fdc7bbd8649326172d16a4904bb41e792b7297" +checksum = "5742f993d1812d6bb3cdc4ce2a0aa99e10f6dc0daa11dd69b0ff57f2d8e7518c" dependencies = [ "arrow", "chrono", @@ -1480,27 +1607,87 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "31.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "811fd084cf2d78aa0c76b74320977c7084ad0383690612528b580795764b4dd0" +checksum = "2d18d36f260bbbd63aafdb55339213a23d540d3419810575850ef0a798a6b768" dependencies = [ "arrow", "arrow-schema", "datafusion-common", "datafusion-expr", "log", - "sqlparser", + "sqlparser 0.41.0", ] [[package]] name = "deltalake" -version = "0.15.0" -source = "git+https://github.com/delta-io/delta-rs.git?rev=63c14b3716428ff65e01404c6f7e62f341c98f05#63c14b3716428ff65e01404c6f7e62f341c98f05" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e5fa38c2d00cc8a96789eaf3e7a2b27f0bd4c4c7a23e1e59bd5b7b4ceb796fe" +dependencies = [ + "deltalake-aws", + "deltalake-azure", + "deltalake-core", +] + +[[package]] +name = "deltalake-aws" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c27bed8839d53570bbaf9c4ae3a6b187b34f9e07cebd2f0abf2118774f466d9" +dependencies = [ + "async-trait", + "backoff", + "bytes", + "deltalake-core", + "futures", + "lazy_static", + "maplit", + "object_store", + "regex", + "rusoto_core", + "rusoto_credential", + "rusoto_dynamodb", + "rusoto_sts", + "thiserror", + "tokio", + "tracing", + "url", + "uuid 1.8.0", +] + +[[package]] +name = "deltalake-azure" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30e12904f946c91b59d9c357d23fbfc10a82fa164021e962cee482dc8df43469" +dependencies = [ + "async-trait", + "bytes", + "deltalake-core", + "futures", + "lazy_static", + "object_store", + "regex", + "thiserror", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "deltalake-core" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "607be097d9bf5998bfbde3c5e6364f775e5adde0be55843130e5e50f2a2a4387" dependencies = [ "arrow", + "arrow-arith", "arrow-array", "arrow-buffer", "arrow-cast", + "arrow-ipc", + "arrow-json", "arrow-ord", "arrow-row", "arrow-schema", @@ -1516,13 +1703,16 @@ dependencies = [ "datafusion-physical-expr", "datafusion-proto", "datafusion-sql", - "dynamodb_lock", + "either", "errno", + "fix-hidden-lifetime-bug", "futures", - "itertools 0.11.0", + "hashbrown 0.14.3", + "indexmap 2.2.5", + "itertools 0.12.1", "lazy_static", "libc", - "log", + "maplit", "num-bigint", "num-traits", "num_cpus", @@ -1531,38 +1721,40 @@ dependencies = [ "parking_lot", "parquet", "percent-encoding", + "pin-project-lite", + "rand", "regex", - "rusoto_core", - "rusoto_credential", - "rusoto_dynamodb", - "rusoto_sts", + "roaring", "serde", "serde_json", - "sqlparser", + "sqlparser 0.41.0", "thiserror", "tokio", + "tracing", "url", - "uuid 1.4.1", + "uuid 1.8.0", + "z85", ] [[package]] name = "deranged" -version = "0.3.8" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ + "powerfmt", "serde", ] [[package]] name = "derive_utils" -version = "0.13.2" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9abcad25e9720609ccb3dcdb795d845e37d8ce34183330a9f48b03a1a71c8e21" +checksum = "61bb5a1014ce6dfc2a378578509abe775a5aa06bff584a547555d9efdb81b926" dependencies = [ "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.53", ] [[package]] @@ -1644,27 +1836,11 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" -[[package]] -name = "dynamodb_lock" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff628406c318f098017c78e203b5b6466e0610fc48be865ebb9ae0eb229b4c8" -dependencies = [ - "async-trait", - "log", - "maplit", - "rusoto_core", - "rusoto_dynamodb", - "thiserror", - "tokio", - "uuid 1.4.1", -] - [[package]] name = "either" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" [[package]] name = "encoding_rs" @@ -1681,17 +1857,27 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" +[[package]] +name = "env_filter" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" +dependencies = [ + "log", + "regex", +] + [[package]] name = "env_logger" -version = "0.10.0" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9" dependencies = [ + "anstream", + "anstyle", + "env_filter", "humantime", - "is-terminal", "log", - "regex", - "termcolor", ] [[package]] @@ -1702,40 +1888,82 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.3" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "error-code" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64f18991e7bf11e7ffee451b5318b5c1a73c52d0d0ada6e5a3017c8c1ced6a21" +dependencies = [ + "libc", + "str-buf", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "event-listener" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d93877bcde0eb80ca09131a08d23f0a5c18a620b01db137dba666d18cd9b30c2" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" +checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys", + "concurrent-queue", + "parking", + "pin-project-lite", ] [[package]] -name = "errno-dragonfly" -version = "0.1.2" +name = "event-listener" +version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +checksum = "2b5fb89194fa3cad959b833185b3063ba881dbfc7030680b314250779fb4cc91" dependencies = [ - "cc", - "libc", + "concurrent-queue", + "parking", + "pin-project-lite", ] [[package]] -name = "error-code" -version = "2.3.1" +name = "event-listener-strategy" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64f18991e7bf11e7ffee451b5318b5c1a73c52d0d0ada6e5a3017c8c1ced6a21" +checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" dependencies = [ - "libc", - "str-buf", + "event-listener 4.0.3", + "pin-project-lite", ] [[package]] -name = "event-listener" -version = "2.5.3" +name = "event-listener-strategy" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +checksum = "feedafcaa9b749175d5ac357452a9d41ea2911da598fde46ce1fe02c37751291" +dependencies = [ + "event-listener 5.2.0", + "pin-project-lite", +] [[package]] name = "fallible-iterator" @@ -1760,9 +1988,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "fd-lock" @@ -1771,8 +1999,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef033ed5e9bad94e55838ca0ca906db0e043f517adda0c8b79c7a8c66c93c1b5" dependencies = [ "cfg-if", - "rustix 0.38.13", - "windows-sys", + "rustix 0.38.32", + "windows-sys 0.48.0", ] [[package]] @@ -1801,6 +2029,26 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" +[[package]] +name = "fix-hidden-lifetime-bug" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4ae9c2016a663983d4e40a9ff967d6dcac59819672f0b47f2b17574e99c33c8" +dependencies = [ + "fix-hidden-lifetime-bug-proc_macros", +] + +[[package]] +name = "fix-hidden-lifetime-bug-proc_macros" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4c81935e123ab0741c4c4f0d9b8377e5fb21d3de7e062fa4b1263b1fbcba1ea" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "fixedbitset" version = "0.4.2" @@ -1819,9 +2067,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", "libz-sys", @@ -1851,9 +2099,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] @@ -1883,7 +2131,7 @@ checksum = "b0fa992f1656e1707946bbba340ad244f0814009ef8c0118eb7b658395f19a2e" dependencies = [ "frunk_proc_macro_helpers", "quote", - "syn 2.0.36", + "syn 2.0.53", ] [[package]] @@ -1895,7 +2143,7 @@ dependencies = [ "frunk_core", "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.53", ] [[package]] @@ -1907,7 +2155,7 @@ dependencies = [ "frunk_core", "frunk_proc_macro_helpers", "quote", - "syn 2.0.36", + "syn 2.0.53", ] [[package]] @@ -1918,9 +2166,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -1933,9 +2181,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -1943,15 +2191,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -1960,9 +2208,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-lite" @@ -1979,34 +2227,44 @@ dependencies = [ "waker-fn", ] +[[package]] +name = "futures-lite" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" +dependencies = [ + "futures-core", + "pin-project-lite", +] + [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.53", ] [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -2032,9 +2290,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if", "libc", @@ -2043,9 +2301,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "glob" @@ -2065,9 +2323,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.21" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "4fbd2820c5e49886948654ab546d0688ff24530286bdcf8fca3cefb16d4618eb" dependencies = [ "bytes", "fnv", @@ -2075,7 +2333,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 1.9.3", + "indexmap 2.2.5", "slab", "tokio", "tokio-util", @@ -2084,9 +2342,9 @@ dependencies = [ [[package]] name = "half" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872" +checksum = "b5eceaaeec696539ddaf7b333340f1af35a5aa87ae3e4f3ead0532f72affab2e" dependencies = [ "cfg-if", "crunchy", @@ -2099,25 +2357,16 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ - "ahash 0.7.6", -] - -[[package]] -name = "hashbrown" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" -dependencies = [ - "ahash 0.8.3", + "ahash 0.7.8", ] [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.11", "allocator-api2", ] @@ -2127,7 +2376,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ - "hashbrown 0.14.0", + "hashbrown 0.14.3", ] [[package]] @@ -2138,18 +2387,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.3.2" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "hex" @@ -2178,9 +2418,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.9" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", @@ -2189,9 +2429,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", "http", @@ -2224,9 +2464,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ "bytes", "futures-channel", @@ -2239,7 +2479,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2 0.5.6", "tokio", "tower-service", "tracing", @@ -2280,14 +2520,14 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http", "hyper", - "rustls 0.21.7", + "rustls 0.21.10", "tokio", "tokio-rustls 0.24.1", ] @@ -2319,16 +2559,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.57" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows", + "windows-core", ] [[package]] @@ -2342,9 +2582,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -2362,12 +2602,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.0" +version = "2.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown 0.14.3", ] [[package]] @@ -2387,12 +2627,11 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "io-enum" -version = "1.1.1" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5305557fa27b460072ae15ce07617e999f5879f14d376c8449f0bfb9f9d8e91e" +checksum = "53b53d712d99a73eec59ee5e4fe6057f8052142d38eeafbbffcb06b36d738a6e" dependencies = [ "derive_utils", - "syn 2.0.36", ] [[package]] @@ -2401,66 +2640,64 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi 0.3.2", + "hermit-abi", "libc", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] name = "ipnet" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] -name = "is-terminal" -version = "0.4.9" +name = "itertools" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" dependencies = [ - "hermit-abi 0.3.2", - "rustix 0.38.13", - "windows-sys", + "either", ] [[package]] name = "itertools" -version = "0.10.5" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" dependencies = [ "either", ] [[package]] name = "itertools" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "jobserver" -version = "0.1.26" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] @@ -2552,25 +2789,36 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.148" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libloading" -version = "0.7.4" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" dependencies = [ "cfg-if", - "winapi", + "windows-targets 0.52.4", ] [[package]] name = "libm" -version = "0.2.7" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + +[[package]] +name = "libredox" +version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" +checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +dependencies = [ + "bitflags 2.5.0", + "libc", + "redox_syscall", +] [[package]] name = "libsqlite3-sys" @@ -2585,21 +2833,15 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.12" +version = "1.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +checksum = "5e143b5e666b2695d28f6bca6497720813f699c9602dd7f5cac91008b8ada7f9" dependencies = [ "cc", "pkg-config", "vcpkg", ] -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" - [[package]] name = "linux-raw-sys" version = "0.3.8" @@ -2608,15 +2850,15 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.4.7" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -2624,9 +2866,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "lru" @@ -2638,23 +2880,12 @@ dependencies = [ ] [[package]] -name = "lz4" -version = "1.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9e2dd86df36ce760a60f6ff6ad526f7ba1f14ba0356f8254fb6905e6494df1" -dependencies = [ - "libc", - "lz4-sys", -] - -[[package]] -name = "lz4-sys" -version = "1.9.4" +name = "lz4_flex" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" +checksum = "912b45c753ff5f7f5208307e8ace7d2a2e30d024e26d3509f3dce546c044ce15" dependencies = [ - "cc", - "libc", + "twox-hash", ] [[package]] @@ -2693,18 +2924,19 @@ dependencies = [ [[package]] name = "md-5" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ + "cfg-if", "digest 0.10.7", ] [[package]] name = "memchr" -version = "2.6.3" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memoffset" @@ -2715,15 +2947,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "memoffset" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" -dependencies = [ - "autocfg", -] - [[package]] name = "mime" version = "0.3.17" @@ -2738,22 +2961,22 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] [[package]] name = "mio" -version = "0.8.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "wasi", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -2777,7 +3000,7 @@ dependencies = [ "percent-encoding", "serde", "serde_json", - "socket2 0.4.9", + "socket2 0.4.10", "twox-hash", "url", ] @@ -2812,12 +3035,12 @@ dependencies = [ "serde", "serde_json", "sha1", - "sha2 0.10.7", + "sha2 0.10.8", "smallvec", "subprocess", "thiserror", "time", - "uuid 1.4.1", + "uuid 1.8.0", ] [[package]] @@ -2866,7 +3089,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "memoffset 0.6.5", + "memoffset", ] [[package]] @@ -2906,28 +3129,33 @@ dependencies = [ [[package]] name = "num-complex" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ba157ca0885411de85d6ca030ba7e2a83a28636056c7c699b07c8b6f7383214" +checksum = "23c6602fda94a57c990fe0df199a035d83576b496aa29f4e634a8ac6004e68a6" dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] [[package]] name = "num-iter" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9" dependencies = [ "autocfg", "num-integer", @@ -2948,9 +3176,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.16" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", "libm", @@ -2962,52 +3190,52 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.2", + "hermit-abi", "libc", ] [[package]] name = "num_threads" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" dependencies = [ "libc", ] [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] [[package]] name = "object_store" -version = "0.7.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d359e231e5451f4f9fa889d56e3ce34f8724f1a61db2107739359717cf2bbf08" +checksum = "d139f545f64630e2e3688fd9f81c470888ab01edeb72d13b4e86c566f1130000" dependencies = [ "async-trait", - "base64 0.21.4", + "base64 0.21.7", "bytes", "chrono", "futures", "humantime", "hyper", - "itertools 0.10.5", + "itertools 0.12.1", "parking_lot", "percent-encoding", - "quick-xml 0.28.2", + "quick-xml", "rand", "reqwest", - "ring", - "rustls-pemfile 1.0.3", + "ring 0.17.8", + "rustls-pemfile 2.1.1", "serde", "serde_json", - "snafu", + "snafu 0.7.5", "tokio", "tracing", "url", @@ -3016,23 +3244,23 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "opaque-debug" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "openssl" -version = "0.10.57" +version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.5.0", "cfg-if", "foreign-types", "libc", @@ -3049,7 +3277,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.53", ] [[package]] @@ -3060,18 +3288,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "300.1.3+3.1.2" +version = "300.2.3+3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd2c101a165fff9935e34def4669595ab1c7847943c42be86e21503e482be107" +checksum = "5cff92b6f71555b61bb9315f7c64da3ca43d87531622120fea0195fc761b4843" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.93" +version = "0.9.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff" dependencies = [ "cc", "libc", @@ -3082,19 +3310,13 @@ dependencies = [ [[package]] name = "ordered-float" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7940cf2ca942593318d07fcf2596cdca60a85c9e7fab408a5e21a4f9dcd40d87" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" dependencies = [ "num-traits", ] -[[package]] -name = "os_str_bytes" -version = "6.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac" - [[package]] name = "owning_ref" version = "0.4.1" @@ -3104,21 +3326,11 @@ dependencies = [ "stable_deref_trait", ] -[[package]] -name = "packed_simd" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f9f08af0c877571712e2e3e686ad79efad9657dbf0f7c3c8ba943ff6c38932d" -dependencies = [ - "cfg-if", - "num-traits", -] - [[package]] name = "parking" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e" +checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] name = "parking_lot" @@ -3132,24 +3344,24 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", + "redox_syscall", "smallvec", - "windows-targets", + "windows-targets 0.48.5", ] [[package]] name = "parquet" -version = "46.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad2cba786ae07da4d73371a88b9e0f9d3ffac1a9badc83922e0e15814f5c5fa" +checksum = "547b92ebf0c1177e3892f44c8f79757ee62e678d564a9834189725f2c5b7a750" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.11", "arrow-array", "arrow-buffer", "arrow-cast", @@ -3157,14 +3369,15 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64 0.21.4", + "base64 0.21.7", "brotli", "bytes", "chrono", "flate2", "futures", - "hashbrown 0.14.0", - "lz4", + "half", + "hashbrown 0.14.3", + "lz4_flex", "num", "num-bigint", "object_store", @@ -3209,9 +3422,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "petgraph" @@ -3220,7 +3433,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.0.0", + "indexmap 2.2.5", ] [[package]] @@ -3263,22 +3476,22 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.3" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.3" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.53", ] [[package]] @@ -3293,11 +3506,22 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "piper" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" +dependencies = [ + "atomic-waker", + "fastrand 2.0.1", + "futures-io", +] + [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "polling" @@ -3312,7 +3536,21 @@ dependencies = [ "libc", "log", "pin-project-lite", - "windows-sys", + "windows-sys 0.48.0", +] + +[[package]] +name = "polling" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24f040dee2588b4963afb4e420540439d126f73fdacf4a9c486a96d840bac3c9" +dependencies = [ + "cfg-if", + "concurrent-queue", + "pin-project-lite", + "rustix 0.38.32", + "tracing", + "windows-sys 0.52.0", ] [[package]] @@ -3361,15 +3599,15 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49b6c5ef183cd3ab4ba005f1ca64c21e8bd97ce4699cfea9e8d9a2c4958ca520" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "byteorder", "bytes", "fallible-iterator", "hmac 0.12.1", - "md-5 0.10.5", + "md-5 0.10.6", "memchr", "rand", - "sha2 0.10.7", + "sha2 0.10.8", "stringprep", ] @@ -3388,6 +3626,12 @@ dependencies = [ "uuid 0.8.2", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -3406,33 +3650,50 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "0.1.5" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" +dependencies = [ + "toml_edit 0.21.1", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ - "toml 0.5.11", + "proc-macro-error-attr", + "proc-macro2", + "quote", + "version_check", ] [[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" +name = "proc-macro-error-attr" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] [[package]] name = "proc-macro2" -version = "1.0.67" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" dependencies = [ "unicode-ident", ] [[package]] name = "prost" -version = "0.11.9" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" dependencies = [ "bytes", "prost-derive", @@ -3440,15 +3701,15 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.11.9" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.11.0", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.53", ] [[package]] @@ -3473,29 +3734,20 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.25.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58e21a144a0ffb5fad7b464babcdab934a325ad69b7c0373bcfef5cbd9799ca9" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" dependencies = [ "encoding_rs", "memchr", -] - -[[package]] -name = "quick-xml" -version = "0.28.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce5e73202a820a31f8a0ee32ada5e21029c81fd9e3ebf668a40832e4219d9d1" -dependencies = [ - "memchr", "serde", ] [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -3539,7 +3791,7 @@ checksum = "99f31323d6161385f385046738df520e0e8694fa74852d35891fc0be08348ddc" dependencies = [ "r2d2", "rusqlite", - "uuid 1.4.1", + "uuid 1.8.0", ] [[package]] @@ -3590,9 +3842,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.7.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +checksum = "e4963ed1bc86e4f3ee217022bd855b297cef07fb9eac5dfa1f788b220b49b3bd" dependencies = [ "either", "rayon-core", @@ -3600,50 +3852,39 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.11.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", -] - -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags 1.3.2", ] [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "redox_users" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" dependencies = [ "getrandom", - "redox_syscall 0.2.16", + "libredox", "thiserror", ] [[package]] name = "regex" -version = "1.9.5" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", @@ -3653,9 +3894,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.8" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ "aho-corasick", "memchr", @@ -3664,26 +3905,26 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "rend" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581008d2099240d37fb08d77ad713bcaec2c4d89d50b5b21a8bb1996bbab68ab" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" dependencies = [ "bytecheck", ] [[package]] name = "reqwest" -version = "0.11.20" +version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", @@ -3692,7 +3933,7 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-rustls 0.24.1", + "hyper-rustls 0.24.2", "hyper-tls", "ipnet", "js-sys", @@ -3702,11 +3943,14 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.7", - "rustls-pemfile 1.0.3", + "rustls 0.21.10", + "rustls-native-certs 0.6.3", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", + "system-configuration", "tokio", "tokio-native-tls", "tokio-rustls 0.24.1", @@ -3730,34 +3974,50 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", - "untrusted", + "spin 0.5.2", + "untrusted 0.7.1", "web-sys", "winapi", ] +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.52.0", +] + [[package]] name = "rkyv" -version = "0.7.42" +version = "0.7.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0200c8230b013893c0b2d6213d6ec64ed2b9be2e0e016682b7224ff82cff5c58" +checksum = "5cba464629b3394fc4dbc6f940ff8f5b4ff5c7aef40f29166fd4ad12acbc99c0" dependencies = [ "bitvec", "bytecheck", + "bytes", "hashbrown 0.12.3", "ptr_meta", "rend", "rkyv_derive", "seahash", "tinyvec", - "uuid 1.4.1", + "uuid 1.8.0", ] [[package]] name = "rkyv_derive" -version = "0.7.42" +version = "0.7.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2e06b915b5c230a17d7a736d1e2e63ee753c256a8614ef3f5147b13a4f5541d" +checksum = "a7dddfff8de25e6f62b9d64e6e432bf1c6736c57d20323e15ee10435fbda7c65" dependencies = [ "proc-macro2", "quote", @@ -3766,7 +4026,7 @@ dependencies = [ [[package]] name = "roapi" -version = "0.10.0" +version = "0.11.1" dependencies = [ "arrow-cast", "arrow-flight", @@ -3774,8 +4034,10 @@ dependencies = [ "async-process", "async-trait", "axum", + "base64 0.22.0", "clap", "columnq", + "constant_time_eq", "convergence", "convergence-arrow", "dashmap", @@ -3791,20 +4053,31 @@ dependencies = [ "serde_derive", "serde_json", "serde_yaml", - "snafu", + "snafu 0.8.2", "snmalloc-rs", "tempfile", "thiserror", "tokio", "tokio-postgres", "tokio-stream", - "toml 0.7.8", + "toml", "tonic", "tower", "tower-http", "tower-layer", + "tower-service", "tracing", - "uuid 1.4.1", + "uuid 1.8.0", +] + +[[package]] +name = "roaring" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1c77081a55300e016cb86f2864415b7518741879db925b8d488a0ee0d2da6bf" +dependencies = [ + "bytemuck", + "byteorder", ] [[package]] @@ -3911,7 +4184,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.5.0", "chrono", "fallible-iterator", "fallible-streaming-iterator", @@ -3922,9 +4195,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.32.0" +version = "1.34.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c4216490d5a413bc6d10fa4742bd7d4955941d062c0ef873141d6b0e7b30fd" +checksum = "b39449a79f45e8da28c57c341891b69a183044b29518bb8f86dbac9df60bb7df" dependencies = [ "arrayvec", "borsh", @@ -3939,9 +4212,9 @@ dependencies = [ [[package]] name = "rust_decimal_macros" -version = "1.32.0" +version = "1.34.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86444b802de0b10ac5e563b5ddb43b541b9705de4e01a50e82194d2b183c1835" +checksum = "e418701588729bef95e7a655f2b483ad64bb97c46e8e79fde83efd92aaab6d82" dependencies = [ "quote", "rust_decimal", @@ -3970,29 +4243,29 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.23" +version = "0.37.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" +checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" dependencies = [ "bitflags 1.3.2", "errno", "io-lifetimes", "libc", "linux-raw-sys 0.3.8", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] name = "rustix" -version = "0.38.13" +version = "0.38.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662" +checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.5.0", "errno", "libc", - "linux-raw-sys 0.4.7", - "windows-sys", + "linux-raw-sys 0.4.13", + "windows-sys 0.52.0", ] [[package]] @@ -4003,7 +4276,7 @@ checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" dependencies = [ "base64 0.13.1", "log", - "ring", + "ring 0.16.20", "sct 0.6.1", "webpki 0.21.4", ] @@ -4015,21 +4288,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" dependencies = [ "log", - "ring", - "sct 0.7.0", - "webpki 0.22.1", + "ring 0.16.20", + "sct 0.7.1", + "webpki 0.22.4", ] [[package]] name = "rustls" -version = "0.21.7" +version = "0.21.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", - "ring", + "ring 0.17.8", "rustls-webpki", - "sct 0.7.0", + "sct 0.7.1", ] [[package]] @@ -4051,7 +4324,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" dependencies = [ "openssl-probe", - "rustls-pemfile 1.0.3", + "rustls-pemfile 1.0.4", "schannel", "security-framework", ] @@ -4067,21 +4340,37 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.3" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + +[[package]] +name = "rustls-pemfile" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" +checksum = "f48172685e6ff52a556baa527774f61fcaa884f59daf3375c62a3f1cd2549dab" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", + "rustls-pki-types", ] +[[package]] +name = "rustls-pki-types" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ede67b28608b4c60685c7d54122d4400d90f62b40caee7700e700380a390fa8" + [[package]] name = "rustls-webpki" -version = "0.101.5" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45a27e3b59326c16e23d30aeb7a36a24cc0d29e71d68ff611cdfb4a01d013bed" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring", - "untrusted", + "ring 0.17.8", + "untrusted 0.9.0", ] [[package]] @@ -4116,9 +4405,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" [[package]] name = "same-file" @@ -4137,11 +4426,11 @@ checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71" [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -4165,18 +4454,18 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] name = "sct" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring", - "untrusted", + "ring 0.17.8", + "untrusted 0.9.0", ] [[package]] @@ -4210,9 +4499,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.18" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" [[package]] name = "seq-macro" @@ -4222,29 +4511,29 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.53", ] [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" dependencies = [ "itoa", "ryu", @@ -4253,9 +4542,9 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.14" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4beec8bce849d58d06238cb50db2e1c417cfeafa4c63f692b15c82b7c80f8335" +checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" dependencies = [ "itoa", "serde", @@ -4263,9 +4552,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.3" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" +checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" dependencies = [ "serde", ] @@ -4284,21 +4573,22 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.8.26" +version = "0.9.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" +checksum = "a0623d197252096520c6f2a5e1171ee436e5af99a5d7caa2891e55e61950e6d9" dependencies = [ - "indexmap 1.9.3", + "indexmap 2.2.5", + "itoa", "ryu", "serde", - "yaml-rust", + "unsafe-libyaml", ] [[package]] name = "sha1" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", @@ -4320,9 +4610,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", @@ -4331,19 +4621,9 @@ dependencies = [ [[package]] name = "shlex" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" - -[[package]] -name = "signal-hook" -version = "0.3.17" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" -dependencies = [ - "libc", - "signal-hook-registry", -] +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" @@ -4377,9 +4657,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "snafu" @@ -4388,7 +4668,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" dependencies = [ "doc-comment", - "snafu-derive", + "snafu-derive 0.7.5", +] + +[[package]] +name = "snafu" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75976f4748ab44f6e5332102be424e7c2dc18daeaf7e725f2040c3ebb133512e" +dependencies = [ + "snafu-derive 0.8.2", ] [[package]] @@ -4403,11 +4692,23 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "snafu-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4b19911debfb8c2fb1107bc6cb2d61868aaf53a988449213959bb1b5b1ed95f" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.53", +] + [[package]] name = "snap" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "snmalloc-rs" @@ -4429,9 +4730,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" dependencies = [ "libc", "winapi", @@ -4439,12 +4740,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.4" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -4453,25 +4754,49 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "sqlparser" version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37ae05a8250b968a3f7db93155a84d68b2e6cea1583949af5ca5b5170c76c075" +dependencies = [ + "log", +] + +[[package]] +name = "sqlparser" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc2c25a6c66789625ef164b4c7d2e548d627902280c13710d33da8222169964" dependencies = [ "log", "sqlparser_derive", ] +[[package]] +name = "sqlparser" +version = "0.43.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f95c4bae5aba7cd30bd506f7140026ade63cff5afd778af8854026f9606bf5d4" +dependencies = [ + "log", +] + [[package]] name = "sqlparser_derive" -version = "0.1.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55fe75cb4a364c7f7ae06c7dbbc8d84bddd85d6cdf9975963c3935bc1991761e" +checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.53", ] [[package]] @@ -4505,15 +4830,9 @@ dependencies = [ [[package]] name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - -[[package]] -name = "strum" -version = "0.24.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" [[package]] name = "strum" @@ -4521,33 +4840,20 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" dependencies = [ - "strum_macros 0.25.2", -] - -[[package]] -name = "strum_macros" -version = "0.24.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn 1.0.109", + "strum_macros", ] [[package]] name = "strum_macros" -version = "0.25.2" +version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8d03b598d3d0fff69bf533ee3ef19b8eeb342729596df84bcc7e1f96ec4059" +checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" dependencies = [ "heck", "proc-macro2", "quote", "rustversion", - "syn 2.0.36", + "syn 2.0.53", ] [[package]] @@ -4579,15 +4885,27 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.36" +version = "2.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e02e55d62894af2a08aca894c6577281f76769ba47c94d5756bec8ac6e7373" +checksum = "7383cd0e49fff4b6b90ca5670bfd3e9d6a733b3f90c686605aa7eec8c4996032" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "syn_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1329189c02ff984e9736652b1631330da25eaa6bc639089ed4915d25446cbe7b" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.53", +] + [[package]] name = "sync_wrapper" version = "0.1.2" @@ -4595,57 +4913,62 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] -name = "tap" -version = "1.0.1" +name = "system-configuration" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] [[package]] -name = "tempfile" -version = "3.8.0" +name = "system-configuration-sys" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" dependencies = [ - "cfg-if", - "fastrand 2.0.0", - "redox_syscall 0.3.5", - "rustix 0.38.13", - "windows-sys", + "core-foundation-sys", + "libc", ] [[package]] -name = "termcolor" -version = "1.2.0" +name = "tap" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" -dependencies = [ - "winapi-util", -] +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] -name = "textwrap" -version = "0.16.0" +name = "tempfile" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if", + "fastrand 2.0.1", + "rustix 0.38.32", + "windows-sys 0.52.0", +] [[package]] name = "thiserror" -version = "1.0.48" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" +checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.48" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" +checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.53", ] [[package]] @@ -4661,13 +4984,15 @@ dependencies = [ [[package]] name = "time" -version = "0.3.28" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" +checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" dependencies = [ "deranged", "libc", + "num-conv", "num_threads", + "powerfmt", "serde", "time-core", "time-macros", @@ -4675,16 +5000,17 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.14" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" +checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" dependencies = [ + "num-conv", "time-core", ] @@ -4714,9 +5040,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.32.0" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" dependencies = [ "backtrace", "bytes", @@ -4726,9 +5052,9 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.4", + "socket2 0.5.6", "tokio-macros", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -4743,13 +5069,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.53", ] [[package]] @@ -4764,9 +5090,9 @@ dependencies = [ [[package]] name = "tokio-openssl" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08f9ffb7809f1b20c1b398d92acf4cc719874b3b2b2d9ea2f09b4a80350878a" +checksum = "6ffab79df67727f6acf57f1ff743091873c24c579b1e2ce4d8f53e47ded4d63d" dependencies = [ "futures-util", "openssl", @@ -4794,7 +5120,7 @@ dependencies = [ "postgres-protocol", "postgres-types", "rand", - "socket2 0.5.4", + "socket2 0.5.6", "tokio", "tokio-util", "whoami", @@ -4819,7 +5145,7 @@ checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ "rustls 0.20.9", "tokio", - "webpki 0.22.1", + "webpki 0.22.4", ] [[package]] @@ -4828,15 +5154,15 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.7", + "rustls 0.21.10", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" dependencies = [ "futures-core", "pin-project-lite", @@ -4845,9 +5171,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.8" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", @@ -4857,15 +5183,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "toml" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" -dependencies = [ - "serde", -] - [[package]] name = "toml" version = "0.7.8" @@ -4875,14 +5192,14 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit", + "toml_edit 0.19.15", ] [[package]] name = "toml_datetime" -version = "0.6.3" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" dependencies = [ "serde", ] @@ -4893,26 +5210,35 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.0.0", + "indexmap 2.2.5", "serde", "serde_spanned", "toml_datetime", "winnow", ] +[[package]] +name = "toml_edit" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" +dependencies = [ + "indexmap 2.2.5", + "toml_datetime", + "winnow", +] + [[package]] name = "tonic" -version = "0.9.2" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" +checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" dependencies = [ "async-stream", "async-trait", "axum", - "base64 0.21.4", + "base64 0.21.7", "bytes", - "futures-core", - "futures-util", "h2", "http", "http-body", @@ -4921,7 +5247,8 @@ dependencies = [ "percent-encoding", "pin-project", "prost", - "rustls-pemfile 1.0.3", + "rustls 0.21.10", + "rustls-pemfile 1.0.4", "tokio", "tokio-rustls 0.24.1", "tokio-stream", @@ -4953,11 +5280,11 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.3.5" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" +checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.5.0", "bytes", "futures-core", "futures-util", @@ -4967,6 +5294,7 @@ dependencies = [ "pin-project-lite", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -4983,11 +5311,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -4996,29 +5323,29 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.53", ] [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", ] [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "twox-hash" @@ -5039,9 +5366,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" @@ -5051,24 +5378,24 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "unreachable" @@ -5079,12 +5406,24 @@ dependencies = [ "void", ] +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "uriparse" version = "0.6.4" @@ -5097,9 +5436,9 @@ dependencies = [ [[package]] name = "url" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", @@ -5126,9 +5465,9 @@ checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" [[package]] name = "uuid" -version = "1.4.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" +checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" dependencies = [ "getrandom", "rand", @@ -5155,15 +5494,15 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "waker-fn" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" +checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" [[package]] name = "walkdir" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", @@ -5184,11 +5523,17 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -5196,24 +5541,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.53", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.37" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" dependencies = [ "cfg-if", "js-sys", @@ -5223,9 +5568,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5233,28 +5578,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.36", + "syn 2.0.53", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "wasm-streams" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" dependencies = [ "futures-util", "js-sys", @@ -5265,9 +5610,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" dependencies = [ "js-sys", "wasm-bindgen", @@ -5279,33 +5624,34 @@ version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] name = "webpki" -version = "0.22.1" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0e74f82d49d545ad128049b7e88f6576df2da6b02e9ce565c6f533be576957e" +checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" dependencies = [ - "ring", - "untrusted", + "ring 0.17.8", + "untrusted 0.9.0", ] [[package]] name = "webpki-roots" -version = "0.25.2" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "whoami" -version = "1.4.1" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" +checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" dependencies = [ - "wasm-bindgen", + "redox_syscall", + "wasite", "web-sys", ] @@ -5327,9 +5673,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" dependencies = [ "winapi", ] @@ -5341,12 +5687,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows" -version = "0.48.0" +name = "windows-core" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets", + "windows-targets 0.52.4", ] [[package]] @@ -5355,7 +5701,16 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.4", ] [[package]] @@ -5364,13 +5719,28 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", ] [[package]] @@ -5379,47 +5749,89 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" + [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" + [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" + [[package]] name = "winnow" -version = "0.5.15" +version = "0.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" dependencies = [ "memchr", ] @@ -5431,7 +5843,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ "cfg-if", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -5445,9 +5857,9 @@ dependencies = [ [[package]] name = "xml-rs" -version = "0.8.18" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab77e97b50aee93da431f2cee7cd0f43b4d1da3c408042f2d7d164187774f0a" +checksum = "0fcb9cbac069e033553e8bb871be2fbdffcab578eb25bd0f7c508cedc6dcd75a" [[package]] name = "xz2" @@ -5458,15 +5870,6 @@ dependencies = [ "lzma-sys", ] -[[package]] -name = "yaml-rust" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" -dependencies = [ - "linked-hash-map", -] - [[package]] name = "yansi" version = "0.5.1" @@ -5500,11 +5903,37 @@ dependencies = [ "url", ] +[[package]] +name = "z85" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a599daf1b507819c1121f0bf87fa37eb19daac6aff3aefefd4e6e2e0f2020fc" + +[[package]] +name = "zerocopy" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.53", +] + [[package]] name = "zeroize" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" [[package]] name = "zip" @@ -5520,30 +5949,28 @@ dependencies = [ [[package]] name = "zstd" -version = "0.12.4" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "6.0.6" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" dependencies = [ - "libc", "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.8+zstd.1.5.5" +version = "2.0.9+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" dependencies = [ "cc", - "libc", "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index 925c5eea5..cc67237a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,9 @@ resolver = "2" [profile.dev] split-debuginfo = "unpacked" +[profile.test] +split-debuginfo = "unpacked" + [profile.release] lto = true codegen-units = 1 diff --git a/Dockerfile b/Dockerfile index 78dbf7e62..776dac322 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,51 @@ -FROM instrumentisto/rust:nightly-bullseye-2023-09-15 AS builder -WORKDIR /roapi_src -COPY ./ /roapi_src +ARG RUST_VER=nightly-bullseye-2024-03-01 +ARG RUSTFLAGS='-C target-cpu=skylake' +ARG FEATURES="database" + +# Step 0: Install cargo-chef +FROM instrumentisto/rust:${RUST_VER} AS chef +# We only pay the installation cost once, +# it will be cached from the second build onwards +RUN cargo +nightly install cargo-chef +# install cmake for snmalloc RUN apt-get update \ && apt-get install --no-install-recommends -y cmake -RUN RUSTFLAGS='-C target-cpu=skylake' \ - cargo +nightly install --locked --bins roapi --features "simd database" --path roapi +# Step 1: Compute a recipe file +FROM chef AS planner +WORKDIR /roapi_src +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +# Step 2: Cache project dependencies +FROM chef AS cacher +ARG RUSTFLAGS +ARG FEATURES +WORKDIR /roapi_src +COPY --from=planner /roapi_src/recipe.json recipe.json +RUN RUSTFLAGS=${RUSTFLAGS} \ + cargo +nightly chef cook --features ${FEATURES} --release --recipe-path recipe.json +# Step 3: Build the release binary +FROM chef AS builder +ARG RUSTFLAGS +ARG FEATURES +WORKDIR /roapi_src +COPY ./ /roapi_src +COPY --from=cacher /roapi_src/target target +COPY --from=cacher /usr/local/cargo /usr/local/cargo +RUN RUSTFLAGS=${RUSTFLAGS} \ + cargo +nightly build --release --locked --bin roapi --features ${FEATURES} + +# Step 4: Assemble the final image FROM debian:bullseye-slim LABEL org.opencontainers.image.source https://github.com/roapi/roapi -RUN apt-get update && apt-get install -y libssl-dev ca-certificates && rm -rf /var/lib/apt/lists/* +RUN apt-get update \ + && apt-get install -y libssl-dev ca-certificates \ + && rm -rf /var/lib/apt/lists/* COPY test_data /test_data -COPY --from=builder /usr/local/cargo/bin/roapi /usr/local/bin/roapi +COPY --from=builder /roapi_src/target/release/roapi /usr/local/bin/roapi EXPOSE 8080 ENTRYPOINT ["roapi"] diff --git a/README.md b/README.md index 2b53f71bc..afdeb30f9 100644 --- a/README.md +++ b/README.md @@ -318,7 +318,7 @@ Data layer: - [x] JSON - [x] NDJSON - [x] parquet - - [x] xls, xlsx, xlsm, ods: https://github.com/tafia/calamine + - [x] xls, xlsx, xlsb, ods: https://github.com/tafia/calamine - [x] [DeltaLake](https://delta.io/) Misc: @@ -341,6 +341,10 @@ produced by `columnq` into different formats based on client request. Building ROAPI with `simd` optimization requires nightly rust toolchain. +### Debug + +To log all FlightSQL requests in console, set `RUST_LOG=tower_http=trace`. + ### Build Docker image ```bash diff --git a/ci/scripts/setup_azure.sh b/ci/scripts/setup_azure.sh index 88c2168b6..1e790a285 100644 --- a/ci/scripts/setup_azure.sh +++ b/ci/scripts/setup_azure.sh @@ -2,4 +2,4 @@ set -eux -docker run -d -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite:3.26.0 +docker run -d -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite:3.29.0 diff --git a/ci/scripts/setup_gcs.sh b/ci/scripts/setup_gcs.sh index c64c9c388..5edf1621f 100644 --- a/ci/scripts/setup_gcs.sh +++ b/ci/scripts/setup_gcs.sh @@ -4,8 +4,9 @@ set -eux # Custom image - see fsouza/fake-gcs-server#1164 docker run -d -p 4443:4443 \ + --name gcs-server \ tustvold/fake-gcs-server \ -scheme http \ -public-host localhost:4443 -echo '{"gcs_base_url": "http://localhost:4443", "disable_oauth": true, "client_email": "", "private_key": ""}' > "/tmp/gcs.json" +echo '{"gcs_base_url": "http://localhost:4443", "disable_oauth": true, "client_email": "", "private_key_id": "", "private_key": ""}' > "/tmp/gcs.json" diff --git a/columnq-cli/Cargo.toml b/columnq-cli/Cargo.toml index 0e2740298..907694c2a 100644 --- a/columnq-cli/Cargo.toml +++ b/columnq-cli/Cargo.toml @@ -22,7 +22,7 @@ tokio = "1" rustyline = { version = "9" } env_logger = { version = "0" } anyhow = { version = "1" } -clap = { version = "3", features = ["color"] } +clap = { version = "4", features = ["color"] } dirs = { version = "4" } [features] @@ -30,7 +30,6 @@ default = ["rustls", "snmalloc"] rustls = ["columnq/rustls"] native-tls-vendored = ["columnq/native-tls-vendored"] native-tls = ["columnq/native-tls"] -simd = ["columnq/simd"] snmalloc = ["snmalloc-rs"] database-sqlite = ["columnq/database-sqlite"] database-mysql = ["columnq/database-mysql"] diff --git a/columnq-cli/src/main.rs b/columnq-cli/src/main.rs index 0472d0ac6..778b5173d 100644 --- a/columnq-cli/src/main.rs +++ b/columnq-cli/src/main.rs @@ -31,13 +31,13 @@ fn config_path() -> anyhow::Result { Ok(home) } -fn table_arg() -> clap::Arg<'static> { +fn table_arg() -> clap::Arg { clap::Arg::new("table") .long_help("Table sources to load. Table option can be provided as optional setting as part of the table URI, for example: `blogs=s3://bucket/key,format=delta`. Set table uri to `stdin` if you want to consume table data from stdin as part of a UNIX pipe. If no table_name is provided, a table name will be derived from the filename in URI.") - .takes_value(true) + .num_args(1) .required(false) .number_of_values(1) - .multiple_occurrences(true) + .action(clap::ArgAction::Append) .value_name("[table_name=]uri[,option_key=option_value]") .long("table") .short('t') @@ -90,7 +90,7 @@ async fn cmd_console(args: &clap::ArgMatches) -> anyhow::Result<()> { let config = SessionConfig::default().with_information_schema(true); let mut cq = ColumnQ::new_with_config(config); - if let Some(tables) = args.values_of("table") { + if let Some(tables) = args.get_many::<&str>("table") { for v in tables { cq.load_table(&parse_table_uri_arg(v)?).await?; } @@ -110,33 +110,33 @@ async fn cmd_sql(args: &clap::ArgMatches) -> anyhow::Result<()> { let config = SessionConfig::default().with_information_schema(true); let mut cq = ColumnQ::new_with_config(config); - if let Some(tables) = args.values_of("table") { + if let Some(tables) = args.get_many::<&str>("table") { for v in tables { cq.load_table(&parse_table_uri_arg(v)?).await?; } } - match args.value_of("SQL") { + match args.get_one::<&str>("SQL") { Some(query) => match cq.query_sql(query).await { - Ok(batches) => match args.value_of("output").unwrap_or("table") { - "table" => pretty::print_batches(&batches)?, - "json" => { + Ok(batches) => match args.get_one::<&str>("output").unwrap_or(&"table") { + &"table" => pretty::print_batches(&batches)?, + &"json" => { let bytes = encoding::json::record_batches_to_bytes(&batches)?; bytes_to_stdout(&bytes)?; } - "csv" => { + &"csv" => { let bytes = encoding::csv::record_batches_to_bytes(&batches)?; bytes_to_stdout(&bytes)?; } - "parquet" => { + &"parquet" => { let bytes = encoding::parquet::record_batches_to_bytes(&batches)?; bytes_to_stdout(&bytes)?; } - "arrow" => { + &"arrow" => { let bytes = encoding::arrow::record_batches_to_file_bytes(&batches)?; bytes_to_stdout(&bytes)?; } - "arrows" => { + &"arrows" => { let bytes = encoding::arrow::record_batches_to_stream_bytes(&batches)?; bytes_to_stdout(&bytes)?; } @@ -175,18 +175,16 @@ async fn main() -> anyhow::Result<()> { .help("SQL query to execute") .index(1) .required(true) - .takes_value(true) - .number_of_values(1), + .num_args(1), clap::Arg::new("output") .help("Query output format") .long("output") .short('o') .required(false) - .takes_value(true) - .number_of_values(1) + .num_args(1) .default_value("table") // TODO: add yaml - .possible_values(["table", "json", "csv", "parquet", "arrow", "arrows"]), + .value_parser(["table", "json", "csv", "parquet", "arrow", "arrows"]), table_arg(), ]), ) diff --git a/columnq/Cargo.toml b/columnq/Cargo.toml index 7793a6225..9d8a53e4e 100644 --- a/columnq/Cargo.toml +++ b/columnq/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "columnq" -version = "0.7.0" +version = "0.8.1" homepage = "https://github.com/roapi/roapi" license = "MIT" authors = ["QP Hou "] @@ -13,10 +13,10 @@ path = "src/lib.rs" [dependencies] # pulling arrow-schema manually to enable the serde feature. # TODO: add serde feature in datafusion to avoid this workaround -arrow-schema = { version = "46.0.0", features = ["serde"] } +arrow-schema = { version = "50.0.0", features = ["serde"] } -datafusion = "31" -object_store = { version = "0.7", features = ["gcp", "azure"] } +datafusion = "35" +object_store = { version = "0", features = ["aws", "gcp", "azure"] } percent-encoding = "2.2.0" url = "2.2" @@ -24,7 +24,7 @@ log = "0" regex = "1" lazy_static = "1" graphql-parser = "0" -sqlparser = "0.37" # version need to be in sync with convergence and datafusion +sqlparser = "0.41" # version need to be in sync with convergence and datafusion yup-oauth2 = { version = "6.2", default-features = false, features = [ "service_account", ] } @@ -39,30 +39,30 @@ reqwest = { version = "0.11", default-features = false, features = [ "blocking", "json", ] } -calamine = "0.19.1" +calamine = {version = "0.23.1", features = ["dates"]} tokio = { version = "1", features = ["rt-multi-thread"] } futures = "0.3" hyper-tls = { version = "0.5.0", default-features = false, optional = true } -hyper-rustls = { version = "0.23.2", default-features = false, optional = true } +hyper-rustls = { version = "0.24.1", default-features = false, optional = true } tokio-postgres = { version = "0.7.8", optional = true } [dependencies.deltalake] -version = "0.15" -git = "https://github.com/delta-io/delta-rs.git" -rev = "63c14b3716428ff65e01404c6f7e62f341c98f05" +version = "0.17" +# git = "https://github.com/delta-io/delta-rs.git" +# rev = "63c14b3716428ff65e01404c6f7e62f341c98f05" features = ["datafusion"] default-features = false [dependencies.connectorx] git = "https://github.com/roapi/connector-x.git" -rev = "1e642af502d3076476ab4bc8c7e40807059a42fe" +rev = "7cfe911133afd56a5fa69b24d11f8b5f7283796c" version = "0.3.3-alpha.1" features = ["default", "dst_arrow"] optional = true [dev-dependencies] -serde_yaml = "0.8" +serde_yaml = "0.9" toml = "0.7" tempfile = "3.3.0" pretty_assertions = "*" @@ -92,7 +92,6 @@ native-tls = [ "deltalake/azure", "yup-oauth2/hyper-tls", ] -simd = ["datafusion/simd"] database-sqlite = ["connectorx/src_sqlite"] database-mysql = ["connectorx/src_mysql"] database-postgres = ["connectorx/src_postgres", "dep:tokio-postgres"] diff --git a/columnq/src/columnq.rs b/columnq/src/columnq.rs index 42531d8bb..7e484d3f7 100644 --- a/columnq/src/columnq.rs +++ b/columnq/src/columnq.rs @@ -1,6 +1,5 @@ use std::collections::hash_map::Entry; use std::collections::HashMap; -use std::convert::TryFrom; use std::sync::Arc; use datafusion::arrow; @@ -41,10 +40,17 @@ impl ColumnQ { } pub fn new_with_config(config: SessionConfig) -> Self { - let config = config.with_default_catalog_and_schema("roapi", "public"); + let config = config + .with_default_catalog_and_schema("roapi", "public") + // TODO: fix bug in datafusion to support partitioned table when + // listing_table_ignore_subdirectory is set to false + .set_bool( + "datafusion.execution.listing_table_ignore_subdirectory", + false, + ); let rn_config = RuntimeConfig::new(); let runtime_env = RuntimeEnv::new(rn_config).unwrap(); - let dfctx = SessionContext::with_config_rt(config, Arc::new(runtime_env)); + let dfctx = SessionContext::new_with_config_rt(config, Arc::new(runtime_env)); let schema_map = HashMap::::new(); Self { @@ -57,8 +63,31 @@ impl ColumnQ { pub async fn load_table(&mut self, t: &TableSource) -> Result<(), ColumnQError> { match &t.io_source { TableIoSource::Uri(uri_str) => { - if let Ok(url) = Url::parse(uri_str) { - let _ = self.register_object_storage(&url); + match Url::parse(uri_str) { + Err(url::ParseError::RelativeUrlWithoutBase) => { + // assume file path for relative url without scheme or base + // no need to register object store in this case + } + Ok(url) => { + match self.register_object_storage(&url) { + Ok(_) + | Err(ColumnQError::IoError { + source: crate::io::Error::InvalidUriScheme { .. }, + }) => { + // invalid Uri scheme can be caused by non objectstore related + // uris, for example sqlite://, so safe to ignore for now. + // + // TODO: ideally, we still propagate error if it's an invalid + // objectstore uri + } + Err(e) => { + return Err(e); + } + } + } + Err(e) => { + return Err(ColumnQError::from(e)); + } } } TableIoSource::Memory(_) => {} diff --git a/columnq/src/encoding/json.rs b/columnq/src/encoding/json.rs index 3773a9729..b3f0854d2 100644 --- a/columnq/src/encoding/json.rs +++ b/columnq/src/encoding/json.rs @@ -25,7 +25,6 @@ mod tests { use datafusion::arrow::array::*; use datafusion::arrow::datatypes::*; - use datafusion::arrow::record_batch::*; use pretty_assertions::assert_eq; #[test] diff --git a/columnq/src/encoding/mod.rs b/columnq/src/encoding/mod.rs index 4df4587a5..7c2e054e6 100644 --- a/columnq/src/encoding/mod.rs +++ b/columnq/src/encoding/mod.rs @@ -1,5 +1,4 @@ use serde_derive::Deserialize; -use std::convert::TryFrom; #[derive(Deserialize, Default, Clone, Copy)] pub enum ContentType { diff --git a/columnq/src/error.rs b/columnq/src/error.rs index 5aeae0927..9daa5c844 100644 --- a/columnq/src/error.rs +++ b/columnq/src/error.rs @@ -39,8 +39,8 @@ pub enum ColumnQError { #[error("Error loading Delta table: {0}")] LoadDelta(String), - #[error("Error loading Xlsx table: {0}")] - LoadXlsx(String), + #[error("Error loading Excel table: {0}")] + LoadExcel(String), #[error("Error loading data from HTTP store: {0}")] HttpStore(String), @@ -95,6 +95,12 @@ pub enum ColumnQError { #[error("Invalid URI: {0}")] InvalidUri(String), + + #[error("Invalid URL: {source}")] + InvalidUrl { + #[from] + source: url::ParseError, + }, } impl ColumnQError { diff --git a/columnq/src/io/mod.rs b/columnq/src/io/mod.rs index bc6b71d1b..ed8cc9024 100644 --- a/columnq/src/io/mod.rs +++ b/columnq/src/io/mod.rs @@ -1,5 +1,3 @@ -use std::convert::TryFrom; - use snafu::prelude::*; pub mod fs; @@ -65,7 +63,6 @@ impl TryFrom<&str> for BlobStoreType { #[cfg(test)] mod tests { use crate::io::BlobStoreType; - use std::convert::TryFrom; use uriparse::*; #[test] diff --git a/columnq/src/io/object_store.rs b/columnq/src/io/object_store.rs index 3bda5a3e5..742f59448 100644 --- a/columnq/src/io/object_store.rs +++ b/columnq/src/io/object_store.rs @@ -2,7 +2,6 @@ use crate::table::TableSource; use futures::TryStreamExt; use log::debug; use object_store::ObjectStore; -use percent_encoding; use snafu::prelude::*; use std::str::FromStr; use std::sync::Arc; @@ -117,14 +116,10 @@ where let paths = client .clone() .list(Some(&path)) - .await - .context(ListObjectSnafu { - path: path.to_owned(), - })? .map_ok(|meta| meta.location) .try_collect::>() .await - .unwrap(); + .context(ListObjectSnafu { path })?; for f in paths { let reader = partition_key_to_reader(client.clone(), &f).await?; partitions.push(partition_reader(reader).context(TableSnafu)?); diff --git a/columnq/src/query/graphql.rs b/columnq/src/query/graphql.rs index d26ae12e4..3d0b3d0c0 100644 --- a/columnq/src/query/graphql.rs +++ b/columnq/src/query/graphql.rs @@ -1,5 +1,3 @@ -use std::convert::TryFrom; - use datafusion::arrow; use datafusion::logical_expr::Operator; use datafusion::prelude::{binary_expr, Column, Expr}; diff --git a/columnq/src/query/rest.rs b/columnq/src/query/rest.rs index 571837bfe..63adf4563 100644 --- a/columnq/src/query/rest.rs +++ b/columnq/src/query/rest.rs @@ -172,7 +172,6 @@ mod tests { use super::*; use datafusion::arrow::array::*; - use datafusion::execution::context::SessionContext; use datafusion::prelude::*; use crate::test_util::*; diff --git a/columnq/src/table/arrow_ipc_file.rs b/columnq/src/table/arrow_ipc_file.rs index 4a472c749..cb7edd44d 100644 --- a/columnq/src/table/arrow_ipc_file.rs +++ b/columnq/src/table/arrow_ipc_file.rs @@ -75,6 +75,7 @@ pub async fn to_mem_table( mod tests { use super::*; + use datafusion::common::stats::Precision; use datafusion::datasource::TableProvider; use datafusion::prelude::SessionContext; use std::fs; @@ -112,8 +113,9 @@ mod tests { .scan(&ctx.state(), None, &[], None) .await .unwrap() - .statistics(); - assert_eq!(stats.num_rows, Some(37 * 3)); + .statistics() + .unwrap(); + assert_eq!(stats.num_rows, Precision::Exact(37 * 3)); } #[tokio::test] @@ -129,7 +131,8 @@ mod tests { .scan(&ctx.state(), None, &[], None) .await .unwrap() - .statistics(); - assert_eq!(stats.num_rows, Some(37)); + .statistics() + .unwrap(); + assert_eq!(stats.num_rows, Precision::Exact(37)); } } diff --git a/columnq/src/table/arrow_ipc_stream.rs b/columnq/src/table/arrow_ipc_stream.rs index ee788c0c1..b01e915fd 100644 --- a/columnq/src/table/arrow_ipc_stream.rs +++ b/columnq/src/table/arrow_ipc_stream.rs @@ -75,6 +75,7 @@ pub async fn to_mem_table( mod tests { use super::*; + use datafusion::common::stats::Precision; use datafusion::datasource::TableProvider; use datafusion::prelude::SessionContext; use std::fs; @@ -112,8 +113,9 @@ mod tests { .scan(&ctx.state(), None, &[], None) .await .unwrap() - .statistics(); - assert_eq!(stats.num_rows, Some(37 * 3)); + .statistics() + .unwrap(); + assert_eq!(stats.num_rows, Precision::Exact(37 * 3)); } #[tokio::test] @@ -129,7 +131,8 @@ mod tests { .scan(&ctx.state(), None, &[], None) .await .unwrap() - .statistics(); - assert_eq!(stats.num_rows, Some(37)); + .statistics() + .unwrap(); + assert_eq!(stats.num_rows, Precision::Exact(37)); } } diff --git a/columnq/src/table/csv.rs b/columnq/src/table/csv.rs index 3d8632789..42f90551a 100644 --- a/columnq/src/table/csv.rs +++ b/columnq/src/table/csv.rs @@ -11,7 +11,9 @@ use datafusion::datasource::TableProvider; use log::debug; use snafu::prelude::*; -use crate::table::{self, TableLoadOption, TableOptionCsv, TableSource}; +use crate::table::{ + self, datafusion_get_or_infer_schema, TableLoadOption, TableOptionCsv, TableSource, +}; #[derive(Debug, Snafu)] pub enum Error { @@ -41,21 +43,30 @@ pub async fn to_datafusion_table( .option .clone() .unwrap_or_else(|| TableLoadOption::csv(TableOptionCsv::default())); - if opt.as_csv().unwrap().use_memory_table { + if opt + .as_csv() + .expect("Invalid table format option, expect csv") + .use_memory_table + { return to_mem_table(t, dfctx).await; } let table_url = ListingTableUrl::parse(t.get_uri_str()).with_context(|_| table::ListingTableUriSnafu { uri: t.get_uri_str().to_string(), })?; - let options = ListingOptions::new(Arc::new(CsvFormat::default())); - let schemaref = match &t.schema { - Some(s) => Arc::new(s.into()), - None => options - .infer_schema(&dfctx.state(), &table_url) - .await - .context(table::InferListingTableSchemaSnafu)?, - }; + let mut options = ListingOptions::new(Arc::new(CsvFormat::default())); + if let Some(partition_cols) = t.datafusion_partition_cols() { + options = options.with_table_partition_cols(partition_cols) + } + + let schemaref = datafusion_get_or_infer_schema( + dfctx, + &table_url, + &options, + &t.schema, + &t.schema_from_files, + ) + .await?; let table_config = ListingTableConfig::new(table_url) .with_listing_options(options) @@ -64,6 +75,7 @@ pub async fn to_datafusion_table( ListingTable::try_new(table_config).context(table::CreateListingTableSnafu)?, )) } + pub async fn to_mem_table( t: &TableSource, dfctx: &datafusion::execution::context::SessionContext, @@ -120,7 +132,7 @@ pub async fn to_mem_table( t, |r| -> Result, table::Error> { let mut builder = arrow::csv::reader::ReaderBuilder::new(schema_ref.clone()) - .has_header(has_header) + .with_header(has_header) .with_delimiter(delimiter) .with_batch_size(batch_size); if let Some(p) = projection { @@ -151,11 +163,12 @@ pub async fn to_mem_table( #[cfg(test)] mod tests { use super::*; + use datafusion::common::stats::Precision; use datafusion::prelude::SessionContext; use std::fs; use tempfile::Builder; - use crate::table::{TableIoSource, TableLoadOption}; + use crate::table::TableIoSource; use crate::test_util::*; #[tokio::test] @@ -189,8 +202,9 @@ mod tests { .scan(&ctx.state(), None, &[], None) .await .unwrap() - .statistics(); - assert_eq!(stats.num_rows, Some(37 * 3)); + .statistics() + .unwrap(); + assert_eq!(stats.num_rows, Precision::Exact(37 * 3)); } #[tokio::test] @@ -214,7 +228,8 @@ c1,c2,c3 .scan(&ctx.state(), None, &[], None) .await .unwrap() - .statistics(); - assert_eq!(stats.num_rows, Some(3)); + .statistics() + .unwrap(); + assert_eq!(stats.num_rows, Precision::Exact(3)); } } diff --git a/columnq/src/table/database.rs b/columnq/src/table/database.rs index 1fe5c8c07..9c2932db9 100644 --- a/columnq/src/table/database.rs +++ b/columnq/src/table/database.rs @@ -10,8 +10,6 @@ pub enum DatabaseLoader { feature = "database-postgres" ))] mod imp { - use std::convert::TryFrom; - use crate::table::{self, TableSource}; use connectorx::prelude::*; use log::debug; diff --git a/columnq/src/table/delta.rs b/columnq/src/table/delta.rs index 21b828d17..71475342f 100644 --- a/columnq/src/table/delta.rs +++ b/columnq/src/table/delta.rs @@ -4,13 +4,11 @@ use datafusion::datasource::TableProvider; use datafusion::parquet::arrow::arrow_reader::ArrowReaderOptions; use datafusion::parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use snafu::prelude::*; -use std::convert::{TryFrom, TryInto}; use std::io::Read; use std::sync::Arc; use crate::io::{self, BlobStoreType}; use crate::table::{self, TableLoadOption, TableOptionDelta, TableSource}; -use deltalake; #[derive(Debug, Snafu)] pub enum Error { @@ -44,6 +42,10 @@ pub enum Error { OpenTable { source: deltalake::errors::DeltaTableError, }, + #[snafu(display("Failed to load table: {source}"))] + LoadTable { + source: deltalake::errors::DeltaTableError, + }, } pub async fn to_datafusion_table( @@ -120,7 +122,12 @@ pub async fn to_mem_table( batch_size: usize, dfctx: &datafusion::execution::context::SessionContext, ) -> Result, table::Error> { - if delta_table.get_files().is_empty() { + let paths = delta_table + .get_file_uris() + .context(LoadTableSnafu) + .context(table::LoadDeltaSnafu)? + .collect::>(); + if paths.is_empty() { return Err(Error::EmptyTable {}).context(table::LoadDeltaSnafu); } @@ -129,7 +136,6 @@ pub async fn to_mem_table( .context(GetSchemaSnafu) .context(table::LoadDeltaSnafu)?; - let paths = delta_table.get_file_uris().collect::>(); let path_iter = paths.iter().map(|s| s.as_ref()); let partitions: Vec> = match blob_type { @@ -177,6 +183,7 @@ pub async fn to_mem_table( mod tests { use super::*; + use datafusion::common::stats::Precision; use datafusion::datasource::MemTable; use datafusion::physical_plan::Statistics; use datafusion::prelude::SessionContext; @@ -203,7 +210,8 @@ mod tests { t.scan(&ctx.state(), None, &[], None) .await .unwrap() - .statistics(), + .statistics() + .unwrap(), ); if t.as_any().downcast_ref::().is_none() { @@ -234,10 +242,10 @@ mod tests { } fn validate_statistics(stats: Statistics) { - assert_eq!(stats.num_rows, Some(500)); - let column_stats = stats.column_statistics.unwrap(); - assert_eq!(column_stats[0].null_count, Some(245)); - assert_eq!(column_stats[1].null_count, Some(373)); - assert_eq!(column_stats[2].null_count, Some(237)); + assert_eq!(stats.num_rows, Precision::Exact(500)); + let column_stats = stats.column_statistics; + assert_eq!(column_stats[0].null_count, Precision::Exact(245)); + assert_eq!(column_stats[1].null_count, Precision::Exact(373)); + assert_eq!(column_stats[2].null_count, Precision::Exact(237)); } } diff --git a/columnq/src/table/excel.rs b/columnq/src/table/excel.rs new file mode 100644 index 000000000..301543cd2 --- /dev/null +++ b/columnq/src/table/excel.rs @@ -0,0 +1,775 @@ +use crate::table::{self, TableOptionExcel, TableSchema, TableSource}; +use arrow_schema::TimeUnit; +use calamine::{open_workbook_auto, DataType as ExcelDataType, Range, Reader, Sheets}; +use datafusion::arrow::array::{ + ArrayRef, BooleanArray, DurationSecondArray, NullArray, PrimitiveArray, StringArray, + TimestampSecondArray, +}; +use datafusion::arrow::datatypes::{ + DataType, Date32Type, Date64Type, Field, Float64Type, Int64Type, Schema, +}; +use datafusion::arrow::record_batch::RecordBatch; +use snafu::prelude::*; +use std::collections::HashMap; +use std::sync::Arc; +use std::vec; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("Failed to load Excel: {msg}"))] + Load { msg: String }, + #[snafu(display("Incorrect schema: {msg}"))] + IncorrectSchema { msg: String }, + #[snafu(display("Excel schema inference error"))] + SchemaInference, + #[snafu(display("Failed to create record batch: {source}"))] + CreateRecordBatch { + source: datafusion::arrow::error::ArrowError, + }, + #[snafu(display("Failed to open workbook: {source}"))] + OpenWorkbook { source: calamine::Error }, +} + +struct ExcelSubrange<'a> { + rows: calamine::Rows<'a, ExcelDataType>, + columns_range_start: usize, + columns_range_end: usize, + total_rows: usize, + current_row_id: usize, +} + +impl<'a> ExcelSubrange<'a> { + fn new( + range: &'a Range, + rows_range_start: Option, + rows_range_end: Option, + columns_range_start: Option, + columns_range_end: Option, + ) -> ExcelSubrange { + let rows_range_start = rows_range_start.unwrap_or(usize::MIN); + let rows_range_end = rows_range_end + .or(range.end().map(|v| v.0 as usize)) + .unwrap(); + + let mut rows = range.rows(); + if rows_range_start > 0 { + // rows skipping + rows.nth(rows_range_start - 1); + } + + ExcelSubrange { + rows, + columns_range_start: columns_range_start.unwrap_or(usize::MIN), + columns_range_end: columns_range_end.unwrap_or(usize::MAX), + total_rows: rows_range_end - rows_range_start + 1, + current_row_id: 0, + } + } + + fn size(&self) -> usize { + self.total_rows + } +} + +impl<'a> Iterator for ExcelSubrange<'a> { + type Item = &'a [ExcelDataType]; + + fn next(&mut self) -> Option { + if self.current_row_id < self.total_rows { + self.current_row_id += 1; + self.rows + .next() + .map(|x| &x[self.columns_range_start..=self.columns_range_end.min(x.len() - 1)]) + } else { + None + } + } + + fn size_hint(&self) -> (usize, Option) { + (0, Some(self.total_rows)) + } +} + +fn infer_value_type(v: &ExcelDataType) -> Result { + match v { + ExcelDataType::Int(_) => Ok(DataType::Int64), + ExcelDataType::Float(_) => Ok(DataType::Float64), + ExcelDataType::String(_) => Ok(DataType::Utf8), + ExcelDataType::Bool(_) => Ok(DataType::Boolean), + ExcelDataType::DateTime(_) | ExcelDataType::DateTimeIso(_) => { + Ok(DataType::Timestamp(TimeUnit::Second, None)) + } + ExcelDataType::Duration(_) | ExcelDataType::DurationIso(_) => { + Ok(DataType::Duration(TimeUnit::Second)) + } + ExcelDataType::Error(e) => Err(Error::Load { msg: e.to_string() }), + ExcelDataType::Empty => Ok(DataType::Null), + } +} + +fn infer_schema_from_data(mut range: ExcelSubrange) -> Result { + let mut col_types: HashMap<&str, DataType> = HashMap::new(); + let col_names: Vec<&str> = range + .next() + .ok_or(Error::Load { + msg: String::from("Failed to infer schema for empty excel table"), + })? + .iter() + .enumerate() + .map(|(i, c)| { + c.get_string().ok_or_else(|| Error::Load { + msg: format!("The {i}th column name is empty"), + }) + }) + .collect::, _>>()?; + + for row in range { + for (i, col_val) in row.iter().enumerate() { + let col_name = col_names.get(i).ok_or(Error::Load { + msg: String::from( + "Failed to infer schema. Number of values in row is more then column names.", + ), + })?; + let col_type = infer_value_type(col_val)?; + col_types + .entry(col_name) + .and_modify(|ct| { + if !ct.equals_datatype(&col_type) && ct.equals_datatype(&DataType::Null) { + *ct = col_type.clone(); + } + // if column values has more than one not null type then we upcast column type to the most general datatype Utf8. + else if !ct.equals_datatype(&col_type) + && !&col_type.equals_datatype(&DataType::Null) + { + *ct = DataType::Utf8; + } + }) + .or_insert(col_type); + } + } + + let fields: Vec = col_names + .iter() + .map(|col_name| { + let dt = col_types.get(col_name).unwrap_or(&DataType::Utf8).clone(); + Field::new(col_name.replace(' ', "_"), dt, true) + }) + .collect(); + Ok(Schema::new(fields)) +} + +fn infer_schema_from_config(table_schema: &TableSchema) -> Result { + let unsupported_data_types = table_schema + .columns + .iter() + .filter(|c| { + !matches!( + c.data_type, + DataType::Boolean + | DataType::Int64 + | DataType::Float64 + | DataType::Duration(TimeUnit::Second) + | DataType::Date32 + | DataType::Date64 + | DataType::Null + | DataType::Utf8 + | DataType::Timestamp(TimeUnit::Second, None) + ) + }) + .map(|c| c.name.clone()) + .collect::>() + .join(", "); + + if unsupported_data_types.is_empty() { + Ok(table_schema.into()) + } else { + Err(Error::IncorrectSchema{msg: format!("Configured schema for excel file contains unsupported data types in columns {}. Supported datatype: \ + Boolean, Int64, Float64, Date32, Date64, !Timestamp [Second, null], !Duration [Second], Null, Utf8", unsupported_data_types)}) + } +} + +fn empty_or_panic(v: &ExcelDataType, field_name: &String) -> Option { + if v.is_empty() { + None + } else { + panic!("Incorrect value {:?} in column {}", v, field_name) + } +} + +fn infer_schema( + r: &Range, + option: &TableOptionExcel, + schema: &Option, +) -> Result { + let TableOptionExcel { + rows_range_start, + rows_range_end, + columns_range_start, + columns_range_end, + schema_inference_lines, + .. + } = *option; + + if let Some(schema) = schema { + infer_schema_from_config(schema) + } else { + let last_row_for_schema_inference = schema_inference_lines + .map(|r| r + rows_range_start.unwrap_or(0)) + .or(rows_range_end); + + let range = ExcelSubrange::new( + r, + rows_range_start, + last_row_for_schema_inference, + columns_range_start, + columns_range_end, + ); + infer_schema_from_data(range) + } +} + +fn excel_range_to_record_batch( + r: Range, + option: &TableOptionExcel, + schema: Schema, +) -> Result { + let TableOptionExcel { + rows_range_start, + rows_range_end, + columns_range_start, + columns_range_end, + .. + } = *option; + + let arrays = schema + .fields() + .iter() + .enumerate() + .map(|(i, field)| { + let rows = ExcelSubrange::new( + &r, + rows_range_start.map(|x| x + 1).or(Some(1)), // skip first row because it is header + rows_range_end, + columns_range_start, + columns_range_end, + ); + let field_name = field.name(); + + match field.data_type() { + DataType::Boolean => Arc::new( + rows.map(|r| { + r.get(i) + .and_then(|v| v.get_bool().or_else(|| empty_or_panic(v, field_name))) + }) + .collect::(), + ) as ArrayRef, + DataType::Int64 => Arc::new( + rows.map(|r| { + r.get(i) + .and_then(|v| v.get_int().or_else(|| empty_or_panic(v, field_name))) + }) + .collect::>(), + ) as ArrayRef, + DataType::Float64 => Arc::new( + rows.map(|r| { + r.get(i) + .and_then(|v| v.get_float().or_else(|| empty_or_panic(v, field_name))) + }) + .collect::>(), + ) as ArrayRef, + DataType::Duration(TimeUnit::Second) => Arc::new( + rows.map(|r| { + r.get(i).and_then(|v| { + v.as_duration() + .map(|v| v.num_seconds()) + .or_else(|| empty_or_panic(v, field_name)) + }) + }) + .collect::(), + ) as ArrayRef, + DataType::Null => Arc::new(NullArray::new(rows.size())) as ArrayRef, + DataType::Utf8 => Arc::new( + rows.map(|r| { + r.get(i).and_then(|v| match v { + ExcelDataType::Bool(x) => Some(x.to_string()), + ExcelDataType::Float(_) + | ExcelDataType::Int(_) + | ExcelDataType::String(_) => v.as_string(), + ExcelDataType::DateTime(_) | ExcelDataType::DateTimeIso(_) => { + v.as_datetime().map(|x| x.to_string()) + } + ExcelDataType::Duration(_) | ExcelDataType::DurationIso(_) => { + v.as_duration().map(|x| x.to_string()) + } + ExcelDataType::Empty => None, + ExcelDataType::Error(e) => Some(e.to_string()), + }) + }) + .collect::(), + ) as ArrayRef, + DataType::Timestamp(TimeUnit::Second, None) => Arc::new( + rows.map(|r| { + r.get(i).and_then(|v| { + v.as_datetime() + .map(|v| v.and_utc().timestamp()) + .or_else(|| empty_or_panic(v, field_name)) + }) + }) + .collect::(), + ) as ArrayRef, + DataType::Date64 => Arc::new( + rows.map(|r| { + r.get(i).and_then(|v| { + v.as_datetime() + .map(|v| v.timestamp_millis()) + .or_else(|| empty_or_panic(v, field_name)) + }) + }) + .collect::>(), + ) as ArrayRef, + DataType::Date32 => Arc::new( + rows.map(|r| { + r.get(i).and_then(|v| { + v.as_datetime() + .map(|v| (v.timestamp() / 86400) as i32) + .or_else(|| empty_or_panic(v, field_name)) + }) + }) + .collect::>(), + ) as ArrayRef, + unsupported => panic!("Unsupported data type for excel table {:?}", unsupported), + } + }) + .collect::>(); + + RecordBatch::try_new(Arc::new(schema), arrays).context(CreateRecordBatchSnafu) +} + +pub async fn to_mem_table( + t: &TableSource, +) -> Result { + let opt = t + .option + .as_ref() + .ok_or(table::Error::MissingOption {})? + .as_excel()?; + let uri = t.get_uri_str(); + let mut workbook: Sheets<_> = open_workbook_auto(uri) + .context(OpenWorkbookSnafu) + .context(table::LoadExcelSnafu)?; + + let worksheet_range = match &opt.sheet_name { + Some(sheet) => Some(workbook.worksheet_range(sheet)), + None => workbook.worksheet_range_at(0), + }; + + if let Some(Ok(range)) = worksheet_range { + let shema = infer_schema(&range, opt, &t.schema).context(table::LoadExcelSnafu)?; + let batch = + excel_range_to_record_batch(range, opt, shema).context(table::LoadExcelSnafu)?; + let schema_ref = batch.schema(); + let partitions = vec![vec![batch]]; + + datafusion::datasource::MemTable::try_new(schema_ref, partitions) + .context(table::CreateMemTableSnafu) + } else { + Err(Error::Load { + msg: "Failed to open excel file.".to_owned(), + }) + .context(table::LoadExcelSnafu) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::arrow::array::{Float64Array, Int64Array}; + use crate::table::{TableColumn, TableIoSource}; + use crate::test_util::*; + use datafusion::common::stats::Precision; + use datafusion::datasource::TableProvider; + use datafusion::prelude::SessionContext; + + use calamine::{Cell, DataType as ExcelDataType}; + + #[test] + fn excel_subrange_iteration() { + let range = calamine::Range::::from_sparse(vec![ + Cell::new((0, 0), ExcelDataType::Int(0)), + Cell::new((0, 1), ExcelDataType::Bool(true)), + Cell::new((0, 2), ExcelDataType::Float(0.333)), + Cell::new((1, 0), ExcelDataType::Int(1)), + Cell::new((1, 1), ExcelDataType::Bool(false)), + Cell::new((1, 2), ExcelDataType::Float(1.333)), + Cell::new((2, 0), ExcelDataType::Int(2)), + Cell::new((2, 1), ExcelDataType::Empty), + Cell::new((2, 2), ExcelDataType::Float(2.333)), + Cell::new((3, 0), ExcelDataType::Int(3)), + Cell::new((3, 1), ExcelDataType::Bool(true)), + Cell::new((3, 2), ExcelDataType::Float(3.333)), + ]); + let mut subrange = ExcelSubrange::new(&range, None, None, None, None); + assert_eq!(subrange.size(), 4); + assert_eq!( + subrange.next(), + Some( + &vec![ + ExcelDataType::Int(0), + ExcelDataType::Bool(true), + ExcelDataType::Float(0.333) + ][..] + ) + ); + assert_eq!( + subrange.next(), + Some( + &vec![ + ExcelDataType::Int(1), + ExcelDataType::Bool(false), + ExcelDataType::Float(1.333) + ][..] + ) + ); + assert_eq!( + subrange.next(), + Some( + &vec![ + ExcelDataType::Int(2), + ExcelDataType::Empty, + ExcelDataType::Float(2.333) + ][..] + ) + ); + assert_eq!( + subrange.next(), + Some( + &vec![ + ExcelDataType::Int(3), + ExcelDataType::Bool(true), + ExcelDataType::Float(3.333) + ][..] + ) + ); + assert_eq!(subrange.next(), None); + + let mut subrange = ExcelSubrange::new(&range, Some(1), Some(2), Some(1), Some(1)); + assert_eq!(subrange.size(), 2); + assert_eq!(subrange.next(), Some(&vec![ExcelDataType::Bool(false)][..])); + assert_eq!(subrange.next(), Some(&vec![ExcelDataType::Empty][..])); + assert_eq!(subrange.next(), None); + } + + #[test] + fn inferes_schema_from_data() { + let range = calamine::Range::::from_sparse(vec![ + Cell::new((0, 0), ExcelDataType::String(String::from("int_column"))), + Cell::new((0, 1), ExcelDataType::String(String::from("bool_column"))), + Cell::new((0, 2), ExcelDataType::String(String::from("float column"))), + Cell::new((0, 3), ExcelDataType::String(String::from("string_column"))), + Cell::new( + (0, 4), + ExcelDataType::String(String::from("datetime_column")), + ), + Cell::new( + (0, 5), + ExcelDataType::String(String::from("datetime iso column")), + ), + Cell::new( + (0, 6), + ExcelDataType::String(String::from("duration column")), + ), + Cell::new( + (0, 7), + ExcelDataType::String(String::from("duration iso column")), + ), + Cell::new((1, 0), ExcelDataType::Int(0)), + Cell::new((1, 1), ExcelDataType::Bool(true)), + Cell::new((1, 2), ExcelDataType::Float(0.333)), + Cell::new((1, 3), ExcelDataType::String(String::from("test"))), + Cell::new((1, 4), ExcelDataType::DateTime(44986.12)), + Cell::new((1, 5), ExcelDataType::DateTimeIso(String::from("test"))), + Cell::new((1, 6), ExcelDataType::Duration(44986.12)), + Cell::new((1, 7), ExcelDataType::DurationIso(String::from("test"))), + Cell::new((2, 0), ExcelDataType::Empty), + Cell::new((2, 0), ExcelDataType::Empty), + Cell::new((2, 1), ExcelDataType::Empty), + Cell::new((2, 2), ExcelDataType::Empty), + Cell::new((2, 3), ExcelDataType::Empty), + Cell::new((2, 4), ExcelDataType::Empty), + Cell::new((2, 5), ExcelDataType::Empty), + Cell::new((2, 6), ExcelDataType::Empty), + Cell::new((2, 7), ExcelDataType::Empty), + ]); + + let schema = infer_schema(&range, &TableOptionExcel::default(), &None).unwrap(); + + assert_eq!( + schema, + Schema::new(vec![ + Field::new("int_column", DataType::Int64, true), + Field::new("bool_column", DataType::Boolean, true), + Field::new("float_column", DataType::Float64, true), + Field::new("string_column", DataType::Utf8, true), + Field::new( + "datetime_column", + DataType::Timestamp(TimeUnit::Second, None), + true + ), + Field::new( + "datetime_iso_column", + DataType::Timestamp(TimeUnit::Second, None), + true + ), + Field::new( + "duration_column", + DataType::Duration(TimeUnit::Second), + true + ), + Field::new( + "duration_iso_column", + DataType::Duration(TimeUnit::Second), + true + ), + ]) + ); + + let range = calamine::Range::::from_sparse(vec![ + Cell::new((0, 0), ExcelDataType::String(String::from("test_column"))), + Cell::new((1, 0), ExcelDataType::Int(0)), + Cell::new((2, 0), ExcelDataType::Empty), + Cell::new((2, 0), ExcelDataType::Float(0.5)), + ]); + + let schema = infer_schema(&range, &TableOptionExcel::default(), &None).unwrap(); + + assert_eq!( + schema, + Schema::new(vec![Field::new("test_column", DataType::Utf8, true)]) + ); + + let range = calamine::Range::::from_sparse(vec![ + Cell::new((0, 0), ExcelDataType::String(String::from("int_column"))), + Cell::new((0, 1), ExcelDataType::Empty), + Cell::new((0, 2), ExcelDataType::String(String::from("float column"))), + ]); + + assert!(infer_schema(&range, &TableOptionExcel::default(), &None).is_err()); + + let range = calamine::Range::::from_sparse(vec![ + Cell::new((0, 0), ExcelDataType::String(String::from("column1"))), + Cell::new((0, 1), ExcelDataType::String(String::from("column2"))), + Cell::new((1, 0), ExcelDataType::Int(1)), + Cell::new((1, 1), ExcelDataType::Int(1)), + Cell::new((1, 3), ExcelDataType::Int(1)), + ]); + assert!(infer_schema(&range, &TableOptionExcel::default(), &None).is_err()); + } + + #[test] + fn inferes_schema_from_config() { + let range = calamine::Range::::from_sparse(vec![]); + let table_schema = TableSchema { + columns: vec![ + TableColumn { + name: String::from("float_column"), + data_type: DataType::Float64, + nullable: true, + }, + TableColumn { + name: String::from("integer_column"), + data_type: DataType::Int64, + nullable: true, + }, + ], + }; + let schema = + infer_schema(&range, &TableOptionExcel::default(), &Some(table_schema)).unwrap(); + + assert_eq!( + schema.all_fields(), + vec![ + &Field::new("float_column", DataType::Float64, true), + &Field::new("integer_column", DataType::Int64, true), + ] + ); + + let table_schema = TableSchema { + columns: vec![ + TableColumn { + name: String::from("float_column"), + data_type: DataType::Float16, + nullable: true, + }, + TableColumn { + name: String::from("integer_column"), + data_type: DataType::Int16, + nullable: true, + }, + ], + }; + assert!(infer_schema(&range, &TableOptionExcel::default(), &Some(table_schema)).is_err()); + } + + #[tokio::test] + async fn load_xlsx_with_toml_config() { + let mut table_source: TableSource = toml::from_str( + r#" +name = "test" +uri = "test_data/uk_cities_with_headers.xlsx" +[option] +format = "xlsx" +sheet_name = "uk_cities_with_headers" +"#, + ) + .unwrap(); + // patch uri path with the correct test data path + table_source.io_source = TableIoSource::Uri(test_data_path("uk_cities_with_headers.xlsx")); + + let t = to_mem_table(&table_source).await.unwrap(); + let ctx = SessionContext::new(); + let stats = t + .scan(&ctx.state(), None, &[], None) + .await + .unwrap() + .statistics() + .unwrap(); + assert_eq!(stats.num_rows, Precision::Exact(37)); + } + + #[tokio::test] + async fn load_xlsx_with_yaml_config() { + let mut table_source: TableSource = serde_yaml::from_str( + r#" +name: "test" +uri: "test_data/uk_cities_with_headers.xlsx" +option: + format: "xlsx" + sheet_name: "uk_cities_with_headers" +"#, + ) + .unwrap(); + // patch uri path with the correct test data path + table_source.io_source = TableIoSource::Uri(test_data_path("uk_cities_with_headers.xlsx")); + + let t = to_mem_table(&table_source).await.unwrap(); + let ctx = SessionContext::new(); + let stats = t + .scan(&ctx.state(), None, &[], None) + .await + .unwrap() + .statistics() + .unwrap(); + assert_eq!(stats.num_rows, Precision::Exact(37)); + } + + #[tokio::test] + async fn load_ods_with_custom_range_and_without_sheet_name() { + let mut table_source: TableSource = serde_yaml::from_str( + r#" +name: "test" +uri: "test_data/excel_range.ods" +option: + format: "ods" + rows_range_start: 2 + rows_range_end: 5 + columns_range_start: 1 + columns_range_end: 6 + schema_inference_lines: 3 +"#, + ) + .unwrap(); + // patch uri path with the correct test data path + table_source.io_source = TableIoSource::Uri(test_data_path("excel_range.ods")); + + let t = to_mem_table(&table_source).await.unwrap(); + let ctx = SessionContext::new(); + let stats = t + .scan(&ctx.state(), None, &[], None) + .await + .unwrap() + .statistics() + .unwrap(); + assert_eq!(stats.column_statistics.len(), 6); + assert_eq!(stats.num_rows, Precision::Exact(3)); + } + + #[test] + fn transforms_excel_range_to_record_batch() { + let range: calamine::Range = + calamine::Range::::from_sparse(vec![ + Cell::new((0, 0), ExcelDataType::String("float_column".to_string())), + Cell::new((1, 0), ExcelDataType::Float(1.333)), + Cell::new((2, 0), ExcelDataType::Empty), + Cell::new((3, 0), ExcelDataType::Float(3.333)), + Cell::new((0, 1), ExcelDataType::String("integer_column".to_string())), + Cell::new((1, 1), ExcelDataType::Int(1)), + Cell::new((2, 1), ExcelDataType::Int(3)), + Cell::new((3, 1), ExcelDataType::Empty), + Cell::new((0, 2), ExcelDataType::String("boolean_column".to_string())), + Cell::new((1, 2), ExcelDataType::Empty), + Cell::new((2, 2), ExcelDataType::Bool(true)), + Cell::new((3, 2), ExcelDataType::Bool(false)), + Cell::new((0, 3), ExcelDataType::String("string_column".to_string())), + Cell::new((1, 3), ExcelDataType::String("foo".to_string())), + Cell::new((2, 3), ExcelDataType::String("bar".to_string())), + Cell::new((3, 3), ExcelDataType::String("baz".to_string())), + Cell::new((0, 4), ExcelDataType::String("mixed_column".to_string())), + Cell::new((1, 4), ExcelDataType::Float(1.1)), + Cell::new((2, 4), ExcelDataType::Int(1)), + Cell::new((3, 4), ExcelDataType::Empty), + Cell::new((0, 5), ExcelDataType::String("datetime_column".to_string())), + Cell::new((1, 5), ExcelDataType::DateTime(44986.12)), // 2023-03-01T02:52:48 + Cell::new((2, 5), ExcelDataType::Empty), + Cell::new((3, 5), ExcelDataType::DateTime(44900.12)), // 2022-12-05T02:52:48 + ]); + + let shema = infer_schema(&range, &TableOptionExcel::default(), &None).unwrap(); + let rb = excel_range_to_record_batch(range, &TableOptionExcel::default(), shema).unwrap(); + + assert_eq!( + rb.schema().all_fields(), + vec![ + &Field::new("float_column", DataType::Float64, true), + &Field::new("integer_column", DataType::Int64, true), + &Field::new("boolean_column", DataType::Boolean, true), + &Field::new("string_column", DataType::Utf8, true), + &Field::new("mixed_column", DataType::Utf8, true), + &Field::new( + "datetime_column", + DataType::Timestamp(TimeUnit::Second, None), + true + ), + ] + ); + + assert_eq!( + rb.column(0).as_ref(), + Arc::new(Float64Array::from(vec![Some(1.333), None, Some(3.333)])).as_ref(), + ); + assert_eq!( + rb.column(1).as_ref(), + Arc::new(Int64Array::from(vec![Some(1), Some(3), None])).as_ref(), + ); + assert_eq!( + rb.column(2).as_ref(), + Arc::new(BooleanArray::from(vec![None, Some(true), Some(false)])).as_ref(), + ); + assert_eq!( + rb.column(3).as_ref(), + Arc::new(StringArray::from(vec!["foo", "bar", "baz"])).as_ref(), + ); + assert_eq!( + rb.column(4).as_ref(), + Arc::new(StringArray::from(vec![Some("1.1"), Some("1"), None])).as_ref(), + ); + assert_eq!( + rb.column(5).as_ref(), + Arc::new(TimestampSecondArray::from(vec![ + Some(1677639168), // Unix timestamp for 2023-03-01T02:52:48 UTC + None, + Some(1670208768) // Unix timestamp for 2022-12-05T02:52:48 UTC + ])) + .as_ref(), + ); + } +} diff --git a/columnq/src/table/google_spreadsheets.rs b/columnq/src/table/google_spreadsheets.rs index 6b06707f7..7ea1ecc04 100644 --- a/columnq/src/table/google_spreadsheets.rs +++ b/columnq/src/table/google_spreadsheets.rs @@ -1,5 +1,4 @@ use std::collections::{HashMap, HashSet}; -use std::convert::TryFrom; use std::sync::Arc; use datafusion::arrow::array::{ArrayRef, BooleanArray, PrimitiveArray, StringArray}; @@ -372,7 +371,7 @@ pub async fn to_mem_table( #[cfg(test)] mod tests { use super::*; - use datafusion::arrow::array::{BooleanArray, Int64Array}; + use datafusion::arrow::array::Int64Array; fn row(raw: &[&str]) -> Vec { raw.iter().map(|s| s.to_string()).collect() diff --git a/columnq/src/table/mod.rs b/columnq/src/table/mod.rs index 117bd5d88..1acc38631 100644 --- a/columnq/src/table/mod.rs +++ b/columnq/src/table/mod.rs @@ -1,4 +1,3 @@ -use std::convert::TryFrom; use std::ffi::OsStr; use std::io::Read; use std::path::Path; @@ -7,6 +6,7 @@ use std::sync::Arc; use datafusion::datasource::TableProvider; use datafusion::arrow; +use datafusion::datasource::listing::{ListingOptions, ListingTableUrl}; use serde::de::{Deserialize, Deserializer}; use serde_derive::Deserialize; use snafu::prelude::*; @@ -20,11 +20,11 @@ pub mod arrow_ipc_stream; pub mod csv; pub mod database; pub mod delta; +pub mod excel; pub mod google_spreadsheets; pub mod json; pub mod ndjson; pub mod parquet; -pub mod xlsx; #[derive(Debug, Snafu)] pub enum Error { @@ -44,8 +44,8 @@ pub enum Error { LoadArrowIpcFile { source: arrow_ipc_file::Error }, #[snafu(display("Failed to load Google Sheet data: {source}"))] LoadGoogleSheet { source: google_spreadsheets::Error }, - #[snafu(display("Failed to load XLSX data: {source}"))] - LoadXlsx { source: xlsx::Error }, + #[snafu(display("Failed to load Excel data: {source}"))] + LoadExcel { source: excel::Error }, #[snafu(display("Failed to load database data: {source}"))] LoadDatabase { source: database::Error }, #[snafu(display("Failed to cast IO source to memory bytes for source: {table_source}"))] @@ -85,6 +85,10 @@ pub enum Error { MissingOption {}, #[snafu(display("Generic error: {msg}"))] Generic { msg: String }, + #[snafu(display("Failed to infer table schema: {source}"))] + InferSchema { + source: datafusion::error::DataFusionError, + }, } #[derive(Deserialize, Clone, Debug, Eq, PartialEq)] @@ -148,14 +152,14 @@ pub struct TableOptionGoogleSpreadsheet { #[derive(Deserialize, Debug, Clone, Eq, PartialEq)] pub struct TableOptionCsv { #[serde(default = "TableOptionCsv::default_has_header")] - has_header: bool, + pub has_header: bool, #[serde(default = "TableOptionCsv::default_delimiter")] #[serde(deserialize_with = "TableOptionCsv::deserialize_delimiter")] - delimiter: u8, + pub delimiter: u8, #[serde(default = "TableOptionCsv::default_projection")] - projection: Option>, + pub projection: Option>, #[serde(default = "TableOptionCsv::default_use_memory_table")] - use_memory_table: bool, + pub use_memory_table: bool, } impl TableOptionCsv { @@ -188,6 +192,13 @@ impl TableOptionCsv { self } + #[inline] + #[must_use] + pub fn with_use_memory_table(mut self, use_memory: bool) -> Self { + self.use_memory_table = use_memory; + self + } + fn deserialize_delimiter<'de, D>(deserializer: D) -> Result where D: Deserializer<'de>, @@ -200,6 +211,7 @@ impl TableOptionCsv { )), } } + #[inline] pub fn default_use_memory_table() -> bool { true @@ -220,7 +232,7 @@ impl Default for TableOptionCsv { #[derive(Deserialize, Debug, Clone, Eq, PartialEq)] pub struct TableOptionParquet { #[serde(default = "TableOptionParquet::default_use_memory_table")] - use_memory_table: bool, + pub use_memory_table: bool, } impl TableOptionParquet { @@ -238,15 +250,20 @@ impl Default for TableOptionParquet { } } -#[derive(Deserialize, Debug, Clone, Eq, PartialEq)] -pub struct TableOptionXlsx { - sheet_name: Option, +#[derive(Deserialize, Default, Debug, Clone, Eq, PartialEq)] +pub struct TableOptionExcel { + pub sheet_name: Option, + pub rows_range_start: Option, + pub rows_range_end: Option, + pub columns_range_start: Option, + pub columns_range_end: Option, + pub schema_inference_lines: Option, } #[derive(Deserialize, Debug, Clone, Eq, PartialEq)] pub struct TableOptionDelta { #[serde(default = "TableOptionDelta::default_use_memory_table")] - use_memory_table: bool, + pub use_memory_table: bool, } impl TableOptionDelta { @@ -285,7 +302,10 @@ pub enum TableLoadOption { jsonl {}, parquet(TableOptionParquet), google_spreadsheet(TableOptionGoogleSpreadsheet), - xlsx(TableOptionXlsx), + xls(TableOptionExcel), + xlsx(TableOptionExcel), + xlsb(TableOptionExcel), + ods(TableOptionExcel), delta(TableOptionDelta), arrow {}, arrows {}, @@ -295,7 +315,7 @@ pub enum TableLoadOption { } impl TableLoadOption { - fn as_google_spreadsheet(&self) -> Result<&TableOptionGoogleSpreadsheet, Error> { + pub fn as_google_spreadsheet(&self) -> Result<&TableOptionGoogleSpreadsheet, Error> { match self { Self::google_spreadsheet(opt) => Ok(opt), _ => Err(Error::ExpectFormatOption { @@ -304,28 +324,28 @@ impl TableLoadOption { } } - fn as_xlsx(&self) -> Result<&TableOptionXlsx, Error> { + pub fn as_excel(&self) -> Result<&TableOptionExcel, Error> { match self { - Self::xlsx(opt) => Ok(opt), - _ => Err(Error::ExpectFormatOption { fmt: "xlsx" }), + Self::xls(opt) | Self::xlsx(opt) | Self::xlsb(opt) | Self::ods(opt) => Ok(opt), + _ => Err(Error::ExpectFormatOption { fmt: "excel" }), } } - fn as_csv(&self) -> Result<&TableOptionCsv, Error> { + pub fn as_csv(&self) -> Result<&TableOptionCsv, Error> { match self { Self::csv(opt) => Ok(opt), _ => Err(Error::ExpectFormatOption { fmt: "csv" }), } } - fn as_parquet(&self) -> Result<&TableOptionParquet, Error> { + pub fn as_parquet(&self) -> Result<&TableOptionParquet, Error> { match self { Self::parquet(opt) => Ok(opt), _ => Err(Error::ExpectFormatOption { fmt: "parquet" }), } } - fn as_delta(&self) -> Result<&TableOptionDelta, Error> { + pub fn as_delta(&self) -> Result<&TableOptionDelta, Error> { match self { Self::delta(opt) => Ok(opt), _ => Err(Error::ExpectFormatOption { fmt: "delta" }), @@ -340,7 +360,10 @@ impl TableLoadOption { Self::csv { .. } => "csv", Self::parquet { .. } => "parquet", Self::google_spreadsheet(_) | Self::delta { .. } => "", + Self::xls { .. } => "xls", Self::xlsx { .. } => "xlsx", + Self::ods { .. } => "ods", + Self::xlsb { .. } => "xlsb", Self::arrow { .. } => "arrow", Self::arrows { .. } => "arrows", Self::mysql { .. } => "mysql", @@ -390,9 +413,11 @@ pub struct TableSource { #[serde(flatten)] pub io_source: TableIoSource, pub schema: Option, + pub schema_from_files: Option>, pub option: Option, #[serde(default = "TableSource::default_batch_size")] pub batch_size: usize, + pub partition_columns: Option>, } impl From for TableSource { @@ -401,8 +426,10 @@ impl From for TableSource { name: kv.name, io_source: kv.io_source, schema: kv.schema, + schema_from_files: kv.schema_from_files, option: kv.option, batch_size: Self::default_batch_size(), + partition_columns: None, } } } @@ -415,11 +442,21 @@ impl TableSource { name: name.into(), io_source, schema: None, + schema_from_files: None, option, batch_size: Self::default_batch_size(), + partition_columns: None, } } + pub fn datafusion_partition_cols(&self) -> Option> { + self.partition_columns.as_ref().map(|cols| { + cols.iter() + .map(|col| (col.name.to_string(), col.data_type.clone())) + .collect::>() + }) + } + pub fn new_with_uri(name: impl Into, uri: impl Into) -> Self { Self::new(name, uri.into()) } @@ -443,6 +480,18 @@ impl TableSource { self } + #[inline] + #[must_use] + pub fn with_partition_columns(mut self, partitions: Vec) -> Self { + self.partition_columns = Some(partitions); + self + } + + pub fn with_schema_from_files(mut self, files: Vec) -> Self { + self.schema_from_files = Some(files); + self + } + pub fn get_uri_str(&self) -> &str { match &self.io_source { TableIoSource::Uri(uri) => uri.as_str(), @@ -498,7 +547,7 @@ impl TableSource { match Path::new(uri).extension().and_then(OsStr::to_str) { Some(ext) => match ext { "csv" | "json" | "ndjson" | "jsonl" | "parquet" | "arrow" | "arrows" - | "xlsx" => ext, + | "xls" | "xlsx" | "xlsb" | "ods" => ext, "sqlite" | "sqlite3" | "db" => "sqlite", _ => { return Err(Error::Extension { @@ -530,6 +579,50 @@ impl TableSource { } } +pub async fn datafusion_get_or_infer_schema( + dfctx: &datafusion::execution::context::SessionContext, + table_url: &ListingTableUrl, + listing_options: &ListingOptions, + schema: &Option, + schema_from_files: &Option>, +) -> Result { + Ok(match (schema, schema_from_files) { + (Some(s), _) => Arc::new(s.into()), + (None, Some(files)) => { + if files.is_empty() { + return Err(Error::Generic { + msg: "schema_from_files is an empty list".to_string(), + }); + } + let mut schemas = vec![]; + let table_root = Path::new(table_url.as_str()); + for f in files.iter() { + let file_url = ListingTableUrl::parse( + table_root + .join(f.as_str()) + .to_str() + .expect("Failed to create file url"), + ) + .context(InferSchemaSnafu)?; + schemas.push( + Arc::into_inner( + listing_options + .infer_schema(&dfctx.state(), &file_url) + .await + .context(InferSchemaSnafu)?, + ) + .expect("Failed to unwrap schemaref into schema on merge"), + ); + } + Arc::new(arrow::datatypes::Schema::try_merge(schemas).context(MergeSchemaSnafu)?) + } + (None, None) => listing_options + .infer_schema(&dfctx.state(), table_url) + .await + .context(InferSchemaSnafu)?, + }) +} + pub async fn load( t: &TableSource, dfctx: &datafusion::execution::context::SessionContext, @@ -545,7 +638,10 @@ pub async fn load( TableLoadOption::google_spreadsheet(_) => { Arc::new(google_spreadsheets::to_mem_table(t).await?) } - TableLoadOption::xlsx { .. } => Arc::new(xlsx::to_mem_table(t).await?), + TableLoadOption::xlsx { .. } + | TableLoadOption::xls { .. } + | TableLoadOption::xlsb { .. } + | TableLoadOption::ods { .. } => Arc::new(excel::to_mem_table(t).await?), TableLoadOption::delta { .. } => delta::to_datafusion_table(t, dfctx).await?, TableLoadOption::arrow { .. } => { Arc::new(arrow_ipc_file::to_mem_table(t, dfctx).await?) @@ -569,6 +665,7 @@ pub async fn load( "json" => Arc::new(json::to_mem_table(t, dfctx).await?), "ndjson" | "jsonl" => Arc::new(ndjson::to_mem_table(t, dfctx).await?), "parquet" => parquet::to_datafusion_table(t, dfctx).await?, + "xls" | "xlsx" | "xlsb" | "ods" => Arc::new(excel::to_mem_table(t).await?), "arrow" => Arc::new(arrow_ipc_file::to_mem_table(t, dfctx).await?), "arrows" => Arc::new(arrow_ipc_stream::to_mem_table(t, dfctx).await?), "mysql" => Arc::new(database::DatabaseLoader::MySQL.to_mem_table(t)?), @@ -662,6 +759,7 @@ pub struct KeyValueSource { #[serde(flatten)] pub io_source: TableIoSource, pub schema: Option, + pub schema_from_files: Option>, pub option: Option, } @@ -678,6 +776,7 @@ impl KeyValueSource { value: value.into(), io_source: source.into(), schema: None, + schema_from_files: None, option: None, } } @@ -740,6 +839,28 @@ batch_size: 512 assert_eq!(table_source.batch_size, 512); } + #[test] + fn deserialize_datetime() { + let table_source: TableSource = serde_yaml::from_str( + r#" +name: "ts_table" +uri: "test_data/a.parquet" +schema: + columns: + - name: "ts" + data_type: !Timestamp [!Second, null] +"#, + ) + .unwrap(); + + use arrow::datatypes::*; + + assert_eq!( + DataType::Timestamp(TimeUnit::Second, None), + table_source.schema.unwrap().columns[0].data_type, + ); + } + #[test] fn test_parse_table_uri() { let t = parse_table_uri_arg("t=a/b/c").unwrap(); @@ -767,6 +888,8 @@ batch_size: 512 #[cfg(feature = "database-sqlite")] #[tokio::test] async fn test_load_sqlite_table() { + use datafusion::common::stats::Precision; + let t = TableSource::new("uk_cities", "sqlite://../test_data/sqlite/sample.db"); let ctx = datafusion::prelude::SessionContext::new(); let table = load(&t, &ctx).await.unwrap(); @@ -774,13 +897,16 @@ batch_size: 512 .scan(&ctx.state(), None, &[], None) .await .unwrap() - .statistics(); - assert_eq!(stats.num_rows, Some(37)); + .statistics() + .unwrap(); + assert_eq!(stats.num_rows, Precision::Exact(37)); } #[cfg(feature = "database-sqlite")] #[tokio::test] async fn test_load_sqlite_table_with_config() { + use datafusion::common::stats::Precision; + for ext in vec!["db", "sqlite", "sqlite3"] { let t: TableSource = serde_yaml::from_str(&format!( r#" @@ -796,8 +922,9 @@ uri: "sqlite://../test_data/sqlite/sample.{}" .scan(&ctx.state(), None, &[], None) .await .unwrap() - .statistics(); - assert_eq!(stats.num_rows, Some(37)); + .statistics() + .unwrap(); + assert_eq!(stats.num_rows, Precision::Exact(37)); } } } diff --git a/columnq/src/table/ndjson.rs b/columnq/src/table/ndjson.rs index 78f64f2e4..7886939af 100644 --- a/columnq/src/table/ndjson.rs +++ b/columnq/src/table/ndjson.rs @@ -29,9 +29,10 @@ pub enum Error { fn json_schema_from_reader(r: R) -> Result { let mut reader = BufReader::new(r); - infer_json_schema(&mut reader, None) + let (schema, _) = infer_json_schema(&mut reader, None) .context(InferSchemaSnafu) - .context(table::LoadNdJsonSnafu) + .context(table::LoadNdJsonSnafu)?; + Ok(schema) } fn decode_json_from_reader( @@ -93,7 +94,7 @@ mod tests { use datafusion::{datasource::TableProvider, prelude::SessionContext}; use super::*; - use crate::{table::TableSource, test_util::test_data_path}; + use crate::test_util::test_data_path; #[tokio::test] async fn load_simple_ndjson_file() { diff --git a/columnq/src/table/parquet.rs b/columnq/src/table/parquet.rs index ab1f4b404..35da7147b 100644 --- a/columnq/src/table/parquet.rs +++ b/columnq/src/table/parquet.rs @@ -14,7 +14,9 @@ use datafusion::parquet::arrow::arrow_reader::ArrowReaderOptions; use datafusion::parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use snafu::prelude::*; -use crate::table::{self, TableLoadOption, TableOptionParquet, TableSource}; +use crate::table::{ + self, datafusion_get_or_infer_schema, TableLoadOption, TableOptionParquet, TableSource, +}; #[derive(Debug, Snafu)] pub enum Error { @@ -36,10 +38,6 @@ pub enum Error { EmptyPartition {}, #[snafu(display("Schema not found"))] SchemaNotFound {}, - #[snafu(display("Failed to infer table schema: {source}"))] - InferSchema { - source: datafusion::error::DataFusionError, - }, #[snafu(display("Failed to parse table uri: {source}"))] ParseUri { source: datafusion::error::DataFusionError, @@ -62,15 +60,19 @@ pub async fn to_datafusion_table( let table_url = ListingTableUrl::parse(t.get_uri_str()) .context(ParseUriSnafu) .context(table::LoadParquetSnafu)?; - let options = ListingOptions::new(Arc::new(ParquetFormat::default())); - let schemaref = match &t.schema { - Some(s) => Arc::new(s.into()), - None => options - .infer_schema(&dfctx.state(), &table_url) - .await - .context(InferSchemaSnafu) - .context(table::LoadParquetSnafu)?, - }; + let mut options = ListingOptions::new(Arc::new(ParquetFormat::default())); + if let Some(partition_cols) = t.datafusion_partition_cols() { + options = options.with_table_partition_cols(partition_cols) + } + + let schemaref = datafusion_get_or_infer_schema( + dfctx, + &table_url, + &options, + &t.schema, + &t.schema_from_files, + ) + .await?; let table_config = ListingTableConfig::new(table_url) .with_listing_options(options) @@ -155,8 +157,8 @@ mod tests { use std::fs; use tempfile::Builder; - use crate::table::TableLoadOption; use crate::test_util::*; + use datafusion::common::stats::Precision; use datafusion::prelude::SessionContext; #[tokio::test] @@ -179,12 +181,13 @@ mod tests { .scan(&ctx.state(), None, &[], None) .await .unwrap() - .statistics(); - assert_eq!(stats.num_rows, Some(500)); - let stats = stats.column_statistics.unwrap(); - assert_eq!(stats[0].null_count, Some(245)); - assert_eq!(stats[1].null_count, Some(373)); - assert_eq!(stats[2].null_count, Some(237)); + .statistics() + .unwrap(); + assert_eq!(stats.num_rows, Precision::Exact(500)); + let stats = stats.column_statistics; + assert_eq!(stats[0].null_count, Precision::Exact(245)); + assert_eq!(stats[1].null_count, Precision::Exact(373)); + assert_eq!(stats[2].null_count, Precision::Exact(237)); match t.as_any().downcast_ref::() { Some(_) => {} @@ -206,8 +209,9 @@ mod tests { .scan(&ctx.state(), None, &[], None) .await .unwrap() - .statistics(); - assert_eq!(stats.num_rows, Some(500)); + .statistics() + .unwrap(); + assert_eq!(stats.num_rows, Precision::Exact(500)); } #[tokio::test] @@ -236,7 +240,8 @@ mod tests { .scan(&ctx.state(), None, &[], None) .await .unwrap() - .statistics(); - assert_eq!(stats.num_rows, Some(1500)); + .statistics() + .unwrap(); + assert_eq!(stats.num_rows, Precision::Exact(1500)); } } diff --git a/columnq/src/table/xlsx.rs b/columnq/src/table/xlsx.rs deleted file mode 100644 index 3efc3303c..000000000 --- a/columnq/src/table/xlsx.rs +++ /dev/null @@ -1,272 +0,0 @@ -use crate::table::{self, TableSource}; -use calamine::{open_workbook, Range, Reader, Xlsx}; -use datafusion::arrow::array::{ArrayRef, BooleanArray, PrimitiveArray, StringArray}; -use datafusion::arrow::datatypes::{DataType, Field, Float64Type, Int64Type, Schema}; -use datafusion::arrow::record_batch::RecordBatch; -use snafu::prelude::*; -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; -use std::vec; - -#[derive(Debug, Snafu)] -pub enum Error { - #[snafu(display("Failed to load XLSX: {msg}"))] - Load { msg: String }, - #[snafu(display("Failed to create record batch: {source}"))] - CreateRecordBatch { - source: datafusion::arrow::error::ArrowError, - }, - #[snafu(display("Failed to open workbook: {source}"))] - OpenWorkbook { source: calamine::XlsxError }, -} - -fn infer_value_type(v: &calamine::DataType) -> Result { - match v { - calamine::DataType::Int(_) if v.get_int().is_some() => Ok(DataType::Int64), - calamine::DataType::Float(_) if v.get_float().is_some() => Ok(DataType::Float64), - calamine::DataType::Bool(_) if v.get_bool().is_some() => Ok(DataType::Boolean), - calamine::DataType::String(_) if v.get_string().is_some() => Ok(DataType::Utf8), - calamine::DataType::Error(e) => Err(Error::Load { msg: e.to_string() }), - // TODO(upstream): support `Date64` - calamine::DataType::DateTime(_) => Err(Error::Load { - msg: "Unsupported data type: DateTime".to_owned(), - }), - calamine::DataType::Empty => Ok(DataType::Null), - _ => Err(Error::Load { - msg: "Failed to parse the cell value".to_owned(), - }), - } -} - -fn infer_schema(r: &Range) -> Result { - let mut col_types: HashMap<&str, HashSet> = HashMap::new(); - let mut rows = r.rows(); - let col_names: Result, _> = rows - .next() - .unwrap() - .iter() - .enumerate() - .map(|(i, c)| { - c.get_string().ok_or_else(|| Error::Load { - msg: format!("The {i}th column name is empty"), - }) - }) - .collect(); - - let col_names = match col_names { - Ok(values) => values, - Err(e) => return Err(e), - }; - - for row in rows { - for (i, col_val) in row.iter().enumerate() { - let col_name = col_names.get(i).unwrap(); - let col_type = infer_value_type(col_val).unwrap(); - let entry = col_types.entry(col_name).or_default(); - entry.insert(col_type); - } - } - - let fields: Vec = col_names - .iter() - .map(|col_name| { - let set = col_types.entry(col_name).or_insert_with(|| { - let mut set = HashSet::new(); - set.insert(DataType::Utf8); - set - }); - - let mut dt_iter = set.iter().cloned(); - let dt = dt_iter.next().unwrap_or(DataType::Utf8); - Field::new(col_name.replace(' ', "_"), dt, true) - }) - .collect(); - Ok(Schema::new(fields)) -} - -fn xlsx_sheet_value_to_record_batch(r: Range) -> Result { - let schema = infer_schema(&r)?; - let arrays = schema - .fields() - .iter() - .enumerate() - .map(|(i, field)| { - let rows = r.rows().skip(1); - match field.data_type() { - DataType::Boolean => Arc::new( - rows.map(|r| r.get(i).map(|v| v.get_bool().unwrap())) - .collect::(), - ) as ArrayRef, - DataType::Int64 => Arc::new( - rows.map(|r| r.get(i).map(|v| v.get_int().unwrap())) - .collect::>(), - ) as ArrayRef, - DataType::Float64 => Arc::new( - rows.map(|r| r.get(i).map(|v| v.get_float().unwrap())) - .collect::>(), - ) as ArrayRef, - _ => Arc::new( - rows.map(|r| r.get(i).map(|v| v.get_string().unwrap_or("null"))) - .collect::(), - ) as ArrayRef, - } - }) - .collect::>(); - - RecordBatch::try_new(Arc::new(schema), arrays).context(CreateRecordBatchSnafu) -} - -pub async fn to_mem_table( - t: &TableSource, -) -> Result { - let opt = t - .option - .as_ref() - .ok_or(table::Error::MissingOption {})? - .as_xlsx()?; - let uri = t.get_uri_str(); - let mut workbook: Xlsx<_> = open_workbook(uri) - .context(OpenWorkbookSnafu) - .context(table::LoadXlsxSnafu)?; - match &opt.sheet_name { - Some(sheet) => { - if let Some(Ok(r)) = workbook.worksheet_range(sheet) { - let batch = xlsx_sheet_value_to_record_batch(r).context(table::LoadXlsxSnafu)?; - let schema_ref = batch.schema(); - let partitions = vec![vec![batch]]; - Ok( - datafusion::datasource::MemTable::try_new(schema_ref, partitions) - .context(table::CreateMemTableSnafu)?, - ) - } else { - Err(Error::Load { - msg: "Failed to open .xlsx file.".to_owned(), - }) - .context(table::LoadXlsxSnafu) - } - } - None => Err(Error::Load { - msg: "`sheet_name` is not specified".to_owned(), - }) - .context(table::LoadXlsxSnafu), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::arrow::array::{BooleanArray, Float64Array, Int64Array, StringArray}; - use crate::table::TableIoSource; - use crate::test_util::*; - use datafusion::datasource::TableProvider; - use datafusion::prelude::SessionContext; - - use calamine::{Cell, DataType as XlsxDataType, Range}; - - fn property_sheet() -> Range { - let cells: Vec> = vec![ - Cell::new((0, 0), XlsxDataType::String("float_column".to_string())), - Cell::new((1, 0), XlsxDataType::Float(1.333)), - Cell::new((2, 0), XlsxDataType::Float(3.333)), - Cell::new((0, 1), XlsxDataType::String("integer_column".to_string())), - Cell::new((1, 1), XlsxDataType::Int(1)), - Cell::new((2, 1), XlsxDataType::Int(3)), - Cell::new((0, 2), XlsxDataType::String("boolean_column".to_string())), - Cell::new((1, 2), XlsxDataType::Bool(true)), - Cell::new((2, 2), XlsxDataType::Bool(false)), - Cell::new((0, 3), XlsxDataType::String("string_column".to_string())), - Cell::new((1, 3), XlsxDataType::String("foo".to_string())), - Cell::new((2, 3), XlsxDataType::String("bar".to_string())), - ]; - calamine::Range::::from_sparse(cells) - } - - #[tokio::test] - async fn load_xlsx_with_toml_config() { - let mut table_source: TableSource = toml::from_str( - r#" -name = "test" -uri = "test_data/uk_cities_with_headers.xlsx" -[option] -format = "xlsx" -sheet_name = "uk_cities_with_headers" -"#, - ) - .unwrap(); - // patch uri path with the correct test data path - table_source.io_source = TableIoSource::Uri(test_data_path("uk_cities_with_headers.xlsx")); - - let t = to_mem_table(&table_source).await.unwrap(); - let ctx = SessionContext::new(); - let stats = t - .scan(&ctx.state(), None, &[], None) - .await - .unwrap() - .statistics(); - assert_eq!(stats.num_rows, Some(37)); - } - - #[tokio::test] - async fn load_xlsx_with_yaml_config() { - let mut table_source: TableSource = serde_yaml::from_str( - r#" -name: "test" -uri: "test_data/uk_cities_with_headers.xlsx" -option: - format: "xlsx" - sheet_name: "uk_cities_with_headers" -"#, - ) - .unwrap(); - // patch uri path with the correct test data path - table_source.io_source = TableIoSource::Uri(test_data_path("uk_cities_with_headers.xlsx")); - - let t = to_mem_table(&table_source).await.unwrap(); - let ctx = SessionContext::new(); - let stats = t - .scan(&ctx.state(), None, &[], None) - .await - .unwrap() - .statistics(); - assert_eq!(stats.num_rows, Some(37)); - } - - #[test] - fn schema_interface() { - let sheet = property_sheet(); - let schema = infer_schema(&sheet).unwrap(); - assert_eq!( - schema, - Schema::new(vec![ - Field::new("float_column", DataType::Float64, true), - Field::new("integer_column", DataType::Int64, true), - Field::new("boolean_column", DataType::Boolean, true), - Field::new("string_column", DataType::Utf8, true), - ]) - ); - } - - #[test] - fn xlsx_value_to_record_batch() { - let sheet = property_sheet(); - let rb = xlsx_sheet_value_to_record_batch(sheet).unwrap(); - - assert_eq!(rb.num_columns(), 4); - assert_eq!( - rb.column(0).as_ref(), - Arc::new(Float64Array::from(vec![1.333, 3.333])).as_ref(), - ); - assert_eq!( - rb.column(1).as_ref(), - Arc::new(Int64Array::from(vec![1, 3])).as_ref(), - ); - assert_eq!( - rb.column(2).as_ref(), - Arc::new(BooleanArray::from(vec![true, false])).as_ref(), - ); - assert_eq!( - rb.column(3).as_ref(), - Arc::new(StringArray::from(vec!["foo", "bar"])).as_ref(), - ); - } -} diff --git a/columnq/src/test_util.rs b/columnq/src/test_util.rs index 1e4ea636c..3d895a305 100644 --- a/columnq/src/test_util.rs +++ b/columnq/src/test_util.rs @@ -3,7 +3,6 @@ use std::sync::Arc; use datafusion::arrow::array::*; use datafusion::arrow::datatypes::{DataType, Field, Schema}; -use datafusion::arrow::record_batch::RecordBatch; use datafusion::dataframe::DataFrame; use datafusion::datasource::MemTable; use datafusion::execution::context::SessionContext; diff --git a/columnq/tests/helpers.rs b/columnq/tests/helpers.rs new file mode 100644 index 000000000..e44e599a3 --- /dev/null +++ b/columnq/tests/helpers.rs @@ -0,0 +1,8 @@ +use std::path::PathBuf; + +pub fn test_data_path(relative_path: &str) -> String { + let mut d = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + d.push("../test_data"); + d.push(relative_path); + d.to_string_lossy().to_string() +} diff --git a/columnq/tests/table_csv_test.rs b/columnq/tests/table_csv_test.rs new file mode 100644 index 000000000..34bdf501f --- /dev/null +++ b/columnq/tests/table_csv_test.rs @@ -0,0 +1,56 @@ +mod helpers; + +use std::sync::Arc; + +use datafusion::arrow; +use datafusion::prelude::SessionContext; + +use columnq::table::csv::to_datafusion_table; +use columnq::table::{TableIoSource, TableLoadOption, TableOptionCsv, TableSource}; + +#[tokio::test] +async fn infer_csv_schema_by_selected_files() { + use arrow::datatypes::{DataType, Field, Schema, SchemaBuilder}; + + let table_path = helpers::test_data_path("partitioned_csv"); + + let ctx = SessionContext::new(); + + let table_io_source = TableIoSource::Uri(table_path); + let table_source = TableSource::new("test", table_io_source) + .with_schema_from_files(vec![]) + .with_option(TableLoadOption::csv( + TableOptionCsv::default().with_use_memory_table(false), + )); + assert!( + !table_source + .option + .as_ref() + .unwrap() + .as_csv() + .unwrap() + .use_memory_table + ); + assert!(table_source.schema_from_files.is_some()); + assert_eq!(table_source.schema, None); + + match to_datafusion_table(&table_source, &ctx).await { + Err(columnq::table::Error::Generic { msg }) => { + assert_eq!(&msg, "schema_from_files is an empty list"); + } + _ => panic!("Empty schema_from_files should result in an error"), + } + + let t = to_datafusion_table( + &table_source.with_schema_from_files(vec!["year=2023/month=1/p001.csv".to_string()]), + &ctx, + ) + .await + .unwrap(); + + let mut builder = SchemaBuilder::new(); + builder.push(Field::new("ts", DataType::Int64, true)); + builder.push(Field::new("value", DataType::Float64, true)); + + assert_eq!(t.schema(), Arc::new(Schema::new(builder.finish().fields))); +} diff --git a/columnq/tests/table_mysql_test.rs b/columnq/tests/table_mysql_test.rs index e2b954ad2..99f290b45 100644 --- a/columnq/tests/table_mysql_test.rs +++ b/columnq/tests/table_mysql_test.rs @@ -20,8 +20,9 @@ mod mysql { .scan(&ctx.state(), None, &[], None) .await .unwrap() - .statistics(); - assert!(stats.num_rows.is_some()); + .statistics() + .unwrap(); + assert!(stats.num_rows.get_value().is_some()); } } } diff --git a/columnq/tests/table_postgres_test.rs b/columnq/tests/table_postgres_test.rs index e03d11d49..85e795e0c 100644 --- a/columnq/tests/table_postgres_test.rs +++ b/columnq/tests/table_postgres_test.rs @@ -20,8 +20,9 @@ mod postgres { .scan(&ctx.state(), None, &[], None) .await .unwrap() - .statistics(); - assert!(stats.num_rows.is_some()); + .statistics() + .unwrap(); + assert!(stats.num_rows.get_value().is_some()); } } } diff --git a/roapi/Cargo.toml b/roapi/Cargo.toml index d2f12773e..0b09c933c 100644 --- a/roapi/Cargo.toml +++ b/roapi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "roapi" -version = "0.10.0" +version = "0.11.1" authors = ["QP Hou "] homepage = "https://github.com/roapi/roapi" license = "MIT" @@ -21,28 +21,32 @@ snmalloc-rs = { version = "0.2", optional = true } # dependencies related to axum tokio = { version = "1", features = ["rt-multi-thread"] } -hyper = { version = "0", features = ["http1", "server", "stream", "runtime"] } +hyper = { version = "0", features = ["http1", "server"] } axum = { version = "0.6", features = ["default", "http2"] } -tower-http = { version = "0", features = ["cors"] } +tower = { version = "0" } # introduced only for tower::layer::util::Stack +tower-http = { version = "0.4.4", features = ["cors", "trace"] } tower-layer = "0" +tower-service = "0" tracing = "0" pin-project-lite = "0" async-trait = "0" +constant_time_eq = "0" +base64 = "0.22" env_logger = "0" log = "0" serde = { version = "1", features = ["rc"] } serde_json = "1" serde_derive = "1" -serde_yaml = "0.8" +serde_yaml = "0.9" toml = "0.7" -clap = { version = "3", features = ["color"] } +clap = { version = "4", features = ["color"] } thiserror = "1" snafu = "0" # flight-sql -arrow-flight = { version = "46", features = ["flight-sql-experimental"] } -tonic = { version = "0", features = ["tls"] } +arrow-flight = { version = "50", features = ["flight-sql-experimental"] } +tonic = { version = "0.10", features = ["tls"] } prost = "0" futures = "0" # TODO: remove once_cell dependency @@ -50,25 +54,28 @@ once_cell = "*" dashmap = "5" uuid = "1" -# convergence = "0.13" -# convergence-arrow = "0.13" -# patch for datafusion 31 -convergence = { git = "https://github.com/roapi/convergence", rev = "95fe472e429d02f18016a0d03c884a78fa62861e" } -convergence-arrow = { git = "https://github.com/roapi/convergence", rev = "95fe472e429d02f18016a0d03c884a78fa62861e" } +[dependencies.convergence] +version = "0" +git = "https://github.com/returnString/convergence.git" +rev = "3d61a5b9bbf9848b075f5b3ae791c90e5380e5e3" + +[dependencies.convergence-arrow] +version = "0" +git = "https://github.com/returnString/convergence.git" +rev = "3d61a5b9bbf9848b075f5b3ae791c90e5380e5e3" [features] default = ["rustls", "snmalloc"] rustls = ["columnq/rustls"] native-tls-vendored = ["columnq/native-tls-vendored"] native-tls = ["columnq/native-tls"] -simd = ["columnq/simd"] snmalloc = ["snmalloc-rs"] database = ["columnq/database"] database-sqlite = ["columnq/database-sqlite"] database-mysql = ["columnq/database-mysql"] [dev-dependencies] -reqwest = { version = "0.11", default-features = false, features = [ +reqwest = { version = "0", default-features = false, features = [ "json", "rustls-tls", ] } @@ -77,8 +84,8 @@ tokio-postgres = "0.7" tower = "*" tempfile = "*" tokio-stream = { version = "*", features = ["net"] } -arrow-cast = "*" -arrow-ipc = "*" # for flight_sql test +arrow-cast = "50" +arrow-ipc = "50" # for flight_sql test # TODO: uncomment this when we exclude roapi from root workspace # [profile.release] diff --git a/roapi/src/api/mod.rs b/roapi/src/api/mod.rs index 82a1f5d78..0ecb6fbb8 100644 --- a/roapi/src/api/mod.rs +++ b/roapi/src/api/mod.rs @@ -1,5 +1,3 @@ -use std::convert::TryFrom; - use axum::body::Body; use axum::http::header; use axum::http::Response; diff --git a/roapi/src/config.rs b/roapi/src/config.rs index 74cf03aad..67fdeebda 100644 --- a/roapi/src/config.rs +++ b/roapi/src/config.rs @@ -25,9 +25,17 @@ pub struct FlightSqlTlsConfig { pub client_ca: String, } +#[derive(Deserialize, Default, Clone)] +pub struct BasicAuth { + pub username: String, + pub password: String, +} + #[derive(Deserialize, Default, Clone)] pub struct FlightSqlConfig { pub tls: Option, + pub auth_token: Option, + pub auth_basic: Option, } #[derive(Deserialize, Default, Clone)] @@ -48,80 +56,80 @@ pub struct Config { pub flight_sql_config: Option, } -fn table_arg() -> clap::Arg<'static> { +fn table_arg() -> clap::Arg { clap::Arg::new("table") .help("Table sources to load. Table option can be provided as optional setting as part of the table URI, for example: `blogs=s3://bucket/key,format=delta`. Set table uri to `stdin` if you want to consume table data from stdin as part of a UNIX pipe. If no table_name is provided, a table name will be derived from the filename in URI.") - .takes_value(true) + .num_args(1) .required(false) .number_of_values(1) - .multiple_occurrences(true) + .action(clap::ArgAction::Append) .value_name("[table_name=]uri[,option_key=option_value]") .long("table") .short('t') } -fn address_http_arg() -> clap::Arg<'static> { +fn address_http_arg() -> clap::Arg { clap::Arg::new("addr-http") .help("HTTP endpoint bind address") .required(false) - .takes_value(true) + .num_args(1) .value_name("IP:PORT") .long("addr-http") .short('a') } -fn address_postgres_arg() -> clap::Arg<'static> { +fn address_postgres_arg() -> clap::Arg { clap::Arg::new("addr-postgres") .help("Postgres endpoint bind address") .required(false) - .takes_value(true) + .num_args(1) .value_name("IP:PORT") .long("addr-postgres") .short('p') } -fn address_flight_sql_arg() -> clap::Arg<'static> { +fn address_flight_sql_arg() -> clap::Arg { clap::Arg::new("addr-flight-sql") .help("FlightSQL endpoint bind address") .required(false) - .takes_value(true) + .num_args(1) .value_name("IP:PORT") .long("addr-flight-sql") } -fn read_only_arg() -> clap::Arg<'static> { +fn read_only_arg() -> clap::Arg { clap::Arg::new("disable-read-only") .help("Start roapi in read write mode") .required(false) - .takes_value(false) + .num_args(0) .long("disable-read-only") .short('d') } -fn reload_interval_arg() -> clap::Arg<'static> { +fn reload_interval_arg() -> clap::Arg { clap::Arg::new("reload-interval") .help("maximum age in seconds before triggering rescan and reload of the tables") .required(false) - .takes_value(true) + .num_args(1) .long("reload-interval") .short('r') } -fn response_format_arg() -> clap::Arg<'static> { +fn response_format_arg() -> clap::Arg { clap::Arg::new("response-format") .help("change response serialization: Json (default), Csv, ArrowFile, ArrowStream, Parquet") .required(false) - .takes_value(true) + .num_args(1) .value_name("ResponseFormat") .long("response-format") .short('f') } -fn config_arg() -> clap::Arg<'static> { +fn config_arg() -> clap::Arg { clap::Arg::new("config") .help("config file path") .required(false) - .takes_value(true) + .num_args(1) .long("config") .short('c') } @@ -146,7 +154,7 @@ pub fn get_configuration() -> Result { ]) .get_matches(); - let mut config: Config = match matches.value_of("config") { + let mut config: Config = match matches.get_one::("config") { None => Config::default(), Some(config_path) => { let config_content = whatever!( @@ -169,7 +177,7 @@ pub fn get_configuration() -> Result { } }; - if let Some(tables) = matches.values_of("table") { + if let Some(tables) = matches.get_many::("table") { for v in tables { config.tables.push(whatever!( parse_table_uri_arg(v), @@ -178,32 +186,30 @@ pub fn get_configuration() -> Result { } } - if let Some(addr) = matches.value_of("addr-http") { - config.addr.http = Some(addr.to_string()); + if let Some(addr) = matches.get_one::("addr-http") { + config.addr.http = Some(addr.to_owned()); } - if let Some(addr) = matches.value_of("addr-postgres") { - config.addr.postgres = Some(addr.to_string()); + if let Some(addr) = matches.get_one::("addr-postgres") { + config.addr.postgres = Some(addr.to_owned()); } - if let Some(addr) = matches.value_of("addr-flight-sql") { - config.addr.flight_sql = Some(addr.to_string()); + if let Some(addr) = matches.get_one::("addr-flight-sql") { + config.addr.flight_sql = Some(addr.to_owned()); } - if matches.is_present("disable-read-only") { + if matches.contains_id("disable-read-only") { config.disable_read_only = true; } - if let Some(reload_interval) = matches.value_of("reload-interval") { + if let Some(reload_interval) = matches.get_one::("reload-interval") { if !config.disable_read_only { whatever!("Table reload not supported in read-only mode. Try specify the --disable-read-only option."); } - config.reload_interval = Some(Duration::from_secs( - reload_interval.to_string().parse().unwrap(), - )); + config.reload_interval = Some(Duration::from_secs(reload_interval.to_owned())); } - if let Some(response_format) = matches.value_of("response-format") { + if let Some(response_format) = matches.get_one::("response-format") { config.response_format = whatever!( serde_yaml::from_str(response_format), "Failed parse response-format", diff --git a/roapi/src/error.rs b/roapi/src/error.rs index 1497596d6..35e008061 100644 --- a/roapi/src/error.rs +++ b/roapi/src/error.rs @@ -134,9 +134,13 @@ impl fmt::Display for ApiErrResp { impl axum::response::IntoResponse for ApiErrResp { fn into_response(self) -> axum::response::Response { - let payload = serde_json::to_string(&self).unwrap(); + let payload = serde_json::to_string(&self).expect("failed to serialize json into string"); let body = axum::body::boxed(axum::body::Full::from(payload)); + // NOTE: uncomment for axum 0.7 upgrade + // let payload = serde_json::to_string(&self).expect("failed to serialize json into string"); + // let body = axum::body::Body::from(payload); + Response::builder().status(self.code).body(body).unwrap() } } diff --git a/roapi/src/server/flight_sql.rs b/roapi/src/server/flight_sql.rs index 795f551e9..685d935e3 100644 --- a/roapi/src/server/flight_sql.rs +++ b/roapi/src/server/flight_sql.rs @@ -4,6 +4,7 @@ use arrow_flight::flight_service_server::FlightServiceServer; use arrow_flight::sql::metadata::{ SqlInfoData, SqlInfoDataBuilder, XdbcTypeInfo, XdbcTypeInfoData, XdbcTypeInfoDataBuilder, }; +use arrow_flight::sql::server::PeekableFlightDataStream; use arrow_flight::sql::{ server::FlightSqlService, ActionBeginSavepointRequest, ActionBeginSavepointResult, ActionBeginTransactionRequest, ActionBeginTransactionResult, ActionCancelQueryRequest, @@ -18,15 +19,17 @@ use arrow_flight::sql::{ SqlInfo, TicketStatementQuery, XdbcDataType, }; use arrow_flight::{ - flight_service_server::FlightService, Action, FlightData, FlightDescriptor, FlightEndpoint, - FlightInfo, HandshakeRequest, HandshakeResponse, IpcMessage, SchemaAsIpc, Ticket, + flight_service_server::FlightService, Action, FlightDescriptor, FlightEndpoint, FlightInfo, + HandshakeRequest, HandshakeResponse, IpcMessage, SchemaAsIpc, Ticket, }; use async_trait::async_trait; +use base64::prelude::*; use columnq::arrow_schema::Schema; use columnq::datafusion::arrow::ipc::writer::IpcWriteOptions; use columnq::datafusion::arrow::record_batch::RecordBatch; use columnq::datafusion::logical_expr::LogicalPlan; use columnq::datafusion::prelude::{DataFrame, SessionContext}; +use constant_time_eq::constant_time_eq; use dashmap::DashMap; use futures::{Stream, StreamExt, TryStreamExt}; use log::{debug, info}; @@ -41,14 +44,10 @@ use tonic::transport::{Certificate, Identity, ServerTlsConfig}; use tonic::{Request, Response, Status, Streaming}; use uuid::Uuid; -use crate::config::Config; +use crate::config::{BasicAuth, Config}; use crate::context::RoapiContext; use crate::server::RunnableServer; -// FIXME: uncomment for arrow 0.47 upgrade -// use arrow_flight::sql::server::PeekableFlightDataStream; -type PeekableFlightDataStream = Streaming; - macro_rules! internal_error { ($desc:expr, $err:expr) => { Status::internal(format!("{}: {} at {}:{}", $desc, $err, file!(), line!())) @@ -59,8 +58,9 @@ macro_rules! internal_error { // see: https://docs.rs/datafusion/latest/datafusion/execution/context/struct.SessionState.html#method.resolve_table_references const CATALOG_NAME: &str = "roapi"; const SCHEMA_NAME: &str = "public"; -const FAKE_TOKEN: &str = "uuid_token"; const FAKE_UPDATE_RESULT: i64 = 1; +const AUTH_HEADER: &str = "authorization"; +const BEARER_PREFIX: &str = "Bearer "; static INSTANCE_SQL_DATA: Lazy = Lazy::new(|| { let mut builder = SqlInfoDataBuilder::new(); @@ -108,14 +108,20 @@ pub struct RoapiFlightSqlService { ctx: Arc, statements: Arc>, results: Arc>>, + auth_token: Option, + auth_basic_encoded: Option, } impl RoapiFlightSqlService { - fn new(ctx: Arc) -> Self { + fn new(ctx: Arc, auth_token: Option, auth_basic: Option) -> Self { Self { ctx, statements: Arc::new(DashMap::new()), results: Arc::new(DashMap::new()), + auth_token, + auth_basic_encoded: auth_basic.map(|auth| { + BASE64_STANDARD_NO_PAD.encode(format!("{}:{}", auth.username, auth.password)) + }), } } @@ -125,9 +131,9 @@ impl RoapiFlightSqlService { Ok(self.ctx.get_dfctx().await) } - fn get_result(&self, handle: &str) -> Result, Status> { - if let Some(result) = self.results.get(handle) { - Ok(result.clone()) + fn pop_result(&self, handle: &str) -> Result, Status> { + if let Some((_, result)) = self.results.remove(handle) { + Ok(result) } else { Err(Status::internal(format!( "Request handle not found: {handle}" @@ -154,25 +160,26 @@ impl RoapiFlightSqlService { } fn check_token(&self, req: &Request) -> Result<(), Status> { - let metadata = req.metadata(); - if let Some(auth) = metadata.get("authorization") { - let s = auth + if let Some(token) = &self.auth_token { + let metadata = req.metadata(); + let auth_header = metadata + .get(AUTH_HEADER) + .ok_or_else(|| Status::unauthenticated("token not found"))?; + let auth_header = auth_header .to_str() .map_err(|e| Status::internal(format!("Error parsing header: {e}")))?; - let authorization = s.to_string(); - let bearer = "Bearer "; - if !authorization.starts_with(bearer) { - Err(Status::internal("Invalid auth header!"))?; + if !auth_header.starts_with(BEARER_PREFIX) { + Err(Status::internal("invalid auth type"))?; + } + if auth_header.len() <= BEARER_PREFIX.len() { + return Err(Status::unauthenticated("invalid token")); } - let token = authorization[bearer.len()..].to_string(); - if token == FAKE_TOKEN { - return Ok(()); - } else { - return Err(Status::unauthenticated("invalid token ")); + let user_token = &auth_header[BEARER_PREFIX.len()..]; + if !constant_time_eq(token.as_bytes(), user_token.as_bytes()) { + return Err(Status::unauthenticated("invalid token")); } } - Ok(()) } } @@ -202,18 +209,69 @@ impl FlightSqlService for RoapiFlightSqlService { async fn do_handshake( &self, - _request: Request>, + request: Request>, ) -> Result< Response> + Send>>>, Status, > { + let auth_basic_encoded = self + .auth_basic_encoded + .as_ref() + .ok_or_else(|| Status::unauthenticated("no basic auth cred configured"))?; + + let auth = request + .metadata() + .get(AUTH_HEADER) + .ok_or_else(|| Status::unauthenticated("missing authorization header"))?; + + let (auth_type, auth_value) = auth + .to_str() + .map_err(|_| Status::internal("Failed to parse authorization header"))? + .split_once(' ') + .ok_or_else(|| Status::invalid_argument("Invalid authorization header"))?; + + if auth_type.to_lowercase() != "basic" { + return Err(Status::invalid_argument( + "Invalid authorization type, basic auth is the only supported type", + )); + } + + // auth_value could contain `==` base64 paddings, while auth_basic_encoded doesn't + if auth_basic_encoded.len() > auth_value.len() + || !constant_time_eq( + auth_basic_encoded.as_bytes(), + auth_value[..auth_basic_encoded.len()].as_bytes(), + ) + { + return Err(Status::unauthenticated("unauthorized")); + } + + let token = self + .auth_token + .as_ref() + .ok_or_else(|| Status::internal("token not found"))?; + let auth_header = format!("{}{}", BEARER_PREFIX, &token); let result = HandshakeResponse { protocol_version: 0, - payload: FAKE_TOKEN.into(), + payload: token.clone().into(), }; let result = Ok(result); let output = futures::stream::iter(vec![result]); - return Ok(Response::new(Box::pin(output))); + + let mut resp: Response< + Pin> + Send>>, + > = Response::new(Box::pin(output)); + // Reference implementation returns token in auth header instead of payload, see: + // https://github.com/apache/arrow/blob/6a7a6ee308b69c12f46f874cb3d52892e172d7b7/go/arrow/flight/client.go#L335 + // https://github.com/apache/arrow/blob/6a7a6ee308b69c12f46f874cb3d52892e172d7b7/cpp/src/arrow/flight/transport/grpc/grpc_client.cc#L449C54-L449C65 + resp.metadata_mut().insert( + AUTH_HEADER, + auth_header + .parse() + .map_err(|_| Status::internal("failed to encode auth header"))?, + ); + + Ok(resp) } async fn do_get_fallback( @@ -238,9 +296,9 @@ impl FlightSqlService for RoapiFlightSqlService { let handle = fr.handle; info!("getting results for {handle}"); - let result = self.get_result(&handle)?; + let result = self.pop_result(&handle)?; // if we get an empty result, create an empty schema - let (schema, batches) = match result.get(0) { + let (schema, batches) = match result.first() { None => (Arc::new(Schema::empty()), vec![]), Some(batch) => (batch.schema(), result.clone()), }; @@ -286,7 +344,7 @@ impl FlightSqlService for RoapiFlightSqlService { .map_err(|e| internal_error!("Error executing query", e))?; // if we get an empty result, create an empty schema - let schema = match result.get(0) { + let schema = match result.first() { None => Schema::empty(), Some(batch) => (*batch.schema()).clone(), }; @@ -364,7 +422,7 @@ impl FlightSqlService for RoapiFlightSqlService { .map_err(|e| internal_error!("Error executing query", e))?; // if we get an empty result, create an empty schema - let schema = match result.get(0) { + let schema = match result.first() { None => Schema::empty(), Some(batch) => (*batch.schema()).clone(), }; @@ -632,7 +690,7 @@ impl FlightSqlService for RoapiFlightSqlService { // TODO: ignore SYSTEM TABLE and VIEW let table_type = query .table_types - .get(0) + .first() .map(|s| s.as_str()) .unwrap_or_else(|| "table") .to_string(); @@ -908,28 +966,8 @@ pub struct RoapiFlightSqlServer { pub ctx: Arc, pub addr: std::net::SocketAddr, pub tls_config: Option, -} - -fn tonic_server_builder( - ctx: Arc, - tls_config: &Option, -) -> Result, Whatever> { - let svc = FlightServiceServer::with_interceptor( - RoapiFlightSqlService::new(ctx), - |req: Request<()>| -> Result, Status> { - info!("FlightSQL req: {:?}", &req); - Ok(req) - }, - ); - let mut builder = tonic::transport::Server::builder(); - if let Some(cfg) = tls_config { - builder = whatever!( - builder.tls_config(cfg.clone()), - "Failed to build TLS config" - ); - } - - Ok(builder.add_service(svc)) + pub auth_token: Option, + pub auth_basic: Option, } impl RoapiFlightSqlServer { @@ -962,12 +1000,40 @@ impl RoapiFlightSqlServer { }) .transpose()?; + let auth_basic = config + .flight_sql_config + .as_ref() + .and_then(|c| c.auth_basic.as_ref()) + .cloned(); + let auth_token = match ( + config + .flight_sql_config + .as_ref() + .and_then(|c| c.auth_token.as_ref()) + .cloned(), + &auth_basic, + ) { + (Some(token), None) => Some(token), + // when both basic auth and token auth are specified, handshake will return the + // specified token + (Some(token), Some(_)) => Some(token), + // when only basic auth is specified, handshake will return encoded basic auth + // value as token to keep it constant + (None, Some(BasicAuth { username, password })) => { + let token = BASE64_STANDARD_NO_PAD.encode(format!("{}:{}", username, password)); + Some(token) + } + (None, None) => None, + }; + Ok(Self { ctx, addr: listener .local_addr() .expect("Failed to get address from listener"), tls_config, + auth_token, + auth_basic, }) } } @@ -979,19 +1045,31 @@ impl RunnableServer for RoapiFlightSqlServer { } async fn run(&self) -> Result<(), Whatever> { - let router = tonic_server_builder(self.ctx.clone(), &self.tls_config)?; - let listener = whatever!( - TcpListener::bind(self.addr).await, - "Failed to bind address for FlightSQL server", - ); + let svc = FlightServiceServer::new(RoapiFlightSqlService::new( + self.ctx.clone(), + self.auth_token.clone(), + self.auth_basic.clone(), + )); + + let mut builder = tonic::transport::Server::builder(); + if let Some(cfg) = &self.tls_config { + builder = whatever!( + builder.tls_config(cfg.clone()), + "Failed to build TLS config" + ); + } - let incoming = tonic::transport::server::TcpIncoming::from_listener(listener, true, None) - .expect("Failed to create incoming TCP Router"); + let layer = tower::ServiceBuilder::new() + .layer(tower_http::trace::TraceLayer::new_for_grpc()) + .into_inner(); + + let router = builder.layer(layer).add_service(svc); + + router + .serve(self.addr) + .await + .expect("Failed to bind address for FlightSQL server"); - whatever!( - router.serve_with_incoming(incoming).await, - "Failed to start FlightSQL server on." - ); Ok(()) } } diff --git a/roapi/src/server/http/layers.rs b/roapi/src/server/http/layers.rs index 550810f3e..22af927b7 100644 --- a/roapi/src/server/http/layers.rs +++ b/roapi/src/server/http/layers.rs @@ -2,7 +2,6 @@ use axum::http::uri::Uri; use axum::http::Method; use axum::http::Request; use axum::http::Response; -use hyper::service::Service; use log::error; use log::info; use pin_project_lite::pin_project; @@ -12,6 +11,7 @@ use std::task::Context; use std::task::Poll; use std::time::Instant; use tower_layer::Layer; +use tower_service::Service; #[derive(Clone)] pub struct HttpLoggerLayer {} diff --git a/roapi/src/server/http/mod.rs b/roapi/src/server/http/mod.rs index 5fd4bf46b..1d021b3ca 100644 --- a/roapi/src/server/http/mod.rs +++ b/roapi/src/server/http/mod.rs @@ -20,14 +20,18 @@ pub enum Error { BindTcp { source: std::io::Error }, } +// NOTE: uncomment for axum 0.7 upgrade +// pub type HttpApiServe = axum::serve::Serve; pub type HttpApiServer = axum::Server>; -pub fn build_http_server( +pub async fn build_http_server( ctx_ext: Arc, tables: Arc>>, config: &Config, default_host: String, + // NOTE: uncomment for axum 0.7 upgrade + // ) -> Result<(HttpApiServe, std::net::SocketAddr), Error> { ) -> Result<(HttpApiServer, std::net::SocketAddr), Error> { let default_http_port = std::env::var("PORT").unwrap_or_else(|_| "8080".to_string()); let default_http_addr = [default_host, default_http_port].join(":"); @@ -41,6 +45,8 @@ pub fn build_http_server( let mut app = routes.layer(Extension(ctx_ext)); let cors = tower_http::cors::CorsLayer::new() + // NOTE: uncomment for axum 0.7 upgrade + // .allow_methods([Method::GET, Method::POST, Method::OPTIONS]) .allow_methods(vec![Method::GET, Method::POST, Method::OPTIONS]) .allow_origin(tower_http::cors::Any) .allow_credentials(false); @@ -52,12 +58,19 @@ pub fn build_http_server( app = app.layer(layers::HttpLoggerLayer::new()); } + // NOTE: uncomment for axum 0.7 upgrade + // let listener = tokio::net::TcpListener::bind(http_addr) + // .await + // .context(BindTcpSnafu)?; let listener = TcpListener::bind(http_addr).context(BindTcpSnafu)?; let addr = listener .local_addr() .expect("Failed to get address from listener"); + + // NOTE: uncomment for axum 0.7 upgrade + // let serve = axum::serve(listener, app); + // Ok((serve, addr)) let http_server = axum::Server::from_tcp(listener).unwrap(); let http_server = http_server.serve(app.into_make_service()); - Ok((http_server, addr)) } diff --git a/roapi/src/server/postgres.rs b/roapi/src/server/postgres.rs index 577d92082..86ad2e7bb 100644 --- a/roapi/src/server/postgres.rs +++ b/roapi/src/server/postgres.rs @@ -7,10 +7,10 @@ use std::sync::Arc; use columnq::datafusion::dataframe::DataFrame; use columnq::datafusion::error::DataFusionError; -use columnq::sqlparser::ast::Statement; use convergence::engine::{Engine, Portal}; use convergence::protocol::{ErrorResponse, FieldDescription, SqlState}; use convergence::protocol_ext::DataRowBatch; +use convergence::sqlparser::ast::Statement; use convergence_arrow::table::{record_batch_to_rows, schema_to_field_desc}; use log::info; use snafu::{whatever, Whatever}; diff --git a/roapi/src/startup.rs b/roapi/src/startup.rs index 2e08210ca..5cfa73d69 100644 --- a/roapi/src/startup.rs +++ b/roapi/src/startup.rs @@ -103,6 +103,7 @@ impl Application { &config, default_host, ) + .await .context(BuildHttpServerSnafu)?; Ok(Self { @@ -137,6 +138,7 @@ impl Application { &config, default_host, ) + .await .context(BuildHttpServerSnafu)?; Ok(Self { diff --git a/roapi/tests/flight_sql_test.rs b/roapi/tests/flight_sql_test.rs index 8da190a5c..a23651f5f 100644 --- a/roapi/tests/flight_sql_test.rs +++ b/roapi/tests/flight_sql_test.rs @@ -2,8 +2,6 @@ mod helpers; use arrow_cast::pretty::pretty_format_batches; use arrow_flight::sql::client::FlightSqlServiceClient; -use arrow_flight::utils::flight_data_to_batches; -use arrow_flight::FlightData; use arrow_flight::FlightInfo; use arrow_ipc::convert::try_schema_from_ipc_buffer; use columnq::arrow::datatypes::{DataType, Field}; @@ -42,10 +40,8 @@ async fn flight_info_to_batches( flight_info: FlightInfo, ) -> Vec { let ticket = flight_info.endpoint[0].ticket.as_ref().unwrap().clone(); - let flight_data = client.do_get(ticket).await.unwrap(); - let flight_data: Vec = flight_data.try_collect().await.unwrap(); - - flight_data_to_batches(&flight_data).unwrap() + let response = client.do_get(ticket).await.unwrap(); + response.try_collect().await.unwrap() } async fn spawn_server_for_table(tables: Vec) -> std::net::SocketAddr { diff --git a/roapi/tests/partitioned_table_test.rs b/roapi/tests/partitioned_table_test.rs new file mode 100644 index 000000000..42c334721 --- /dev/null +++ b/roapi/tests/partitioned_table_test.rs @@ -0,0 +1,49 @@ +mod helpers; + +use columnq::datafusion::arrow::datatypes::DataType; +use columnq::table::{TableColumn, TableLoadOption, TableOptionCsv, TableSource}; + +#[tokio::test] +async fn test_partitioned_csv_table() { + let table_path = helpers::test_data_path("partitioned_csv"); + let table = TableSource::new("partitioned_csv".to_string(), table_path) + .with_option(TableLoadOption::csv( + TableOptionCsv::default().with_use_memory_table(false), + )) + .with_partition_columns(vec![ + TableColumn { + name: "year".to_string(), + data_type: DataType::UInt16, + nullable: false, + }, + TableColumn { + name: "month".to_string(), + data_type: DataType::UInt16, + nullable: false, + }, + ]); + + let (app, address) = helpers::test_api_app_with_tables(vec![table]).await; + tokio::spawn(app.run_until_stopped()); + + let response = helpers::http_post( + &format!("{address}/api/sql"), + "SELECT * FROM partitioned_csv ORDER BY ts ASC", + ) + .await; + + let status = response.status(); + let data = response.json::().await.unwrap(); + assert_eq!( + data, + serde_json::json!([ + {"year": 2022, "month": 12, "ts": 100, "value": 0.5}, + {"year": 2022, "month": 12, "ts": 101, "value": 7.8}, + {"year": 2022, "month": 12, "ts": 102, "value": 4.0}, + {"year": 2023, "month": 1, "ts": 201, "value": -1.0}, + {"year": 2023, "month": 1, "ts": 202, "value": 100.0}, + {"year": 2023, "month": 1, "ts": 203, "value": 0.0}, + ]) + ); + assert_eq!(status, 200); +} diff --git a/test_data/excel_range.ods b/test_data/excel_range.ods new file mode 100644 index 000000000..7f596e276 Binary files /dev/null and b/test_data/excel_range.ods differ diff --git a/test_data/partitioned_csv/year=2022/month=12/p001.csv b/test_data/partitioned_csv/year=2022/month=12/p001.csv new file mode 100644 index 000000000..3c9554cff --- /dev/null +++ b/test_data/partitioned_csv/year=2022/month=12/p001.csv @@ -0,0 +1,4 @@ +ts,value +100,0.5 +101,7.8 +102,4 diff --git a/test_data/partitioned_csv/year=2023/month=1/p001.csv b/test_data/partitioned_csv/year=2023/month=1/p001.csv new file mode 100644 index 000000000..51cdd3917 --- /dev/null +++ b/test_data/partitioned_csv/year=2023/month=1/p001.csv @@ -0,0 +1,4 @@ +ts,value +201,-1 +202,100.0 +203,0 diff --git a/test_end_to_end/query_blogs.sh b/test_end_to_end/query_blogs.sh index e2e0ed5a7..ddbf8d199 100755 --- a/test_end_to_end/query_blogs.sh +++ b/test_end_to_end/query_blogs.sh @@ -1,57 +1,31 @@ #!/bin/bash -http_status=$(curl -o /dev/null -s -w "%{http_code}" -X POST -d "SELECT count(1) from s3_blogs" 127.0.0.1:8000/api/sql) +set -eu -echo $http_status -if [[ $http_status != "200" ]] -then - echo "error" - exit 1 -else - echo "success" -fi +function check_status() { + http_status=$1 + echo $http_status + if [[ $http_status != "200" ]] + then + echo "error" + exit 1 + else + echo "success" + fi +} -http_status=$(curl -o /dev/null -s -w "%{http_code}" -X POST -d "SELECT count(1) from s3_blogs_space_encode" 127.0.0.1:8000/api/sql) +SQL_ENDPOINT="127.0.0.1:8000/api/sql" -echo $http_status -if [[ $http_status != "200" ]] -then - echo "error" - exit 1 -else - echo "success" -fi +http_status=$(curl -o /dev/null -s -w "%{http_code}" -X POST -d "SELECT count(1) from s3_blogs" "${SQL_ENDPOINT}") +check_status ${http_status} -http_status=$(curl -o /dev/null -s -w "%{http_code}" -X POST -d "SELECT count(1) from s3_blogs_dir" 127.0.0.1:8000/api/sql) +http_status=$(curl -o /dev/null -s -w "%{http_code}" -X POST -d "SELECT count(1) from s3_blogs_space_encode" "${SQL_ENDPOINT}") +check_status ${http_status} -echo $http_status -if [[ $http_status != "200" ]] -then - echo "error" - exit 1 -else - echo "success" -fi +http_status=$(curl -o /dev/null -s -w "%{http_code}" -X POST -d "SELECT count(1) from s3_blogs_dir" "${SQL_ENDPOINT}") +check_status ${http_status} -http_status=$(curl -o /dev/null -s -w "%{http_code}" -X POST -d "SELECT count(1) from gcs_blogs" 127.0.0.1:8000/api/sql) +http_status=$(curl -o /dev/null -s -w "%{http_code}" -X POST -d "SELECT count(1) from gcs_blogs" "${SQL_ENDPOINT}") +check_status ${http_status} -echo $http_status -if [[ $http_status != "200" ]] -then - echo "error" - exit 1 -else - echo "success" -fi - -http_status=$(curl -o /dev/null -s -w "%{http_code}" -X POST -d "SELECT count(1) from azure_blogs" 127.0.0.1:8000/api/sql) - -echo $http_status -if [[ $http_status != "200" ]] -then - echo "error" - exit 1 -else - echo "success" -fi - -exit 0 +http_status=$(curl -o /dev/null -s -w "%{http_code}" -X POST -d "SELECT count(1) from azure_blogs" "${SQL_ENDPOINT}") +check_status ${http_status}